@desklink/host 0.2.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/NOTICE +26 -0
  2. package/README.md +134 -56
  3. package/bin/desklink-host.mjs +13 -5
  4. package/dist/bridge.d.ts +1 -0
  5. package/dist/bridge.d.ts.map +1 -1
  6. package/dist/bridge.js +12 -4
  7. package/dist/bridge.js.map +1 -1
  8. package/dist/engineProcess.d.ts +6 -1
  9. package/dist/engineProcess.d.ts.map +1 -1
  10. package/dist/engineProcess.js +35 -4
  11. package/dist/engineProcess.js.map +1 -1
  12. package/dist/protocol.d.ts +58 -0
  13. package/dist/protocol.d.ts.map +1 -1
  14. package/dist/protocol.js.map +1 -1
  15. package/dist/resolveEngine.d.ts +7 -1
  16. package/dist/resolveEngine.d.ts.map +1 -1
  17. package/dist/resolveEngine.js +66 -14
  18. package/dist/resolveEngine.js.map +1 -1
  19. package/docs/BOUNDARY.md +15 -7
  20. package/docs/PROTOCOL.md +267 -27
  21. package/engine/Cargo.lock +47 -2
  22. package/engine/Cargo.toml +18 -13
  23. package/engine/build.rs +86 -4
  24. package/engine/native/agent_overlay_mac.m +51 -4
  25. package/engine/native/agent_overlay_wayland.c +145 -12
  26. package/engine/native/agent_overlay_x11.c +72 -0
  27. package/engine/native/nvenc_shim.c +453 -0
  28. package/engine/native/vaapi_shim.c +727 -0
  29. package/engine/native/vpx_shim.c +54 -4
  30. package/engine/native/vt_shim.c +448 -0
  31. package/engine/src/capture.rs +167 -58
  32. package/engine/src/capture_mac.rs +24 -14
  33. package/engine/src/capture_win.rs +23 -0
  34. package/engine/src/clipboard_win.rs +8 -0
  35. package/engine/src/convert.rs +286 -61
  36. package/engine/src/cursor.rs +118 -0
  37. package/engine/src/encoder.rs +122 -13
  38. package/engine/src/h264.rs +188 -0
  39. package/engine/src/h264_nvenc.rs +308 -0
  40. package/engine/src/h264_openh264.rs +735 -0
  41. package/engine/src/h264_vaapi.rs +263 -0
  42. package/engine/src/h264_vt.rs +368 -0
  43. package/engine/src/indicator.rs +104 -3
  44. package/engine/src/input_win.rs +91 -0
  45. package/engine/src/keymap_win.rs +39 -0
  46. package/engine/src/mac.rs +34 -9
  47. package/engine/src/main.rs +186 -26
  48. package/engine/src/peer.rs +466 -28
  49. package/engine/src/portal.rs +42 -4
  50. package/engine/src/portal_win.rs +37 -0
  51. package/engine/src/protocol.rs +64 -2
  52. package/engine/src/session.rs +1252 -105
  53. package/engine/src/win.rs +110 -0
  54. package/engine/src/x11.rs +225 -9
  55. package/engine/src/x11_mac.rs +12 -0
  56. package/engine/vendor/libva/COPYING +19 -0
  57. package/engine/vendor/libva/va/va.h +5450 -0
  58. package/engine/vendor/libva/va/va_dec_av1.h +695 -0
  59. package/engine/vendor/libva/va/va_dec_hevc.h +616 -0
  60. package/engine/vendor/libva/va/va_dec_jpeg.h +157 -0
  61. package/engine/vendor/libva/va/va_dec_vp8.h +249 -0
  62. package/engine/vendor/libva/va/va_dec_vp9.h +312 -0
  63. package/engine/vendor/libva/va/va_dec_vvc.h +688 -0
  64. package/engine/vendor/libva/va/va_drm.h +61 -0
  65. package/engine/vendor/libva/va/va_enc_av1.h +1010 -0
  66. package/engine/vendor/libva/va/va_enc_h264.h +659 -0
  67. package/engine/vendor/libva/va/va_enc_hevc.h +958 -0
  68. package/engine/vendor/libva/va/va_enc_jpeg.h +162 -0
  69. package/engine/vendor/libva/va/va_enc_mpeg2.h +308 -0
  70. package/engine/vendor/libva/va/va_enc_vp8.h +350 -0
  71. package/engine/vendor/libva/va/va_enc_vp9.h +639 -0
  72. package/engine/vendor/libva/va/va_fei.h +170 -0
  73. package/engine/vendor/libva/va/va_fei_h264.h +505 -0
  74. package/engine/vendor/libva/va/va_fei_hevc.h +254 -0
  75. package/engine/vendor/libva/va/va_prot.h +525 -0
  76. package/engine/vendor/libva/va/va_version.h +87 -0
  77. package/engine/vendor/libva/va/va_vpp.h +1590 -0
  78. package/engine/vendor/nv-codec-headers/LICENSE +24 -0
  79. package/engine/vendor/nv-codec-headers/README +7 -0
  80. package/engine/vendor/nv-codec-headers/include/ffnvcodec/nvEncodeAPI.h +3825 -0
  81. package/examples/reference-client.html +1 -1
  82. package/package.json +4 -3
@@ -8,16 +8,18 @@
8
8
  use crate::capture::{self, Capture};
9
9
  use crate::clipboard;
10
10
  use crate::convert::{fit, I420};
11
- use crate::encoder::Encoder;
11
+ use crate::encoder::{Encoder, VideoEncoder};
12
12
  use crate::input::{Button, HeldState, InputDevices};
13
13
  use crate::keymap::{self, Layout};
14
- use crate::peer::{h264_profile_level_id, PeerEvent, TransportOptions, VideoCodec, VideoPeer};
14
+ use crate::peer::{
15
+ h264_profile_level_id, PathReport, PeerEvent, TransportOptions, VideoCodec, VideoPeer,
16
+ };
15
17
  use crate::portal::{self, SelectedSource};
16
18
  use crate::protocol::{
17
19
  ControlMessage, ControlReply, EncodedCodec, Permission, PointerPhase, SourceRequest,
18
20
  };
19
21
  use crate::x11::X11Desktop;
20
- use anyhow::Result;
22
+ use anyhow::{Context, Result};
21
23
  use std::sync::atomic::{AtomicBool, Ordering};
22
24
  use std::sync::{Arc, Condvar, Mutex};
23
25
  use std::time::{Duration, Instant};
@@ -51,14 +53,41 @@ pub struct Metrics {
51
53
  pub dropped_frames: u64,
52
54
  pub encoded_frames: u64,
53
55
  pub encoded_bytes: u64,
54
- /// Key frames sent: the first, and one per receiver request.
56
+ /// Key frames sent: the first, one per receiver request, and one each time
57
+ /// motion starts or stops being coded at half size.
55
58
  pub key_frames: u64,
56
59
  /// Refinement passes: a still desktop re-coded once, sharp.
57
60
  pub refined_frames: u64,
61
+ /// Motion frames coded at half size because full size could not keep up.
62
+ pub halved_frames: u64,
63
+ /// Fresh captures coded at the pace of change: a motion pass on a frame
64
+ /// the encoder had not coded yet — the population `queue_micros` is over.
65
+ pub motion_frames: u64,
58
66
  /// Time spent in the encoder, in microseconds, over every encoded frame.
59
67
  pub encode_micros: u64,
68
+ /// Time spent grabbing and converting a frame on the capture thread, in
69
+ /// microseconds, over every frame forwarded to the encoder. On X11 this
70
+ /// is `GetImage` plus the downscale to I420; unchanged polls are hashed,
71
+ /// recognised, and never forwarded, so their grab cost stays out; the lab
72
+ /// harness reads it as the grab cost L1 owns.
73
+ pub capture_micros: u64,
74
+ /// Time spent hashing a captured frame and working out its damage on the
75
+ /// capture thread, in microseconds, over every forwarded frame; the hash
76
+ /// branch runs only when `local_frames` is on.
77
+ pub convert_micros: u64,
78
+ /// Time a freshly captured frame waited between the capture handoff and
79
+ /// the start of its encode, in microseconds, over motion frames.
80
+ pub queue_micros: u64,
81
+ /// Time spent packetizing a frame and handing it to the transport, in
82
+ /// microseconds, over every sent frame.
83
+ pub send_micros: u64,
60
84
  /// The current rate target, after any loss back-off.
61
85
  pub target_kbps: u32,
86
+ /// The codec frames go out in, `vp9` or `h264`, and what codes them
87
+ /// (`libvpx`, `nvenc`, `vaapi`, `openh264`, `videotoolbox`); empty for an
88
+ /// encoded source, whose consumer is the encoder.
89
+ pub codec: &'static str,
90
+ pub encoder: &'static str,
62
91
  pub input_applied: u64,
63
92
  pub input_rejected: u64,
64
93
  /// Access units the consumer fed to an encoded source.
@@ -173,18 +202,21 @@ struct Selected {
173
202
 
174
203
  /// Open an X display, start reading its root window, and hand back the same
175
204
  /// shape the portal path produces.
205
+ #[allow(clippy::too_many_arguments)]
176
206
  fn select_x11(
177
207
  display: Option<&str>,
178
208
  max_width: usize,
179
209
  max_height: usize,
180
210
  max_fps: u32,
181
211
  metrics: Arc<Mutex<Metrics>>,
212
+ #[cfg(target_os = "linux")] cursor_sink: Option<crate::cursor::CursorSink>,
182
213
  sink: capture::FrameSink,
183
214
  on_stop: Box<dyn Fn(String) + Send>,
184
215
  ) -> Result<Selected> {
185
216
  let desktop = Arc::new(Mutex::new(X11Desktop::connect(display)?));
186
217
  let (width, height) = {
187
218
  let desktop = lock(&desktop);
219
+ eprintln!("x11 capture path: {}", desktop.capture_path());
188
220
  desktop.screen_size()
189
221
  };
190
222
  let source = SelectedSource {
@@ -207,26 +239,81 @@ fn select_x11(
207
239
  .spawn(move || {
208
240
  let interval = frame_interval(max_fps);
209
241
  let mut sequence = 0u64;
210
- // An X server has no damage signal here, so an unchanged screen
211
- // is recognised by its pixels: handing it on would keep the
212
- // encoder busy and a still desktop would never be refined.
242
+ let mut last_sent: Option<Instant> = None;
243
+ // XDamage wakes the loop, but pixels decide whether the root
244
+ // image actually changed; without XDamage, polling does so.
245
+ let hashed = !lock(&desktop).damage_armed();
213
246
  let mut last_hash = None;
247
+ // The first frame is always grabbed; afterwards Damage wakes
248
+ // the loop, so a still screen costs no grab, no conversion
249
+ // and no hash. The fps cap stays a minimum spacing.
250
+ let mut first = true;
251
+ #[cfg(target_os = "linux")]
252
+ let mut cursor = cursor_sink.map(crate::cursor::Reporter::new);
214
253
  while !stop.load(Ordering::SeqCst) {
254
+ #[cfg(target_os = "linux")]
255
+ if let Some(cursor) = cursor.as_mut() {
256
+ if let Ok((x, y, visible)) = lock(&desktop).cursor_position() {
257
+ cursor.update(x, y, visible);
258
+ }
259
+ }
260
+ if !first {
261
+ let changed = lock(&desktop).take_damage();
262
+ if !changed {
263
+ std::thread::sleep(interval);
264
+ continue;
265
+ }
266
+ if let Some(sent) = last_sent {
267
+ let elapsed = sent.elapsed();
268
+ if elapsed < interval {
269
+ std::thread::sleep(interval - elapsed);
270
+ }
271
+ }
272
+ }
215
273
  let started = Instant::now();
274
+ let grabbed = Instant::now();
216
275
  let frame = {
217
276
  let mut desktop = lock(&desktop);
218
277
  desktop.capture_with_pixels(max_width, max_height)
219
278
  };
279
+ let grab_micros = grabbed.elapsed().as_micros() as u64;
220
280
  match frame {
221
281
  Ok((frame, raw)) => {
222
- use std::hash::{Hash, Hasher};
223
- let mut hasher = std::collections::hash_map::DefaultHasher::new();
224
- raw.hash(&mut hasher);
225
- let hash = hasher.finish();
226
- if last_hash != Some(hash) {
227
- last_hash = Some(hash);
228
- sink(frame, sequence, raw);
282
+ let changed = {
283
+ use std::hash::{Hash, Hasher};
284
+ let mut hasher = std::collections::hash_map::DefaultHasher::new();
285
+ raw.hash(&mut hasher);
286
+ let hash = hasher.finish();
287
+ if last_hash == Some(hash) {
288
+ false
289
+ } else {
290
+ last_hash = Some(hash);
291
+ true
292
+ }
293
+ };
294
+ if changed {
295
+ // Handing an unchanged frame on would keep the
296
+ // encoder busy and a still desktop would never
297
+ // be refined.
298
+ if let Ok(mut m) = captured.lock() {
299
+ m.capture_micros += grab_micros;
300
+ }
301
+ let (w, h) = (frame.width, frame.height);
302
+ let (screen_w, screen_h) = (width as usize, height as usize);
303
+ sink(frame, sequence, &mut || {
304
+ crate::convert::to_bgrx(
305
+ &raw,
306
+ screen_w,
307
+ screen_h,
308
+ screen_w * 4,
309
+ crate::convert::PixelFormat::Bgrx,
310
+ w,
311
+ h,
312
+ )
313
+ });
229
314
  sequence += 1;
315
+ last_sent = Some(started);
316
+ first = false;
230
317
  }
231
318
  }
232
319
  Err(error) => {
@@ -239,11 +326,21 @@ fn select_x11(
239
326
  on_stop(format!("X11 capture stopped: {error}"));
240
327
  break;
241
328
  }
329
+ if !hashed {
330
+ let elapsed = started.elapsed();
331
+ if elapsed < interval {
332
+ std::thread::sleep(interval - elapsed);
333
+ }
334
+ }
242
335
  }
243
336
  }
244
- let elapsed = started.elapsed();
245
- if elapsed < interval {
246
- std::thread::sleep(interval - elapsed);
337
+ // With Damage the fps cap paced the grab above; without
338
+ // it the loop keeps its old trailing sleep.
339
+ if hashed {
340
+ let elapsed = started.elapsed();
341
+ if elapsed < interval {
342
+ std::thread::sleep(interval - elapsed);
343
+ }
247
344
  }
248
345
  }
249
346
  })
@@ -391,10 +488,10 @@ impl InputTarget {
391
488
  Ok(())
392
489
  }
393
490
 
394
- #[cfg(target_os = "macos")]
491
+ #[cfg(any(target_os = "macos", target_os = "windows"))]
395
492
  fn unicode_text(&mut self, text: &str) -> Result<()> {
396
493
  match &mut self.applier {
397
- #[cfg(target_os = "macos")]
494
+ #[cfg(any(target_os = "macos", target_os = "windows"))]
398
495
  Applier::Uinput(devices) => devices.unicode_text(text),
399
496
  #[cfg(target_os = "linux")]
400
497
  Applier::Uinput(_) => anyhow::bail!("Unicode text input is unavailable on Linux"),
@@ -493,6 +590,7 @@ impl SessionError {
493
590
  }
494
591
 
495
592
  pub struct OpenRequest {
593
+ pub cursor: crate::protocol::CursorMode,
496
594
  /** Which desktop to capture; absent means the portal. */
497
595
  pub source: Option<SourceRequest>,
498
596
  pub permissions: Vec<Permission>,
@@ -506,11 +604,17 @@ pub struct OpenRequest {
506
604
  /// Offer a passive ICE-TCP candidate on the loopback, for a forwarded client.
507
605
  pub loopback_tcp: bool,
508
606
  pub agent_indicator: bool,
607
+ /// Keep the latest frame and its tile hashes for `session.frame` and
608
+ /// `session.frame.changed`; off, a frame is converted once, for the encoder.
609
+ pub local_frames: bool,
509
610
  }
510
611
 
511
612
  /// Events a session raises for its consumer.
512
613
  #[derive(Debug)]
513
614
  pub enum SessionEvent {
615
+ Cursor {
616
+ position: crate::protocol::CursorPosition,
617
+ },
514
618
  Description {
515
619
  generation: u64,
516
620
  sdp: String,
@@ -527,6 +631,7 @@ pub enum SessionEvent {
527
631
  first_frame: bool,
528
632
  },
529
633
  Frame {
634
+ timestamp_us: u64,
530
635
  seq: u64,
531
636
  damage: Vec<[usize; 4]>,
532
637
  },
@@ -546,8 +651,13 @@ pub enum SessionEvent {
546
651
  CaptureStopped {
547
652
  reason: String,
548
653
  },
654
+ /// The session ended on the engine's side. `code` says whether a new
655
+ /// session may recover it: `transport` is the path to the consumer's peer
656
+ /// lost, which a fresh `session.open` can heal; `lease` and `error` are
657
+ /// the session's authority ending or a fault, which it cannot.
549
658
  Revoked {
550
659
  reason: String,
660
+ code: &'static str,
551
661
  },
552
662
  }
553
663
 
@@ -564,6 +674,13 @@ type FrameSnapshot = (u64, Instant, usize, usize, Vec<u8>, Vec<u64>, Vec<u64>);
564
674
 
565
675
  /// Everything a session's background tasks need, shared rather than borrowed so
566
676
  /// the consumer can own the `Session` handle while the pipeline runs.
677
+ #[cfg(target_os = "linux")]
678
+ enum PointDesktop {
679
+ X11(String),
680
+ Wayland,
681
+ Unavailable(&'static str),
682
+ }
683
+
567
684
  struct Inner {
568
685
  id: String,
569
686
  generation: u64,
@@ -579,9 +696,12 @@ struct Inner {
579
696
  /// offer has to name the profile the stream's own SPS declares.
580
697
  peer: Mutex<Option<Arc<VideoPeer>>>,
581
698
  /// Absent for an encoded source: the consumer is the encoder.
582
- encoder: Mutex<Option<Encoder>>,
699
+ encoder: Mutex<Option<VideoEncoder>>,
583
700
  input: Mutex<Option<InputTarget>>,
584
701
  indicator: Mutex<Option<crate::indicator::Indicator>>,
702
+ point_overlay: Mutex<Option<crate::indicator::Indicator>>,
703
+ #[cfg(target_os = "linux")]
704
+ point_desktop: PointDesktop,
585
705
  capture: Mutex<Option<FrameSource>>,
586
706
  /// Absent for an encoded source: there is no local keyboard to translate a
587
707
  /// character through, because the client's keys are forwarded instead.
@@ -595,6 +715,9 @@ struct Inner {
595
715
  events: tokio_mpsc::UnboundedSender<Notice>,
596
716
  /// Present only for an encoded source.
597
717
  encoded: Mutex<Option<Encoded>>,
718
+ /// `OpenRequest::local_frames`: whether `latest` is kept at all.
719
+ local_frames: bool,
720
+ cursor_info: serde_json::Value,
598
721
  }
599
722
 
600
723
  /// A source whose video arrives already encoded.
@@ -673,9 +796,11 @@ pub fn capabilities() -> serde_json::Value {
673
796
  "backends": ["portal-screencast+pipewire", "x11-root"],
674
797
  "formats": ["bgrx", "bgra", "rgbx", "rgba"],
675
798
  "cursor": "embedded",
799
+ "cursor_modes": ["embedded", "hidden"],
800
+ "cursor_positions": {"x11": true, "portal": "negotiated"},
676
801
  "audio": false,
677
802
  },
678
- "encode": { "codecs": ["vp9"], "hardware": false },
803
+ "encode": encode_capabilities(),
679
804
  "input": {
680
805
  "mechanism": "inputtino/uinput",
681
806
  "pointer": input,
@@ -725,6 +850,7 @@ pub fn capabilities() -> serde_json::Value {
725
850
  "backends": ["screencapturekit"],
726
851
  "formats": ["bgra"],
727
852
  "cursor": "embedded",
853
+ "cursor_modes": ["embedded"],
728
854
  "audio": false,
729
855
  "displays": crate::mac::displays(),
730
856
  "grant": if capture_granted { "granted" } else { "missing-screen-recording" },
@@ -733,7 +859,7 @@ pub fn capabilities() -> serde_json::Value {
733
859
  "remedy": format!("Allow {} in System Settings › Privacy & Security › Screen & System Audio Recording, then reconnect.", crate::mac::tcc_responsible_app_name())
734
860
  }) }
735
861
  },
736
- "encode": { "codecs": ["vp9"], "hardware": false },
862
+ "encode": encode_capabilities(),
737
863
  "input": {
738
864
  "mechanism": "quartz-cgevent",
739
865
  "pointer": input,
@@ -750,6 +876,34 @@ pub fn capabilities() -> serde_json::Value {
750
876
  })
751
877
  }
752
878
 
879
+ #[cfg(target_os = "windows")]
880
+ pub fn capabilities() -> serde_json::Value {
881
+ crate::win::capabilities()
882
+ }
883
+
884
+ #[cfg(target_os = "windows")]
885
+ fn wayland_clipboard_available() -> bool {
886
+ false
887
+ }
888
+
889
+ /// `capabilities.encode`: VP9 always, H.264 when an encoder for it starts
890
+ /// here now. `hardware` describes VP9, the default.
891
+ fn encode_capabilities() -> serde_json::Value {
892
+ let h264 = crate::h264::probe();
893
+ let mut codecs = vec!["vp9"];
894
+ if h264.is_some() {
895
+ codecs.push("h264");
896
+ }
897
+ serde_json::json!({
898
+ "codecs": codecs,
899
+ "hardware": false,
900
+ "h264": {
901
+ "encoder": h264.map(|info| info.encoder),
902
+ "hardware": h264.is_some_and(|info| info.hardware),
903
+ },
904
+ })
905
+ }
906
+
753
907
  impl Session {
754
908
  pub async fn open(
755
909
  request: OpenRequest,
@@ -761,6 +915,26 @@ impl Session {
761
915
  "session.open requires the view permission",
762
916
  ));
763
917
  }
918
+ #[cfg(target_os = "windows")]
919
+ if request.agent_indicator {
920
+ return Err(SessionError::new(
921
+ "indicator-unavailable",
922
+ "Windows agent indicator is not implemented in this build.",
923
+ ));
924
+ }
925
+ if request.cursor == crate::protocol::CursorMode::Hidden {
926
+ #[cfg(not(target_os = "linux"))]
927
+ return Err(SessionError::new(
928
+ "cursor-unavailable",
929
+ "hidden cursor capture is not implemented on this platform",
930
+ ));
931
+ if matches!(request.source, Some(SourceRequest::Encoded { .. })) {
932
+ return Err(SessionError::new(
933
+ "cursor-unavailable",
934
+ "the engine cannot remove cursor pixels from consumer-encoded video",
935
+ ));
936
+ }
937
+ }
764
938
  let id = opaque_id();
765
939
  let wants_control = request.permissions.contains(&Permission::Control);
766
940
  let wants_x11 = matches!(request.source, Some(SourceRequest::X11 { .. }));
@@ -832,9 +1006,18 @@ impl Session {
832
1006
  let capture_status = Arc::new(on_capture_status);
833
1007
  let frame_events = events.clone();
834
1008
  let frame_session_id = id.clone();
835
- let sink = Box::new(move |frame: I420, _seq: u64, raw: Vec<u8>| {
836
- let hashes = tile_hashes(&raw, frame.width, frame.height);
837
- if let Ok(mut held) = observed.lock() {
1009
+ let local_frames = request.local_frames;
1010
+ let sink = Box::new(move |frame: I420, _seq: u64, pixels: capture::Pixels| {
1011
+ // The frame's RTP time: when capture handed it over, not when it
1012
+ // was coded, so the receiver never sees encode jitter as cadence.
1013
+ // ponytail: stamped after the backend's grab and conversion; a
1014
+ // grab-start instant from each backend needs a FrameSink change.
1015
+ let captured_at = Instant::now();
1016
+ let raw = if local_frames { pixels() } else { None };
1017
+ let hashes = raw
1018
+ .as_ref()
1019
+ .map(|raw| tile_hashes(raw, frame.width, frame.height));
1020
+ if let (Some(raw), Some(hashes), Ok(mut held)) = (raw, hashes, observed.lock()) {
838
1021
  let unchanged =
839
1022
  held.as_ref()
840
1023
  .is_some_and(|(_, _, width, height, _, current, _)| {
@@ -849,7 +1032,7 @@ impl Session {
849
1032
  let damage = dirty_regions(&hashes, &previous, frame.width, frame.height);
850
1033
  *held = Some((
851
1034
  seq,
852
- Instant::now(),
1035
+ captured_at,
853
1036
  frame.width,
854
1037
  frame.height,
855
1038
  raw,
@@ -859,22 +1042,42 @@ impl Session {
859
1042
  frame_tx_signal.send_replace(seq);
860
1043
  let _ = frame_events.send(Notice {
861
1044
  session_id: frame_session_id.clone(),
862
- event: SessionEvent::Frame { seq, damage },
1045
+ event: SessionEvent::Frame {
1046
+ seq,
1047
+ damage,
1048
+ timestamp_us: crate::protocol::timestamp_us(captured_at),
1049
+ },
863
1050
  });
864
1051
  }
865
1052
  }
866
1053
  if let Ok(mut m) = captured.lock() {
867
1054
  m.captured_frames += 1;
1055
+ m.convert_micros += captured_at.elapsed().as_micros() as u64;
868
1056
  }
869
1057
  // An encoder that is behind gets the newest frame, not a queue: a
870
1058
  // desktop stream is live, and the last state is the one that counts.
871
- if frame_tx.put(frame) {
1059
+ if frame_tx.put(frame, captured_at) {
872
1060
  if let Ok(mut m) = captured.lock() {
873
1061
  m.dropped_frames += 1;
874
1062
  }
875
1063
  }
876
1064
  });
877
1065
 
1066
+ #[cfg(target_os = "linux")]
1067
+ let cursor_sink: Option<crate::cursor::CursorSink> =
1068
+ if request.cursor == crate::protocol::CursorMode::Hidden {
1069
+ let events = events.clone();
1070
+ let id = id.clone();
1071
+ Some(Box::new(move |position| {
1072
+ let _ = events.send(Notice {
1073
+ session_id: id.clone(),
1074
+ event: SessionEvent::Cursor { position },
1075
+ });
1076
+ }))
1077
+ } else {
1078
+ None
1079
+ };
1080
+ let mut cursor_info = serde_json::json!({"mode":"embedded", "positions":false});
878
1081
  let Selected {
879
1082
  source,
880
1083
  capture,
@@ -886,6 +1089,8 @@ impl Session {
886
1089
  request.max_height,
887
1090
  max_fps,
888
1091
  metrics.clone(),
1092
+ #[cfg(target_os = "linux")]
1093
+ cursor_sink,
889
1094
  sink,
890
1095
  Box::new({
891
1096
  let status = capture_status.clone();
@@ -898,9 +1103,30 @@ impl Session {
898
1103
  Some(SourceRequest::Display { display_id }) => {
899
1104
  portal::open_display(*display_id).await
900
1105
  }
901
- _ => portal::open(request.restore_token.as_deref()).await,
1106
+ _ => {
1107
+ #[cfg(target_os = "linux")]
1108
+ {
1109
+ portal::open(request.restore_token.as_deref(), request.cursor).await
1110
+ }
1111
+ #[cfg(not(target_os = "linux"))]
1112
+ {
1113
+ portal::open(request.restore_token.as_deref()).await
1114
+ }
1115
+ }
1116
+ }
1117
+ .map_err(|error| {
1118
+ let message = format!("{error:#}");
1119
+ let code = if message.contains("cursor_positions_unavailable:") {
1120
+ "cursor_positions_unavailable"
1121
+ } else {
1122
+ "source"
1123
+ };
1124
+ SessionError::new(code, message)
1125
+ })?;
1126
+ #[cfg(target_os = "linux")]
1127
+ {
1128
+ cursor_info = serde_json::json!({"mode":portal.cursor_mode, "positions":false});
902
1129
  }
903
- .map_err(|error| SessionError::new("source", format!("{error:#}")))?;
904
1130
  if let Some(token) = &portal.restore_token {
905
1131
  let _ = events.send(Notice {
906
1132
  session_id: id.clone(),
@@ -945,12 +1171,26 @@ impl Session {
945
1171
  }),
946
1172
  )
947
1173
  .map_err(|error| SessionError::new("source", format!("{error:#}")))?;
1174
+ #[cfg(target_os = "windows")]
1175
+ let capture = capture::start(
1176
+ portal,
1177
+ width,
1178
+ height,
1179
+ max_fps,
1180
+ sink,
1181
+ Box::new({
1182
+ let status = capture_status.clone();
1183
+ move |running, reason| status(running, reason)
1184
+ }),
1185
+ )
1186
+ .map_err(|error| SessionError::new("source", format!("{error:#}")))?;
948
1187
  #[cfg(target_os = "linux")]
949
1188
  let capture = capture::start(
950
1189
  portal,
951
1190
  width,
952
1191
  height,
953
1192
  max_fps,
1193
+ cursor_sink,
954
1194
  sink,
955
1195
  indicator
956
1196
  .as_ref()
@@ -962,6 +1202,18 @@ impl Session {
962
1202
  )
963
1203
  .map_err(|error| SessionError::new("source", format!("{error:#}")))?;
964
1204
  #[cfg(target_os = "linux")]
1205
+ {
1206
+ if request.cursor == crate::protocol::CursorMode::Hidden
1207
+ && !capture.cursor_positions()
1208
+ {
1209
+ return Err(SessionError::new(
1210
+ "cursor_positions_unavailable",
1211
+ "portal stream did not provide SPA cursor metadata",
1212
+ ));
1213
+ }
1214
+ cursor_info["positions"] = serde_json::json!(capture.cursor_positions());
1215
+ }
1216
+ #[cfg(target_os = "linux")]
965
1217
  if let Some(indicator) = indicator.as_mut() {
966
1218
  indicator.event('A', -1, -1);
967
1219
  }
@@ -972,6 +1224,9 @@ impl Session {
972
1224
  }
973
1225
  }
974
1226
  };
1227
+ if wants_x11 {
1228
+ cursor_info = serde_json::json!({"mode":"hidden", "positions": request.cursor == crate::protocol::CursorMode::Hidden});
1229
+ }
975
1230
 
976
1231
  let source_w = source.width.max(1) as usize;
977
1232
  let source_h = source.height.max(1) as usize;
@@ -1023,7 +1278,7 @@ impl Session {
1023
1278
  applier: Applier::Uinput({
1024
1279
  #[cfg(target_os = "linux")]
1025
1280
  let devices = InputDevices::create(source_w as i32, source_h as i32);
1026
- #[cfg(target_os = "macos")]
1281
+ #[cfg(any(target_os = "macos", target_os = "windows"))]
1027
1282
  let devices = InputDevices::create_for_display(
1028
1283
  source_w as i32,
1029
1284
  source_h as i32,
@@ -1046,6 +1301,16 @@ impl Session {
1046
1301
  SessionError::new("input", format!("no keyboard layout: {error:#}"))
1047
1302
  })?;
1048
1303
 
1304
+ // H.264 is offered beside VP9 only when an encoder for it starts on
1305
+ // this machine; the answer then decides which one is coded.
1306
+ let codec = if !crate::h264::fits(width, height) {
1307
+ VideoCodec::Vp9
1308
+ } else {
1309
+ match tokio::task::spawn_blocking(crate::h264::prepare).await {
1310
+ Ok(Some(_)) => VideoCodec::Vp9OrH264,
1311
+ _ => VideoCodec::Vp9,
1312
+ }
1313
+ };
1049
1314
  let (peer_events_tx, peer_events_rx) = tokio_mpsc::unbounded_channel::<PeerEvent>();
1050
1315
  let (peer, offer) = VideoPeer::offer(
1051
1316
  TransportOptions {
@@ -1056,7 +1321,7 @@ impl Session {
1056
1321
  pace_bps: (bitrate_kbps as f64 * 3_000.0).max(20_000_000.0),
1057
1322
  },
1058
1323
  peer_events_tx,
1059
- VideoCodec::Vp9,
1324
+ codec,
1060
1325
  )
1061
1326
  .await
1062
1327
  .map_err(|error| SessionError::new("transport", format!("{error:#}")))?;
@@ -1081,9 +1346,23 @@ impl Session {
1081
1346
  capture_error,
1082
1347
  frame_changes,
1083
1348
  peer: Mutex::new(Some(peer.clone())),
1084
- encoder: Mutex::new(Some(encoder)),
1349
+ encoder: Mutex::new(Some(VideoEncoder::Vp9(encoder))),
1085
1350
  input: Mutex::new(input),
1086
1351
  indicator: Mutex::new(indicator),
1352
+ point_overlay: Mutex::new(None),
1353
+ #[cfg(target_os = "linux")]
1354
+ point_desktop: if wants_x11 {
1355
+ match &request.source {
1356
+ Some(SourceRequest::X11 { display }) => display
1357
+ .clone()
1358
+ .or_else(|| std::env::var("DISPLAY").ok())
1359
+ .map(PointDesktop::X11)
1360
+ .unwrap_or(PointDesktop::Unavailable("x11-unavailable")),
1361
+ _ => PointDesktop::Unavailable("x11-unavailable"),
1362
+ }
1363
+ } else {
1364
+ PointDesktop::Wayland
1365
+ },
1087
1366
  capture: Mutex::new(Some(capture)),
1088
1367
  layout: Mutex::new(Some(layout)),
1089
1368
  last_seq: Mutex::new(0),
@@ -1092,6 +1371,8 @@ impl Session {
1092
1371
  pipeline: pipeline.clone(),
1093
1372
  events: events.clone(),
1094
1373
  encoded: Mutex::new(None),
1374
+ local_frames: request.local_frames,
1375
+ cursor_info,
1095
1376
  });
1096
1377
 
1097
1378
  inner.notify(SessionEvent::Description {
@@ -1170,6 +1451,9 @@ impl Session {
1170
1451
  encoder: Mutex::new(None),
1171
1452
  input: Mutex::new(None),
1172
1453
  indicator: Mutex::new(None),
1454
+ point_overlay: Mutex::new(None),
1455
+ #[cfg(target_os = "linux")]
1456
+ point_desktop: PointDesktop::Unavailable("layer_shell_unavailable"),
1173
1457
  capture: Mutex::new(None),
1174
1458
  layout: Mutex::new(None),
1175
1459
  last_seq: Mutex::new(0),
@@ -1189,6 +1473,8 @@ impl Session {
1189
1473
  peer_events,
1190
1474
  feed,
1191
1475
  })),
1476
+ local_frames: false,
1477
+ cursor_info: serde_json::json!({"mode":"unavailable", "positions":false}),
1192
1478
  });
1193
1479
  inner.notify(SessionEvent::State {
1194
1480
  capture: "consented",
@@ -1246,6 +1532,10 @@ impl Session {
1246
1532
  self.inner.generation
1247
1533
  }
1248
1534
 
1535
+ pub fn cursor_info(&self) -> &serde_json::Value {
1536
+ &self.inner.cursor_info
1537
+ }
1538
+
1249
1539
  pub fn geometry(&self) -> &serde_json::Value {
1250
1540
  &self.inner.geometry
1251
1541
  }
@@ -1254,6 +1544,128 @@ impl Session {
1254
1544
  &self.inner.source
1255
1545
  }
1256
1546
 
1547
+ pub fn point(
1548
+ &self,
1549
+ params: &crate::protocol::PointParams,
1550
+ ) -> std::result::Result<(), SessionError> {
1551
+ // Lock through validation and drawing, so a concurrent close cannot leave
1552
+ // a newly spawned marker alive after teardown.
1553
+ let mut overlay = lock(&self.inner.point_overlay);
1554
+ if self.inner.closed.load(Ordering::SeqCst) {
1555
+ return Err(SessionError::new("session", "the session is closed"));
1556
+ }
1557
+ if !self.inner.permissions.contains(&Permission::View) {
1558
+ return Err(SessionError::new(
1559
+ "permission",
1560
+ "point requires view permission",
1561
+ ));
1562
+ }
1563
+ if lock(&self.inner.encoded).is_some() {
1564
+ return Err(SessionError::new(
1565
+ "operation",
1566
+ "an encoded source has no host desktop",
1567
+ ));
1568
+ }
1569
+ #[cfg(target_os = "linux")]
1570
+ if let PointDesktop::Unavailable(reason) = &self.inner.point_desktop {
1571
+ return Err(SessionError::new(
1572
+ "point_unsupported",
1573
+ format!("reason: {reason}; point overlay is unavailable"),
1574
+ ));
1575
+ }
1576
+ if params.clear {
1577
+ if params.x.is_some()
1578
+ || params.y.is_some()
1579
+ || params.timeout_ms.is_some()
1580
+ || !params.label.is_empty()
1581
+ {
1582
+ return Err(SessionError::new(
1583
+ "malformed",
1584
+ "clear accepts no coordinates, label or timeout",
1585
+ ));
1586
+ }
1587
+ #[cfg(target_os = "linux")]
1588
+ if overlay.is_none() && matches!(self.inner.point_desktop, PointDesktop::Wayland) {
1589
+ let available = crate::indicator::Indicator::wayland_layer_shell_available()
1590
+ .map_err(|error| {
1591
+ SessionError::new("indicator-unavailable", format!("{error:#}"))
1592
+ })?;
1593
+ if !available {
1594
+ return Err(SessionError::new(
1595
+ "point_unsupported",
1596
+ "reason: layer_shell_unavailable; compositor has no layer-shell",
1597
+ ));
1598
+ }
1599
+ }
1600
+ if let Some(helper) = overlay.take() {
1601
+ helper.stop();
1602
+ }
1603
+ return Ok(());
1604
+ }
1605
+ let (x, y) = params
1606
+ .x
1607
+ .zip(params.y)
1608
+ .ok_or_else(|| SessionError::new("malformed", "point requires x and y"))?;
1609
+ let source = &self.inner.source;
1610
+ if x < 0 || y < 0 || x >= i64::from(source.width) || y >= i64::from(source.height) {
1611
+ return Err(SessionError::new(
1612
+ "coordinates",
1613
+ "point is outside the selected desktop",
1614
+ ));
1615
+ }
1616
+ let timeout = params.timeout_ms.unwrap_or(3000);
1617
+ if !(1..=120_000).contains(&timeout)
1618
+ || params.label.len() > 96
1619
+ || !params.label.bytes().all(|b| (32..=126).contains(&b))
1620
+ {
1621
+ return Err(SessionError::new(
1622
+ "malformed",
1623
+ "timeout must be 1..120000 ms; label must be at most 96 printable ASCII bytes",
1624
+ ));
1625
+ }
1626
+ #[cfg(not(target_os = "windows"))]
1627
+ if overlay.is_none() {
1628
+ #[cfg(target_os = "linux")]
1629
+ let helper = match &self.inner.point_desktop {
1630
+ PointDesktop::X11(display) => crate::indicator::Indicator::start_point(display),
1631
+ PointDesktop::Wayland => crate::indicator::Indicator::start_point_wayland(
1632
+ source.width,
1633
+ source.height,
1634
+ source.position,
1635
+ ),
1636
+ PointDesktop::Unavailable(_) => unreachable!("validated point target"),
1637
+ };
1638
+ #[cfg(target_os = "macos")]
1639
+ let helper = crate::indicator::Indicator::start_point(
1640
+ source.node_id,
1641
+ source.width as usize,
1642
+ source.height as usize,
1643
+ );
1644
+ *overlay = Some(helper.map_err(|error| {
1645
+ let message = format!("{error:#}");
1646
+ if message.contains("layer_shell_unavailable") {
1647
+ SessionError::new(
1648
+ "point_unsupported",
1649
+ "reason: layer_shell_unavailable; compositor has no layer-shell",
1650
+ )
1651
+ } else {
1652
+ SessionError::new("indicator-unavailable", message)
1653
+ }
1654
+ })?);
1655
+ }
1656
+ #[cfg(not(target_os = "windows"))]
1657
+ return overlay
1658
+ .as_mut()
1659
+ .unwrap()
1660
+ .point(x, y, timeout, &params.label)
1661
+ .map_err(|error| SessionError::new("indicator-unavailable", format!("{error:#}")));
1662
+ #[cfg(target_os = "windows")]
1663
+ return Err(SessionError::new(
1664
+ "point_unsupported",
1665
+ "Windows point overlays are unavailable in this build.",
1666
+ ));
1667
+ }
1668
+
1257
1669
  pub async fn wait_frame(
1258
1670
  &self,
1259
1671
  after_seq: Option<u64>,
@@ -1266,6 +1678,7 @@ impl Session {
1266
1678
  "this session forwards an encoded stream and keeps no frame",
1267
1679
  ));
1268
1680
  }
1681
+ self.keeps_frames()?;
1269
1682
  let mut changes = self.inner.frame_changes.clone();
1270
1683
  let deadline = tokio::time::Instant::now() + Duration::from_millis(timeout_ms.min(120_000));
1271
1684
  loop {
@@ -1302,6 +1715,16 @@ impl Session {
1302
1715
  }
1303
1716
  }
1304
1717
 
1718
+ fn keeps_frames(&self) -> std::result::Result<(), SessionError> {
1719
+ if self.inner.local_frames {
1720
+ return Ok(());
1721
+ }
1722
+ Err(SessionError::new(
1723
+ "operation",
1724
+ "this session was opened with local_frames off and keeps no frame",
1725
+ ))
1726
+ }
1727
+
1305
1728
  /// Write only the requested lossless frame bytes; JSON carries metadata, never pixels.
1306
1729
  pub fn frame(
1307
1730
  &self,
@@ -1321,6 +1744,7 @@ impl Session {
1321
1744
  "this session forwards an encoded stream and keeps no frame",
1322
1745
  ));
1323
1746
  }
1747
+ self.keeps_frames()?;
1324
1748
  if let Some(reason) = lock(&self.inner.capture_error).clone() {
1325
1749
  return Err(SessionError::new("stream-stopped", reason));
1326
1750
  }
@@ -1356,7 +1780,7 @@ impl Session {
1356
1780
  }
1357
1781
  Some(pixels)
1358
1782
  };
1359
- let result = serde_json::json!({ "seq": seq, "still_ms": at.elapsed().as_millis() as u64,
1783
+ let result = serde_json::json!({ "seq": seq, "timestamp_us": crate::protocol::timestamp_us(*at), "still_ms": at.elapsed().as_millis() as u64,
1360
1784
  "width": w, "height": h, "format": "bgrx", "damage": damage, "written": !path.is_empty() });
1361
1785
  drop(held);
1362
1786
  if let Some(pixels) = pixels {
@@ -1449,7 +1873,7 @@ impl Session {
1449
1873
  }
1450
1874
 
1451
1875
  pub async fn close(&self, reason: &str) {
1452
- self.inner.close(reason).await;
1876
+ self.inner.close(reason, "closed").await;
1453
1877
  }
1454
1878
  }
1455
1879
 
@@ -1464,16 +1888,24 @@ impl Inner {
1464
1888
 
1465
1889
  /// Stop everything and release whatever input this session held. Idempotent,
1466
1890
  /// and the one teardown both an explicit close and a lease expiry take.
1467
- async fn close(&self, reason: &str) {
1891
+ async fn close(&self, reason: &str, code: &str) {
1468
1892
  if self.closed.swap(true, Ordering::SeqCst) {
1469
1893
  return;
1470
1894
  }
1471
1895
  self.control_open.store(false, Ordering::SeqCst);
1472
1896
  self.pipeline.store(false, Ordering::SeqCst);
1473
1897
  drop(lock(&self.encoded).take());
1898
+ // The encoder goes now, not with the last handle: a hardware session
1899
+ // still open when the process exits can hang the driver's exit
1900
+ // handlers, and `shutdown` exits right after this. The encode thread
1901
+ // finds it gone and stops.
1902
+ drop(lock(&self.encoder).take());
1474
1903
  if let Ok(mut indicator) = self.indicator.lock() {
1475
1904
  indicator.take();
1476
1905
  }
1906
+ if let Some(helper) = lock(&self.point_overlay).take() {
1907
+ helper.stop();
1908
+ }
1477
1909
  // Release first, then tear down: a stuck modifier is the one failure the
1478
1910
  // user cannot undo by reconnecting.
1479
1911
  if let Ok(mut input) = self.input.lock() {
@@ -1485,8 +1917,8 @@ impl Inner {
1485
1917
  if let Some(peer) = self.peer.lock().ok().and_then(|peer| peer.clone()) {
1486
1918
  let _ = peer
1487
1919
  .send_control(
1488
- &serde_json::to_string(&ControlReply::Revoked { reason }).unwrap_or_else(
1489
- |_| String::from(r#"{"kind":"revoked","reason":"closed"}"#),
1920
+ &serde_json::to_string(&ControlReply::Revoked { reason, code }).unwrap_or_else(
1921
+ |_| String::from(r#"{"kind":"revoked","reason":"closed","code":"closed"}"#),
1490
1922
  ),
1491
1923
  )
1492
1924
  .await;
@@ -1820,7 +2252,7 @@ impl Inner {
1820
2252
  if text.len() > 4096 {
1821
2253
  return Err(("text-too-large", String::from("text exceeds 4096 bytes")));
1822
2254
  }
1823
- #[cfg(target_os = "macos")]
2255
+ #[cfg(any(target_os = "macos", target_os = "windows"))]
1824
2256
  {
1825
2257
  let _ = seq;
1826
2258
  return self.with_input(|target| target.unicode_text(text));
@@ -1937,11 +2369,13 @@ fn frame_interval(max_fps: u32) -> Duration {
1937
2369
  Duration::from_secs_f64(1.0 / max_fps.max(1) as f64)
1938
2370
  }
1939
2371
 
1940
- /// The newest captured frame, handed from the capture thread to the encoder. A
1941
- /// newer frame replaces one the encoder has not taken, so however far behind
1942
- /// it falls, what it codes next is the desktop as it is now.
2372
+ /// The newest captured frame and when it was captured, handed from the capture
2373
+ /// thread to the encoder. A newer frame replaces one the encoder has not taken,
2374
+ /// so however far behind it falls, what it codes next is the desktop as it is
2375
+ /// now. The instant travels with the picture so the pipeline can report how
2376
+ /// long a coded frame waited for its encode.
1943
2377
  struct FrameSlot {
1944
- frame: Mutex<(Option<I420>, bool)>,
2378
+ frame: Mutex<(Option<(I420, Instant)>, bool)>,
1945
2379
  ready: Condvar,
1946
2380
  }
1947
2381
 
@@ -1958,13 +2392,13 @@ fn latest_frame() -> (FrameSender, FrameReceiver) {
1958
2392
 
1959
2393
  impl FrameSender {
1960
2394
  /// Hand over a frame; true when it replaced one the encoder never took.
1961
- fn put(&self, frame: I420) -> bool {
2395
+ fn put(&self, frame: I420, captured: Instant) -> bool {
1962
2396
  let mut held = self
1963
2397
  .0
1964
2398
  .frame
1965
2399
  .lock()
1966
2400
  .unwrap_or_else(|poisoned| poisoned.into_inner());
1967
- let replaced = held.0.replace(frame).is_some();
2401
+ let replaced = held.0.replace((frame, captured)).is_some();
1968
2402
  self.0.ready.notify_one();
1969
2403
  replaced
1970
2404
  }
@@ -1984,12 +2418,22 @@ impl Drop for FrameSender {
1984
2418
  }
1985
2419
 
1986
2420
  enum Taken {
1987
- Frame(I420),
2421
+ Frame(I420, Instant),
1988
2422
  Timeout,
1989
2423
  Ended,
1990
2424
  }
1991
2425
 
1992
2426
  impl FrameReceiver {
2427
+ /// Whether a frame is waiting to be taken.
2428
+ fn waiting(&self) -> bool {
2429
+ self.0
2430
+ .frame
2431
+ .lock()
2432
+ .unwrap_or_else(|poisoned| poisoned.into_inner())
2433
+ .0
2434
+ .is_some()
2435
+ }
2436
+
1993
2437
  fn take(&self, wait: Duration) -> Taken {
1994
2438
  let held = self
1995
2439
  .0
@@ -2002,7 +2446,7 @@ impl FrameReceiver {
2002
2446
  .wait_timeout_while(held, wait, |(frame, ended)| frame.is_none() && !*ended)
2003
2447
  .unwrap_or_else(|poisoned| poisoned.into_inner());
2004
2448
  match held.0.take() {
2005
- Some(frame) => Taken::Frame(frame),
2449
+ Some((frame, captured)) => Taken::Frame(frame, captured),
2006
2450
  None if held.1 => Taken::Ended,
2007
2451
  None => Taken::Timeout,
2008
2452
  }
@@ -2028,12 +2472,32 @@ fn auto_bitrate(width: usize, height: usize) -> u32 {
2028
2472
  (1500 + (width * height * 18 / 10_000) as u32).min(25_000)
2029
2473
  }
2030
2474
 
2031
- /// Loss-based rate target: back off quickly when the far end reports loss,
2032
- /// creep back when it reports none, never above what the session asked for.
2475
+ /// Queueing delay that means the link carries more than it drains: well under
2476
+ /// the latency budget, well over the noise of a busy host's scheduling.
2477
+ const QUEUE_HIGH: Duration = Duration::from_millis(25);
2478
+ /// Below this the path has room again and the target may creep back up.
2479
+ const QUEUE_LOW: Duration = Duration::from_millis(10);
2480
+ /// How often queueing delay may back the target off: long enough for the last
2481
+ /// back-off to reach the far end and show in its feedback, short enough that a
2482
+ /// queue is drained in a few steps rather than a few seconds.
2483
+ const DELAY_BACKOFF_EVERY: Duration = Duration::from_millis(300);
2484
+
2485
+ /// The rate target: back off when the far end reports a standing queue or
2486
+ /// loss, creep back when it reports neither, never above what the session
2487
+ /// asked for.
2488
+ ///
2489
+ /// Delay leads. A link with a deep buffer (most Wi-Fi and cellular ones) never
2490
+ /// loses a packet it can queue, so loss alone lets queueing delay grow to
2491
+ /// seconds before anything reacts. A standing queue backs the target off to
2492
+ /// 85% of what the path delivered, which drains it; never more than halving
2493
+ /// per step, so a stream that was idle and sent little is not taken for a
2494
+ /// slow link. Receivers without transport-wide feedback leave it loss-only.
2033
2495
  struct RateControl {
2034
2496
  ceiling: u32,
2035
2497
  target: u32,
2036
2498
  last_change: Option<Instant>,
2499
+ /// The last queueing delay reported, while the path reports one.
2500
+ queue: Option<Duration>,
2037
2501
  }
2038
2502
 
2039
2503
  impl RateControl {
@@ -2042,26 +2506,40 @@ impl RateControl {
2042
2506
  ceiling,
2043
2507
  target: ceiling,
2044
2508
  last_change: None,
2509
+ queue: None,
2045
2510
  }
2046
2511
  }
2047
2512
 
2048
- /// The new target when these reports move it, at most once a second.
2049
- fn update(&mut self, reports: &[u8], now: Instant) -> Option<u32> {
2050
- let worst = *reports.iter().max()?;
2051
- if self
2052
- .last_change
2053
- .is_some_and(|at| now.duration_since(at) < Duration::from_secs(1))
2054
- {
2055
- return None;
2513
+ /// The new target when these loss reports (RFC 3550 fractions, /256) or
2514
+ /// this path report move it: down on a standing queue at most every
2515
+ /// `DELAY_BACKOFF_EVERY`, otherwise at most once a second.
2516
+ fn update(&mut self, reports: &[u8], path: Option<PathReport>, now: Instant) -> Option<u32> {
2517
+ if let Some(path) = path {
2518
+ self.queue = Some(path.queue_delay);
2056
2519
  }
2057
- let loss = worst as f64 / 256.0;
2520
+ let since = self.last_change.map(|at| now.duration_since(at));
2521
+ let waited = |gap: Duration| since.is_none_or(|since| since >= gap);
2058
2522
  let floor = (self.ceiling / 8).max(500);
2059
- let next = if loss > 0.10 {
2060
- ((self.target as f64 * 0.7) as u32).max(floor)
2061
- } else if loss < 0.02 {
2062
- ((self.target as f64 * 1.08) as u32 + 64).min(self.ceiling)
2523
+ let next = if let Some(path) = path.filter(|path| path.queue_delay > QUEUE_HIGH) {
2524
+ if !waited(DELAY_BACKOFF_EVERY) {
2525
+ return None;
2526
+ }
2527
+ let delivered = path.delivered_kbps.clamp(self.target / 2, self.target);
2528
+ ((delivered as f64 * 0.85) as u32).max(floor)
2063
2529
  } else {
2064
- self.target
2530
+ let worst = *reports.iter().max()?;
2531
+ if !waited(Duration::from_secs(1)) {
2532
+ return None;
2533
+ }
2534
+ let loss = worst as f64 / 256.0;
2535
+ let roomy = self.queue.is_none_or(|queue| queue < QUEUE_LOW);
2536
+ if loss > 0.10 {
2537
+ ((self.target as f64 * 0.7) as u32).max(floor)
2538
+ } else if loss < 0.02 && roomy {
2539
+ ((self.target as f64 * 1.08) as u32 + 64).min(self.ceiling)
2540
+ } else {
2541
+ self.target
2542
+ }
2065
2543
  };
2066
2544
  if next == self.target {
2067
2545
  return None;
@@ -2072,9 +2550,127 @@ impl RateControl {
2072
2550
  }
2073
2551
  }
2074
2552
 
2553
+ /// How far the stream may run ahead of its rate target before the next frame
2554
+ /// waits: one large frame (a key frame, a refinement) goes at once, but the
2555
+ /// frames behind it hold until the link has had time to carry it, instead of
2556
+ /// queueing behind it. The frame that goes then is the newest one.
2557
+ const FRAME_BURST: Duration = Duration::from_millis(60);
2558
+
2559
+ /// How much faster than the target the budget drains. A rate-controlled
2560
+ /// encoder averages its target but overshoots it frame to frame; drained at
2561
+ /// the target itself, that ordinary overshoot would add up and hold frames on
2562
+ /// a link with room to spare. The rate control, not this, keeps the average
2563
+ /// under what the path carries.
2564
+ const BUDGET_HEADROOM: f64 = 1.25;
2565
+
2566
+ /// What has been sent beyond the rate target, draining a little faster than
2567
+ /// it: a leaky bucket that bounds a burst to `FRAME_BURST` plus one frame.
2568
+ #[derive(Default)]
2569
+ struct SendBudget {
2570
+ bits: f64,
2571
+ at: Option<Instant>,
2572
+ }
2573
+
2574
+ impl SendBudget {
2575
+ /// The drain rate, in bits per second.
2576
+ fn rate(kbps: u32) -> f64 {
2577
+ kbps.max(1) as f64 * 1000.0 * BUDGET_HEADROOM
2578
+ }
2579
+
2580
+ fn drained(&self, now: Instant, kbps: u32) -> f64 {
2581
+ let elapsed = self
2582
+ .at
2583
+ .map_or(0.0, |at| now.saturating_duration_since(at).as_secs_f64());
2584
+ (self.bits - elapsed * Self::rate(kbps)).max(0.0)
2585
+ }
2586
+
2587
+ /// How long until a frame may go; zero when it may go now.
2588
+ fn wait(&self, now: Instant, kbps: u32) -> Duration {
2589
+ let rate = Self::rate(kbps);
2590
+ let over = self.drained(now, kbps) - FRAME_BURST.as_secs_f64() * rate;
2591
+ Duration::from_secs_f64((over / rate).max(0.0))
2592
+ }
2593
+
2594
+ fn sent(&mut self, now: Instant, bytes: usize, kbps: u32) {
2595
+ self.bits = self.drained(now, kbps) + bytes as f64 * 8.0;
2596
+ self.at = Some(now);
2597
+ }
2598
+ }
2599
+
2600
+ /// Whether motion is coded at half size. A full-size frame is slow when it
2601
+ /// took longer to code than a frame interval and the source already had a
2602
+ /// newer frame waiting: the encoder, not the source, set the pace. Content that
2603
+ /// arrives slower than the frame cap (30 fps into a 60 fps session) is never
2604
+ /// slow however long a frame takes, as long as it is done before the next one.
2605
+ /// Several slow frames that each change much of the picture (a packet over a
2606
+ /// quarter of the per-frame rate budget) mean full size cannot reach the frame
2607
+ /// rate, so the rest of the motion is coded at half size, until it stops (the
2608
+ /// refinement pass is always full size) or a run of frames that are both small
2609
+ /// and not slow shows full size would keep up again. Packet size or coding time
2610
+ /// alone never returns it to full size, so heavy motion does not flap between
2611
+ /// sizes; each change costs a key frame. Both directions need a run of frames
2612
+ /// rather than one, so a slow key frame or a busy moment on the host does not
2613
+ /// flip it, and typing never halves.
2614
+ #[derive(Default)]
2615
+ struct MotionSize {
2616
+ half: bool,
2617
+ run: u32,
2618
+ }
2619
+
2620
+ impl MotionSize {
2621
+ const SLOW_RUN: u32 = 5;
2622
+ const SMALL_RUN: u32 = 30;
2623
+
2624
+ /// Account for a motion frame coded at the current size in `took`; `large`
2625
+ /// when its packet was over a quarter of the per-frame rate budget, `behind`
2626
+ /// when a newer frame was already waiting once it was coded.
2627
+ fn coded(&mut self, took: Duration, large: bool, behind: bool, interval: Duration) {
2628
+ let slow = took > interval && behind;
2629
+ let (counts, needed) = if self.half {
2630
+ (!large && !slow, Self::SMALL_RUN)
2631
+ } else {
2632
+ (large && slow, Self::SLOW_RUN)
2633
+ };
2634
+ self.run = if counts { self.run + 1 } else { 0 };
2635
+ if self.run >= needed {
2636
+ self.half = !self.half;
2637
+ self.run = 0;
2638
+ }
2639
+ }
2640
+ }
2641
+
2642
+ /// The rate cap as a deadline rather than a minimum gap. `due` is where the
2643
+ /// next frame falls on the `max_fps` cadence the stream has kept so far, and a
2644
+ /// frame may go up to one interval before it, so a late frame's lost time is
2645
+ /// made up by the next. A capture clock that jitters around the cap is then
2646
+ /// coded the moment each frame arrives, instead of beating against a second
2647
+ /// clock that delays a frame or lets the next one supersede it; the sustained
2648
+ /// rate still never exceeds the cap, and no window holds more than one frame
2649
+ /// over it.
2650
+ struct Cadence {
2651
+ interval: Duration,
2652
+ due: Instant,
2653
+ }
2654
+
2655
+ impl Cadence {
2656
+ fn new(interval: Duration, now: Instant) -> Self {
2657
+ Self { interval, due: now }
2658
+ }
2659
+
2660
+ /// How long until a frame may go; zero when it may go now.
2661
+ fn wait(&self, now: Instant) -> Duration {
2662
+ self.due.saturating_duration_since(now + self.interval)
2663
+ }
2664
+
2665
+ fn sent(&mut self, now: Instant) {
2666
+ self.due = self.due.max(now) + self.interval;
2667
+ }
2668
+ }
2669
+
2075
2670
  enum Pass {
2076
2671
  Motion {
2077
2672
  keyframe: bool,
2673
+ half: bool,
2078
2674
  },
2079
2675
  Refine,
2080
2676
  /// The same picture again, so a still stream never looks stalled.
@@ -2107,8 +2703,11 @@ fn spawn_pipeline(
2107
2703
  let mut rate = RateControl::new(bitrate_kbps);
2108
2704
  if let Ok(mut m) = inner.metrics.lock() {
2109
2705
  m.target_kbps = bitrate_kbps;
2706
+ (m.codec, m.encoder) = ("vp9", "libvpx");
2110
2707
  }
2111
- let mut latest: Option<I420> = None;
2708
+ // Whether motion may be coded at half size: VP9's alone.
2709
+ let mut halves = true;
2710
+ let mut latest: Option<(I420, Instant)> = None;
2112
2711
  // `pending`: the latest frame has not been coded. `refined`: it has
2113
2712
  // had its refinement pass, or there is nothing to refine.
2114
2713
  let mut pending = false;
@@ -2116,11 +2715,15 @@ fn spawn_pipeline(
2116
2715
  let mut keyframe = true;
2117
2716
  let mut still_since = Instant::now();
2118
2717
  let mut last_sent: Option<Instant> = None;
2718
+ let mut size = MotionSize::default();
2719
+ let mut cadence = Cadence::new(interval, Instant::now());
2720
+ let mut budget = SendBudget::default();
2721
+ // What the last frame was stamped with: the RTP clock never runs
2722
+ // backwards, even for a re-coded picture.
2723
+ let mut last_stamp: Option<Instant> = None;
2119
2724
  loop {
2120
2725
  let now = Instant::now();
2121
- let slot = last_sent.map_or(Duration::ZERO, |at| {
2122
- interval.saturating_sub(now.duration_since(at))
2123
- });
2726
+ let slot = cadence.wait(now).max(budget.wait(now, rate.target));
2124
2727
  let connected = peer.is_connected();
2125
2728
  let wait = if !connected {
2126
2729
  // Nothing sent before the transport connects arrives; the
@@ -2140,13 +2743,13 @@ fn spawn_pipeline(
2140
2743
  .clamp(Duration::from_millis(10), Duration::from_millis(100))
2141
2744
  };
2142
2745
  match frame_rx.take(wait) {
2143
- Taken::Frame(frame) => {
2746
+ Taken::Frame(frame, captured) => {
2144
2747
  if pending {
2145
2748
  if let Ok(mut m) = inner.metrics.lock() {
2146
2749
  m.dropped_frames += 1;
2147
2750
  }
2148
2751
  }
2149
- latest = Some(frame);
2752
+ latest = Some((frame, captured));
2150
2753
  pending = true;
2151
2754
  refined = false;
2152
2755
  still_since = Instant::now();
@@ -2161,7 +2764,8 @@ fn spawn_pipeline(
2161
2764
  keyframe = true;
2162
2765
  }
2163
2766
  let reports = peer.take_loss_reports();
2164
- if let Some(kbps) = rate.update(&reports, Instant::now()) {
2767
+ let path = peer.take_path_report();
2768
+ if let Some(kbps) = rate.update(&reports, path, Instant::now()) {
2165
2769
  let applied = inner.encoder.lock().map(|mut encoder| {
2166
2770
  encoder
2167
2771
  .as_mut()
@@ -2174,18 +2778,45 @@ fn spawn_pipeline(
2174
2778
  }
2175
2779
  }
2176
2780
  }
2177
- let Some(frame) = latest.as_ref() else {
2781
+ let Some((frame, captured)) = latest.as_ref() else {
2178
2782
  continue;
2179
2783
  };
2180
2784
  if !connected {
2181
2785
  continue;
2182
2786
  }
2183
2787
  let now = Instant::now();
2184
- if last_sent.is_some_and(|at| now.duration_since(at) < interval) {
2788
+ if !cadence.wait(now).is_zero() || !budget.wait(now, rate.target).is_zero() {
2185
2789
  continue;
2186
2790
  }
2791
+ if halves && peer.sends_h264() {
2792
+ // The answer asked for H.264: nothing has been sent yet,
2793
+ // since frames wait for the transport, which waits for it.
2794
+ match start_h264(&inner, rate.target, max_fps) {
2795
+ Ok(()) => {
2796
+ halves = false;
2797
+ size = MotionSize::default();
2798
+ keyframe = true;
2799
+ }
2800
+ Err(error) => {
2801
+ eprintln!("{error:#}");
2802
+ let reason = String::from("no H.264 encoder could start");
2803
+ inner.notify(SessionEvent::Revoked {
2804
+ reason: reason.clone(),
2805
+ code: "error",
2806
+ });
2807
+ let target = inner.clone();
2808
+ handle.spawn(async move {
2809
+ target.close(&reason, "error").await;
2810
+ });
2811
+ break;
2812
+ }
2813
+ }
2814
+ }
2187
2815
  let pass = if pending || keyframe {
2188
- Pass::Motion { keyframe }
2816
+ Pass::Motion {
2817
+ keyframe,
2818
+ half: size.half && halves,
2819
+ }
2189
2820
  } else if !refined && now.duration_since(still_since) >= REFINE_AFTER {
2190
2821
  Pass::Refine
2191
2822
  } else if refined
@@ -2206,9 +2837,9 @@ fn spawn_pipeline(
2206
2837
  break;
2207
2838
  };
2208
2839
  match pass {
2209
- Pass::Motion { keyframe } => encoder.encode(frame, keyframe),
2840
+ Pass::Motion { keyframe, half } => encoder.encode(frame, keyframe, half),
2210
2841
  Pass::Refine => encoder.refine(frame),
2211
- Pass::Keepalive => encoder.encode(frame, false),
2842
+ Pass::Keepalive => encoder.encode(frame, false, false),
2212
2843
  }
2213
2844
  };
2214
2845
  let packet = match packet {
@@ -2226,22 +2857,29 @@ fn spawn_pipeline(
2226
2857
  let reason = String::from("the encoder rejected a frame");
2227
2858
  inner.notify(SessionEvent::Revoked {
2228
2859
  reason: reason.clone(),
2860
+ code: "error",
2229
2861
  });
2230
2862
  let target = inner.clone();
2231
2863
  handle.spawn(async move {
2232
- target.close(&reason).await;
2864
+ target.close(&reason, "error").await;
2233
2865
  });
2234
2866
  break;
2235
2867
  }
2236
2868
  };
2869
+ let took = started.elapsed();
2237
2870
  if let Ok(mut m) = inner.metrics.lock() {
2238
2871
  m.encoded_frames += 1;
2239
2872
  m.encoded_bytes += packet.data.len() as u64;
2240
- m.encode_micros += started.elapsed().as_micros() as u64;
2873
+ m.encode_micros += took.as_micros() as u64;
2874
+ m.key_frames += packet.keyframe as u64;
2875
+ if matches!(pass, Pass::Motion { .. }) && pending {
2876
+ m.queue_micros += started.duration_since(*captured).as_micros() as u64;
2877
+ m.motion_frames += 1;
2878
+ }
2241
2879
  match pass {
2242
- Pass::Motion { keyframe: true } => m.key_frames += 1,
2880
+ Pass::Motion { half: true, .. } => m.halved_frames += 1,
2243
2881
  Pass::Refine => m.refined_frames += 1,
2244
- Pass::Motion { keyframe: false } | Pass::Keepalive => {}
2882
+ Pass::Motion { half: false, .. } | Pass::Keepalive => {}
2245
2883
  }
2246
2884
  }
2247
2885
  match pass {
@@ -2250,13 +2888,34 @@ fn spawn_pipeline(
2250
2888
  keyframe = false;
2251
2889
  refined = false;
2252
2890
  still_since = now;
2891
+ let budget = rate.target as u64 * 1000 / 8 / max_fps.max(1) as u64;
2892
+ let large = packet.data.len() as u64 * 4 > budget;
2893
+ if halves {
2894
+ size.coded(took, large, frame_rx.waiting(), interval);
2895
+ }
2896
+ }
2897
+ Pass::Refine => {
2898
+ refined = true;
2899
+ size = MotionSize::default();
2253
2900
  }
2254
- Pass::Refine => refined = true,
2255
2901
  Pass::Keepalive => {}
2256
2902
  }
2257
2903
  last_sent = Some(now);
2904
+ cadence.sent(now);
2905
+ budget.sent(now, packet.data.len(), rate.target);
2906
+ // A new picture carries the moment it was captured, so the
2907
+ // receiver paces playout by the desktop's clock rather than by
2908
+ // encode time; a re-coded one (refinement, keepalive) is new
2909
+ // content now.
2910
+ let stamp = match pass {
2911
+ Pass::Motion { .. } => *captured,
2912
+ Pass::Refine | Pass::Keepalive => now,
2913
+ };
2914
+ let stamp = last_stamp.map_or(stamp, |at| stamp.max(at + Duration::from_millis(1)));
2915
+ last_stamp = Some(stamp);
2916
+ let sending = Instant::now();
2258
2917
  if let Err(error) =
2259
- handle.block_on(peer.send_frame(&packet.data, packet.keyframe, now))
2918
+ handle.block_on(peer.send_frame(&packet.data, packet.keyframe, stamp))
2260
2919
  {
2261
2920
  // The transport is gone. End the session rather than leave
2262
2921
  // the desktop captured behind a picture that stopped.
@@ -2264,18 +2923,41 @@ fn spawn_pipeline(
2264
2923
  let reason = String::from("the connection to the phone was lost");
2265
2924
  inner.notify(SessionEvent::Revoked {
2266
2925
  reason: reason.clone(),
2926
+ code: "transport",
2267
2927
  });
2268
2928
  let target = inner.clone();
2269
2929
  handle.spawn(async move {
2270
- target.close(&reason).await;
2930
+ target.close(&reason, "transport").await;
2271
2931
  });
2272
2932
  break;
2273
2933
  }
2934
+ if let Ok(mut m) = inner.metrics.lock() {
2935
+ m.send_micros += sending.elapsed().as_micros() as u64;
2936
+ }
2274
2937
  }
2275
2938
  })
2276
2939
  .ok();
2277
2940
  }
2278
2941
 
2942
+ /// Replace the session's VP9 encoder with an H.264 one of the same size and
2943
+ /// rate target, on the encode thread that will drive it.
2944
+ fn start_h264(inner: &Arc<Inner>, bitrate_kbps: u32, fps: u32) -> Result<()> {
2945
+ let mut held = lock(&inner.encoder);
2946
+ let (width, height) = held
2947
+ .as_ref()
2948
+ .map(VideoEncoder::size)
2949
+ .context("the session has no encoder")?;
2950
+ let (backend, info) = crate::h264::open(width, height, bitrate_kbps, fps)?;
2951
+ *held = Some(VideoEncoder::H264 {
2952
+ backend,
2953
+ size: (width, height),
2954
+ });
2955
+ if let Ok(mut m) = inner.metrics.lock() {
2956
+ (m.codec, m.encoder) = ("h264", info.encoder);
2957
+ }
2958
+ Ok(())
2959
+ }
2960
+
2279
2961
  /// Packetize what a consumer feeds an encoded source, building the peer on the
2280
2962
  /// first access unit that carries an SPS.
2281
2963
  ///
@@ -2338,6 +3020,7 @@ fn spawn_encoded(inner: &Arc<Inner>, mut feed: tokio_mpsc::Receiver<FedAccessUni
2338
3020
  revoke(
2339
3021
  &inner,
2340
3022
  format!("the transport refused the encoded stream: {error:#}"),
3023
+ "error",
2341
3024
  )
2342
3025
  .await;
2343
3026
  return;
@@ -2362,7 +3045,12 @@ fn spawn_encoded(inner: &Arc<Inner>, mut feed: tokio_mpsc::Receiver<FedAccessUni
2362
3045
  }
2363
3046
  Err(error) => {
2364
3047
  eprintln!("the video track refused a fed access unit: {error:#}");
2365
- revoke(&inner, String::from("the connection to the phone was lost")).await;
3048
+ revoke(
3049
+ &inner,
3050
+ String::from("the connection to the phone was lost"),
3051
+ "transport",
3052
+ )
3053
+ .await;
2366
3054
  return;
2367
3055
  }
2368
3056
  }
@@ -2376,11 +3064,12 @@ fn current_peer(inner: &Arc<Inner>) -> Option<Arc<VideoPeer>> {
2376
3064
 
2377
3065
  /// End the session with a reason the consumer can show, exactly as the encode
2378
3066
  /// pipeline does when its transport goes away.
2379
- async fn revoke(inner: &Arc<Inner>, reason: String) {
3067
+ async fn revoke(inner: &Arc<Inner>, reason: String, code: &'static str) {
2380
3068
  inner.notify(SessionEvent::Revoked {
2381
3069
  reason: reason.clone(),
3070
+ code,
2382
3071
  });
2383
- inner.close(&reason).await;
3072
+ inner.close(&reason, code).await;
2384
3073
  }
2385
3074
 
2386
3075
  fn spawn_peer_events(inner: &Arc<Inner>, mut events: tokio_mpsc::UnboundedReceiver<PeerEvent>) {
@@ -2428,8 +3117,9 @@ fn spawn_peer_events(inner: &Arc<Inner>, mut events: tokio_mpsc::UnboundedReceiv
2428
3117
  let reason = String::from("the connection to the phone was lost");
2429
3118
  inner.notify(SessionEvent::Revoked {
2430
3119
  reason: reason.clone(),
3120
+ code: "transport",
2431
3121
  });
2432
- inner.close(&reason).await;
3122
+ inner.close(&reason, "transport").await;
2433
3123
  continue;
2434
3124
  }
2435
3125
  inner.notify(SessionEvent::State {
@@ -2487,10 +3177,11 @@ fn spawn_lease(inner: &Arc<Inner>, ttl: Duration) {
2487
3177
  let reason = String::from("the session lease expired");
2488
3178
  inner.notify(SessionEvent::Revoked {
2489
3179
  reason: reason.clone(),
3180
+ code: "lease",
2490
3181
  });
2491
3182
  // An expiry ends the session exactly as an explicit close does; the
2492
3183
  // notification above is not a substitute for stopping the desktop.
2493
- inner.close(&reason).await;
3184
+ inner.close(&reason, "lease").await;
2494
3185
  });
2495
3186
  }
2496
3187
 
@@ -2558,6 +3249,9 @@ mod tests {
2558
3249
  encoder: Mutex::new(None),
2559
3250
  input: Mutex::new(None),
2560
3251
  indicator: Mutex::new(None),
3252
+ point_overlay: Mutex::new(None),
3253
+ #[cfg(target_os = "linux")]
3254
+ point_desktop: PointDesktop::Unavailable("layer_shell_unavailable"),
2561
3255
  capture: Mutex::new(None),
2562
3256
  layout: Mutex::new(None),
2563
3257
  last_seq: Mutex::new(0),
@@ -2575,10 +3269,12 @@ mod tests {
2575
3269
  peer_events,
2576
3270
  feed,
2577
3271
  })),
3272
+ local_frames: false,
3273
+ cursor_info: serde_json::json!({"mode":"unavailable", "positions":false}),
2578
3274
  });
2579
3275
  spawn_peer_events(&inner, peer_events_rx);
2580
3276
  spawn_encoded(&inner, feed_rx);
2581
- inner.close("test").await;
3277
+ inner.close("test", "closed").await;
2582
3278
  for _ in 0..100 {
2583
3279
  if Arc::strong_count(&inner) == 1 {
2584
3280
  break;
@@ -2600,6 +3296,41 @@ mod tests {
2600
3296
  );
2601
3297
  }
2602
3298
 
3299
+ #[test]
3300
+ fn frames_that_jitter_around_the_cap_go_at_once_and_a_flood_stays_capped() {
3301
+ let interval = Duration::from_millis(20);
3302
+ let start = Instant::now();
3303
+ let mut cadence = Cadence::new(interval, start);
3304
+ // Captured at the cap, each a few ms early or late: none waits, so
3305
+ // none is delayed or superseded by the next.
3306
+ for (n, jitter) in [0i64, -6, 5, -8, 7, -3, 0, -9, 9, -5].iter().enumerate() {
3307
+ let at = start + interval * n as u32 + Duration::from_millis(9);
3308
+ let at = if *jitter < 0 {
3309
+ at - Duration::from_millis(jitter.unsigned_abs())
3310
+ } else {
3311
+ at + Duration::from_millis(*jitter as u64)
3312
+ };
3313
+ assert_eq!(cadence.wait(at), Duration::ZERO, "frame {n} waited");
3314
+ cadence.sent(at);
3315
+ }
3316
+
3317
+ // A source far above the cap: one second of 1 ms frames sends at most
3318
+ // one frame over the cap.
3319
+ let start = start + Duration::from_secs(10);
3320
+ let mut cadence = Cadence::new(interval, start);
3321
+ let sent = (0..1000)
3322
+ .map(|ms| start + Duration::from_millis(ms))
3323
+ .filter(|&at| {
3324
+ let go = cadence.wait(at).is_zero();
3325
+ if go {
3326
+ cadence.sent(at);
3327
+ }
3328
+ go
3329
+ })
3330
+ .count();
3331
+ assert!((50..=51).contains(&sent), "sent {sent} at a 50 fps cap");
3332
+ }
3333
+
2603
3334
  #[test]
2604
3335
  fn a_small_source_is_never_upscaled() {
2605
3336
  assert_eq!(fit(1000, 600, 1280, 800), (1000, 600));
@@ -2658,6 +3389,149 @@ mod tests {
2658
3389
  )
2659
3390
  .await
2660
3391
  .expect("a peer connection builds without a network");
3392
+ test_inner_with(events, peer, 64, 64)
3393
+ }
3394
+
3395
+ #[cfg(target_os = "linux")]
3396
+ #[tokio::test]
3397
+ async fn a_wayland_portal_session_reports_typed_point_unsupported() {
3398
+ let (events, _) = tokio_mpsc::unbounded_channel();
3399
+ let (mut inner, _) = test_inner(events).await;
3400
+ Arc::get_mut(&mut inner)
3401
+ .expect("test session has one owner")
3402
+ .permissions = vec![Permission::View];
3403
+ let session = Session { inner };
3404
+
3405
+ let error = session
3406
+ .point(&crate::protocol::PointParams {
3407
+ session_id: String::from("test-session"),
3408
+ x: Some(10),
3409
+ y: Some(20),
3410
+ label: String::new(),
3411
+ timeout_ms: None,
3412
+ clear: false,
3413
+ })
3414
+ .expect_err("compositors without layer-shell do not support point overlays");
3415
+
3416
+ assert_eq!(error.code, "point_unsupported");
3417
+ assert!(error.message.contains("reason: layer_shell_unavailable"));
3418
+ }
3419
+
3420
+ // Driven only on the private headless compositor created by point-wayland-flow.py.
3421
+ #[cfg(target_os = "linux")]
3422
+ #[tokio::test]
3423
+ #[ignore = "requires the task-owned compositor and screenshot driver"]
3424
+ async fn point_wayland_private_flow() {
3425
+ let evidence = std::path::PathBuf::from(
3426
+ std::env::var_os("DESKLINK_POINT_TEST_DIR").expect("private evidence"),
3427
+ );
3428
+ let socket =
3429
+ std::path::PathBuf::from(std::env::var_os("WAYLAND_DISPLAY").expect("private socket"));
3430
+ assert!(
3431
+ socket.is_absolute() && socket.starts_with(evidence.join("runtime")),
3432
+ "never use an ambient desktop"
3433
+ );
3434
+ fn checkpoint(dir: &std::path::Path, name: &str) {
3435
+ std::fs::write(dir.join(format!("{name}.ready")), b"").unwrap();
3436
+ let deadline = Instant::now() + Duration::from_secs(15);
3437
+ while !dir.join(format!("{name}.continue")).exists() {
3438
+ assert!(
3439
+ Instant::now() < deadline,
3440
+ "screenshot driver timed out at {name}"
3441
+ );
3442
+ std::thread::sleep(Duration::from_millis(20));
3443
+ }
3444
+ }
3445
+ let (events, _) = tokio_mpsc::unbounded_channel();
3446
+ let (mut inner, _) = test_inner(events).await;
3447
+ let state = Arc::get_mut(&mut inner).unwrap();
3448
+ state.permissions = vec![Permission::View];
3449
+ state.point_desktop = PointDesktop::Wayland;
3450
+ state.source.width = 1280;
3451
+ state.source.height = 720;
3452
+ state.source.position = Some((1280, 0));
3453
+ let mut session = Session { inner };
3454
+ let mut params = crate::protocol::PointParams {
3455
+ session_id: String::from("test-session"),
3456
+ x: Some(400),
3457
+ y: Some(300),
3458
+ label: String::from("Here"),
3459
+ timeout_ms: Some(3000),
3460
+ clear: false,
3461
+ };
3462
+ let unsupported_socket = std::env::var_os("DESKLINK_POINT_UNSUPPORTED_SOCKET").unwrap();
3463
+ assert!(std::path::Path::new(&unsupported_socket).starts_with(evidence.join("runtime")));
3464
+ // This test runs alone, in a separate process. Only child helper startup
3465
+ // and the read-only clear probe read this private socket environment.
3466
+ std::env::set_var("WAYLAND_DISPLAY", &unsupported_socket);
3467
+ let refusal = session.point(&params).unwrap_err();
3468
+ assert_eq!(refusal.code, "point_unsupported");
3469
+ assert!(refusal.message.contains("reason: layer_shell_unavailable"));
3470
+ let clear = crate::protocol::PointParams {
3471
+ session_id: params.session_id.clone(),
3472
+ clear: true,
3473
+ x: None,
3474
+ y: None,
3475
+ label: String::new(),
3476
+ timeout_ms: None,
3477
+ };
3478
+ let refusal = session.point(&clear).unwrap_err();
3479
+ assert_eq!(refusal.code, "point_unsupported");
3480
+ std::env::set_var("WAYLAND_DISPLAY", &socket);
3481
+ Arc::get_mut(&mut session.inner).unwrap().source.position = None;
3482
+ let ambiguous = session.point(&params).unwrap_err();
3483
+ assert_eq!(ambiguous.code, "indicator-unavailable");
3484
+ assert!(ambiguous.message.contains("selected_output_unavailable"));
3485
+ Arc::get_mut(&mut session.inner).unwrap().source.position = Some((1280, 0));
3486
+ checkpoint(&evidence, "before");
3487
+ session.point(&params).unwrap();
3488
+ checkpoint(&evidence, "after");
3489
+ params.x = Some(1280);
3490
+ assert_eq!(session.point(&params).unwrap_err().code, "coordinates");
3491
+ params.x = Some(400);
3492
+ params.label = String::from("bad\nlabel");
3493
+ assert_eq!(session.point(&params).unwrap_err().code, "malformed");
3494
+ checkpoint(&evidence, "invalid");
3495
+ tokio::time::sleep(Duration::from_millis(3100)).await;
3496
+ checkpoint(&evidence, "expired");
3497
+ params.x = Some(700);
3498
+ params.y = Some(500);
3499
+ params.label.clear();
3500
+ params.timeout_ms = Some(10000);
3501
+ session.point(&params).unwrap();
3502
+ checkpoint(&evidence, "replaced");
3503
+ params.clear = true;
3504
+ params.x = None;
3505
+ params.y = None;
3506
+ params.timeout_ms = None;
3507
+ session.point(&params).unwrap();
3508
+ session.point(&params).unwrap();
3509
+ assert!(lock(&session.inner.point_overlay).is_none());
3510
+ checkpoint(&evidence, "cleared");
3511
+ params.clear = false;
3512
+ params.x = Some(400);
3513
+ params.y = Some(300);
3514
+ session.point(&params).unwrap();
3515
+ let pid = lock(&session.inner.point_overlay)
3516
+ .as_ref()
3517
+ .unwrap()
3518
+ .test_pid();
3519
+ checkpoint(&evidence, "clickthrough");
3520
+ session.close("test complete").await;
3521
+ assert_eq!(
3522
+ unsafe { libc::kill(pid as i32, 0) },
3523
+ -1,
3524
+ "helper reaped on session close"
3525
+ );
3526
+ checkpoint(&evidence, "closed");
3527
+ }
3528
+
3529
+ fn test_inner_with(
3530
+ events: tokio_mpsc::UnboundedSender<Notice>,
3531
+ peer: VideoPeer,
3532
+ width: usize,
3533
+ height: usize,
3534
+ ) -> (Arc<Inner>, Arc<Mutex<Vec<(i16, bool)>>>) {
2661
3535
  // The pipeline codes nothing before the transport connects.
2662
3536
  peer.assume_connected();
2663
3537
  let recorded = Arc::new(Mutex::new(Vec::new()));
@@ -2684,7 +3558,9 @@ mod tests {
2684
3558
  capture_error: Arc::new(Mutex::new(None)),
2685
3559
  frame_changes: tokio::sync::watch::channel(0).1,
2686
3560
  peer: Mutex::new(Some(Arc::new(peer))),
2687
- encoder: Mutex::new(Some(Encoder::new(64, 64, 1000, 30, 1).expect("an encoder"))),
3561
+ encoder: Mutex::new(Some(VideoEncoder::Vp9(
3562
+ Encoder::new(width, height, 1000, 30, 1).expect("an encoder"),
3563
+ ))),
2688
3564
  input: Mutex::new(Some(InputTarget {
2689
3565
  applier: Applier::Recording(recorded.clone()),
2690
3566
  held: HeldState::default(),
@@ -2695,6 +3571,9 @@ mod tests {
2695
3571
  })),
2696
3572
  capture: Mutex::new(None),
2697
3573
  indicator: Mutex::new(None),
3574
+ point_overlay: Mutex::new(None),
3575
+ #[cfg(target_os = "linux")]
3576
+ point_desktop: PointDesktop::Unavailable("layer_shell_unavailable"),
2698
3577
  layout: Mutex::new(Some(Layout::from_environment().expect("a keymap"))),
2699
3578
  last_seq: Mutex::new(0),
2700
3579
  control_open: AtomicBool::new(true),
@@ -2702,6 +3581,8 @@ mod tests {
2702
3581
  pipeline: Arc::new(AtomicBool::new(true)),
2703
3582
  events,
2704
3583
  encoded: Mutex::new(None),
3584
+ local_frames: true,
3585
+ cursor_info: serde_json::json!({"mode":"embedded", "positions":false}),
2705
3586
  });
2706
3587
  (inner, recorded)
2707
3588
  }
@@ -2711,6 +3592,11 @@ mod tests {
2711
3592
  let (events, mut received) = tokio_mpsc::unbounded_channel();
2712
3593
  let (inner, recorded) = test_inner(events).await;
2713
3594
 
3595
+ #[cfg(target_os = "windows")]
3596
+ inner
3597
+ .with_input(|target| target.chord(1, Vec::new(), true))
3598
+ .unwrap();
3599
+ #[cfg(not(target_os = "windows"))]
2714
3600
  inner.apply(ControlMessage::Key {
2715
3601
  name: None,
2716
3602
  character: Some(String::from("a")),
@@ -2742,11 +3628,11 @@ mod tests {
2742
3628
  matches!(
2743
3629
  received.try_recv(),
2744
3630
  Ok(Notice {
2745
- event: SessionEvent::Revoked { .. },
3631
+ event: SessionEvent::Revoked { code: "lease", .. },
2746
3632
  ..
2747
3633
  })
2748
3634
  ),
2749
- "the consumer is still told why the session ended",
3635
+ "the consumer is still told the session's authority ended",
2750
3636
  );
2751
3637
 
2752
3638
  let rejected = inner.metrics.lock().unwrap().input_rejected;
@@ -2764,6 +3650,7 @@ mod tests {
2764
3650
  );
2765
3651
  }
2766
3652
 
3653
+ #[cfg(not(target_os = "windows"))]
2767
3654
  #[tokio::test]
2768
3655
  async fn a_held_modifier_survives_character_named_and_text_chords() {
2769
3656
  use crate::input::keycode;
@@ -3020,20 +3907,24 @@ mod tests {
3020
3907
 
3021
3908
  // The encoder is 64x64; a 32x32 frame is the dimension mismatch that
3022
3909
  // used to be counted and dropped behind a permanently black picture.
3023
- frame_tx.put(I420 {
3024
- width: 32,
3025
- height: 32,
3026
- data: vec![128u8; 32 * 32 + 2 * 16 * 16],
3027
- });
3910
+ frame_tx.put(
3911
+ I420 {
3912
+ width: 32,
3913
+ height: 32,
3914
+ data: vec![128u8; 32 * 32 + 2 * 16 * 16],
3915
+ },
3916
+ Instant::now(),
3917
+ );
3028
3918
  tokio::time::sleep(Duration::from_millis(150)).await;
3029
3919
 
3030
3920
  let mut reason = None;
3031
3921
  while let Ok(event) = received.try_recv() {
3032
3922
  if let Notice {
3033
- event: SessionEvent::Revoked { reason: why },
3923
+ event: SessionEvent::Revoked { reason: why, code },
3034
3924
  ..
3035
3925
  } = event
3036
3926
  {
3927
+ assert_eq!(code, "error", "a refused frame is a fault, not a lost path");
3037
3928
  reason = Some(why);
3038
3929
  }
3039
3930
  }
@@ -3059,6 +3950,7 @@ mod tests {
3059
3950
 
3060
3951
  fn encoded_open() -> OpenRequest {
3061
3952
  OpenRequest {
3953
+ cursor: crate::protocol::CursorMode::Embedded,
3062
3954
  source: Some(SourceRequest::Encoded {
3063
3955
  codec: EncodedCodec::H264,
3064
3956
  width: 486,
@@ -3074,6 +3966,7 @@ mod tests {
3074
3966
  ttl: Some(Duration::from_secs(30)),
3075
3967
  loopback_tcp: false,
3076
3968
  agent_indicator: false,
3969
+ local_frames: true,
3077
3970
  }
3078
3971
  }
3079
3972
 
@@ -3084,6 +3977,18 @@ mod tests {
3084
3977
  .expect("the channel stays open")
3085
3978
  }
3086
3979
 
3980
+ #[tokio::test]
3981
+ async fn hidden_cursor_is_refused_for_consumer_encoded_pixels() {
3982
+ let (events, _) = tokio_mpsc::unbounded_channel();
3983
+ let mut request = encoded_open();
3984
+ request.cursor = crate::protocol::CursorMode::Hidden;
3985
+ let error = Session::open(request, events)
3986
+ .await
3987
+ .err()
3988
+ .expect("cannot alter encoded pixels");
3989
+ assert_eq!(error.code, "cursor-unavailable");
3990
+ }
3991
+
3087
3992
  #[tokio::test]
3088
3993
  async fn an_encoded_session_opens_without_capture_encoder_or_input() {
3089
3994
  let (events, _received) = tokio_mpsc::unbounded_channel();
@@ -3230,6 +4135,7 @@ mod tests {
3230
4135
  let (events, _received) = tokio_mpsc::unbounded_channel();
3231
4136
  let result = Session::open(
3232
4137
  OpenRequest {
4138
+ cursor: crate::protocol::CursorMode::Embedded,
3233
4139
  source: None,
3234
4140
  permissions: vec![Permission::Control],
3235
4141
  max_width: 640,
@@ -3241,6 +4147,7 @@ mod tests {
3241
4147
  ttl: None,
3242
4148
  loopback_tcp: false,
3243
4149
  agent_indicator: false,
4150
+ local_frames: true,
3244
4151
  },
3245
4152
  events,
3246
4153
  )
@@ -3257,6 +4164,11 @@ mod tests {
3257
4164
  use webrtc::peer_connection::RTCPeerConnectionState as State;
3258
4165
  let (events, mut received) = tokio_mpsc::unbounded_channel();
3259
4166
  let (inner, recorded) = test_inner(events).await;
4167
+ #[cfg(target_os = "windows")]
4168
+ inner
4169
+ .with_input(|target| target.chord(1, Vec::new(), true))
4170
+ .unwrap();
4171
+ #[cfg(not(target_os = "windows"))]
3260
4172
  inner.apply(ControlMessage::Key {
3261
4173
  name: Some(String::from("Control")),
3262
4174
  character: None,
@@ -3282,8 +4194,17 @@ mod tests {
3282
4194
  );
3283
4195
 
3284
4196
  peer_tx.send(PeerEvent::State(State::Failed)).unwrap();
4197
+ let mut transport_lost = false;
3285
4198
  tokio::time::timeout(Duration::from_secs(2), async {
3286
4199
  while let Some(notice) = received.recv().await {
4200
+ // A lost path is the one revocation a new session can heal.
4201
+ transport_lost |= matches!(
4202
+ notice.event,
4203
+ SessionEvent::Revoked {
4204
+ code: "transport",
4205
+ ..
4206
+ }
4207
+ );
3287
4208
  if matches!(
3288
4209
  notice.event,
3289
4210
  SessionEvent::State {
@@ -3298,11 +4219,158 @@ mod tests {
3298
4219
  })
3299
4220
  .await
3300
4221
  .unwrap();
4222
+ assert!(
4223
+ transport_lost,
4224
+ "a failed transport must revoke with code transport"
4225
+ );
3301
4226
  assert!(inner.revoked_reason().is_some());
3302
4227
  assert!(!inner.pipeline.load(Ordering::SeqCst));
3303
4228
  assert!(inner.input.lock().unwrap().is_none());
3304
4229
  }
3305
4230
 
4231
+ #[test]
4232
+ fn a_standing_queue_backs_the_rate_off_to_what_the_path_delivers() {
4233
+ let start = Instant::now();
4234
+ let queued = |ms: u64, delivered_kbps: u32| {
4235
+ Some(PathReport {
4236
+ queue_delay: Duration::from_millis(ms),
4237
+ delivered_kbps,
4238
+ })
4239
+ };
4240
+ let mut rate = RateControl::new(8000);
4241
+ // A burst's worth of queue and no loss moves nothing.
4242
+ assert_eq!(rate.update(&[], queued(15, 3000), start), None);
4243
+ // A standing queue on a 4 Mbps link: 85% of what got through, at once,
4244
+ // with no loss reported at all.
4245
+ assert_eq!(rate.update(&[], queued(80, 4000), start), Some(3400));
4246
+ // Not again until the back-off has had time to show in the feedback.
4247
+ assert_eq!(
4248
+ rate.update(&[], queued(80, 3400), start + Duration::from_millis(100)),
4249
+ None
4250
+ );
4251
+ assert_eq!(
4252
+ rate.update(&[], queued(60, 3400), start + Duration::from_millis(300)),
4253
+ Some(2890)
4254
+ );
4255
+ // A stream that sent little is not taken for a slow link: at most half.
4256
+ assert_eq!(
4257
+ rate.update(&[], queued(60, 100), start + Duration::from_millis(600)),
4258
+ Some(1228)
4259
+ );
4260
+ // Clean loss reports do not raise it while the queue has not drained...
4261
+ assert_eq!(
4262
+ rate.update(&[0], queued(15, 1200), start + Duration::from_secs(2)),
4263
+ None
4264
+ );
4265
+ // ...and do once it has.
4266
+ assert_eq!(
4267
+ rate.update(&[0], queued(2, 1200), start + Duration::from_secs(3)),
4268
+ Some(1390)
4269
+ );
4270
+ // Nothing drops below the floor.
4271
+ let mut rate = RateControl::new(8000);
4272
+ for step in 0..20u64 {
4273
+ rate.update(
4274
+ &[],
4275
+ queued(200, 1),
4276
+ start + DELAY_BACKOFF_EVERY * step as u32,
4277
+ );
4278
+ }
4279
+ assert_eq!(rate.target, 1000);
4280
+ }
4281
+
4282
+ #[test]
4283
+ fn without_transport_feedback_the_rate_follows_loss_alone() {
4284
+ let start = Instant::now();
4285
+ let mut rate = RateControl::new(8000);
4286
+ assert_eq!(rate.update(&[], None, start), None);
4287
+ assert_eq!(rate.update(&[64], None, start), Some(5600));
4288
+ assert_eq!(
4289
+ rate.update(&[64], None, start + Duration::from_millis(500)),
4290
+ None
4291
+ );
4292
+ assert_eq!(
4293
+ rate.update(&[10], None, start + Duration::from_secs(1)),
4294
+ None
4295
+ );
4296
+ assert_eq!(
4297
+ rate.update(&[0], None, start + Duration::from_secs(2)),
4298
+ Some(6112)
4299
+ );
4300
+ }
4301
+
4302
+ #[test]
4303
+ fn a_large_frame_holds_the_next_until_the_link_has_carried_it() {
4304
+ let start = Instant::now();
4305
+ let mut budget = SendBudget::default();
4306
+ // Frames at the target, one per 20 ms at 4000 kbps, never wait, and
4307
+ // neither does the ordinary overshoot of a frame at twice the budget.
4308
+ for n in 0..50u32 {
4309
+ let at = start + Duration::from_millis(20) * n;
4310
+ assert_eq!(budget.wait(at, 4000), Duration::ZERO, "frame {n} waited");
4311
+ budget.sent(at, if n % 5 == 0 { 20_000 } else { 10_000 }, 4000);
4312
+ }
4313
+ // A 100 kB key frame drains in 160 ms (4000 kbps and a quarter): it
4314
+ // goes at once, and the next waits out all of it past the 60 ms burst
4315
+ // allowance.
4316
+ let at = start + Duration::from_secs(1);
4317
+ assert_eq!(budget.wait(at, 4000), Duration::ZERO);
4318
+ budget.sent(at, 100_000, 4000);
4319
+ let wait = budget.wait(at, 4000);
4320
+ assert!(
4321
+ (Duration::from_millis(99)..=Duration::from_millis(101)).contains(&wait),
4322
+ "waited {wait:?}"
4323
+ );
4324
+ assert_eq!(budget.wait(at + wait, 4000), Duration::ZERO);
4325
+ }
4326
+
4327
+ #[test]
4328
+ fn motion_halves_only_when_the_encoder_sets_the_pace_and_does_not_flap() {
4329
+ let interval = Duration::from_millis(16);
4330
+ let (slow, fast) = (Duration::from_millis(30), Duration::from_millis(2));
4331
+ let mut size = MotionSize::default();
4332
+ for _ in 0..MotionSize::SLOW_RUN * 4 {
4333
+ size.coded(slow, false, true, interval);
4334
+ }
4335
+ assert!(
4336
+ !size.half,
4337
+ "slow small frames (typing on a busy host) keep full size"
4338
+ );
4339
+ for _ in 0..MotionSize::SLOW_RUN * 4 {
4340
+ size.coded(slow, true, false, interval);
4341
+ }
4342
+ assert!(
4343
+ !size.half,
4344
+ "a source slower than the cap (30 fps, 30 ms encode) never halves"
4345
+ );
4346
+ for _ in 0..MotionSize::SLOW_RUN - 1 {
4347
+ size.coded(slow, true, true, interval);
4348
+ }
4349
+ size.coded(fast, true, true, interval);
4350
+ assert!(!size.half, "a broken run of slow frames keeps full size");
4351
+ for _ in 0..MotionSize::SLOW_RUN {
4352
+ size.coded(slow, true, true, interval);
4353
+ }
4354
+ assert!(size.half, "a run of slow large frames halves");
4355
+ // The marginal band: half-size frames that are small but still slow,
4356
+ // between large ones. Neither may send it back to full size.
4357
+ for i in 0..MotionSize::SMALL_RUN * 10 {
4358
+ size.coded(slow, i % 7 == 0, true, interval);
4359
+ assert!(size.half, "marginal half-size motion must not flap back");
4360
+ }
4361
+ for _ in 0..MotionSize::SMALL_RUN * 2 {
4362
+ size.coded(fast, true, true, interval);
4363
+ }
4364
+ assert!(size.half, "large half-size frames stay half, however fast");
4365
+ for _ in 0..MotionSize::SMALL_RUN {
4366
+ size.coded(fast, false, true, interval);
4367
+ }
4368
+ assert!(
4369
+ !size.half,
4370
+ "a run of small, fast half-size frames returns to full size"
4371
+ );
4372
+ }
4373
+
3306
4374
  #[tokio::test]
3307
4375
  async fn the_pipeline_caps_the_rate_codes_the_last_frame_and_refines_a_still_desktop_once() {
3308
4376
  let (events, _events_rx) = tokio_mpsc::unbounded_channel();
@@ -3317,19 +4385,29 @@ mod tests {
3317
4385
  1000,
3318
4386
  );
3319
4387
 
3320
- // 40 frames over 200 ms is 200 fps; the requested 20 fps caps what leaves.
4388
+ // 40 frames nominally over 200 ms; Windows timers can round each 5 ms
4389
+ // sleep upward, so its cap assertion uses the actual burst duration.
4390
+ #[cfg(target_os = "windows")]
4391
+ let burst_started = Instant::now();
3321
4392
  for _ in 0..40 {
3322
- frame_tx.put(I420 {
3323
- width: 64,
3324
- height: 64,
3325
- data: vec![128u8; 64 * 64 + 2 * 32 * 32],
3326
- });
4393
+ frame_tx.put(
4394
+ I420 {
4395
+ width: 64,
4396
+ height: 64,
4397
+ data: vec![128u8; 64 * 64 + 2 * 32 * 32],
4398
+ },
4399
+ Instant::now(),
4400
+ );
3327
4401
  tokio::time::sleep(Duration::from_millis(5)).await;
3328
4402
  }
3329
4403
  let motion = inner.metrics.lock().unwrap().encoded_frames;
4404
+ #[cfg(target_os = "windows")]
4405
+ let cap = (burst_started.elapsed().as_secs_f64() * 20.0).ceil() as u64 + 3;
4406
+ #[cfg(not(target_os = "windows"))]
4407
+ let cap = 7;
3330
4408
  assert!(
3331
- motion <= 7,
3332
- "the requested rate is a cap, got {motion} of 40"
4409
+ motion <= cap,
4410
+ "the requested rate is a cap, got {motion} of 40 (cap {cap})"
3333
4411
  );
3334
4412
  assert!(
3335
4413
  motion >= 2,
@@ -3362,5 +4440,74 @@ mod tests {
3362
4440
  "a still stream still sends"
3363
4441
  );
3364
4442
  assert_eq!((later.key_frames, later.refined_frames), (1, 1));
4443
+ assert_eq!((later.codec, later.encoder), ("vp9", "libvpx"));
4444
+ }
4445
+
4446
+ #[tokio::test]
4447
+ async fn a_receiver_that_asks_for_h264_is_sent_h264_from_the_first_frame() {
4448
+ let Some(info) = tokio::task::spawn_blocking(crate::h264::probe)
4449
+ .await
4450
+ .unwrap()
4451
+ else {
4452
+ eprintln!("skipped: no H.264 encoder starts on this machine");
4453
+ return;
4454
+ };
4455
+ let (peer, _answer) =
4456
+ crate::peer::tests::answered(vec![crate::peer::tests::h264_receiver_codec()]).await;
4457
+ let (events, _events_rx) = tokio_mpsc::unbounded_channel();
4458
+ let (inner, _recorded) = test_inner_with(events, peer, 320, 240);
4459
+ // The answer is real, so ICE runs and, with no candidates, never
4460
+ // connects; each state it reports would take back `assume_connected`.
4461
+ let pinned = current_peer(&inner).expect("the test's peer");
4462
+ let pin = tokio::spawn(async move {
4463
+ loop {
4464
+ pinned.assume_connected();
4465
+ tokio::time::sleep(Duration::from_millis(5)).await;
4466
+ }
4467
+ });
4468
+ let (frame_tx, frame_rx) = latest_frame();
4469
+ spawn_pipeline(
4470
+ &inner,
4471
+ current_peer(&inner).expect("the test's peer"),
4472
+ frame_rx,
4473
+ Arc::new(AtomicBool::new(true)),
4474
+ 20,
4475
+ 1000,
4476
+ );
4477
+ for shade in 0..10u8 {
4478
+ frame_tx.put(
4479
+ I420 {
4480
+ width: 320,
4481
+ height: 240,
4482
+ data: vec![shade * 20; 320 * 240 * 3 / 2],
4483
+ },
4484
+ Instant::now(),
4485
+ );
4486
+ tokio::time::sleep(Duration::from_millis(50)).await;
4487
+ }
4488
+ // Opening a hardware session can take a good part of a second, more
4489
+ // with other tests holding the GPU: wait for the still to be refined.
4490
+ let deadline = Instant::now() + Duration::from_secs(10);
4491
+ while inner.metrics.lock().unwrap().refined_frames == 0 && Instant::now() < deadline {
4492
+ tokio::time::sleep(Duration::from_millis(50)).await;
4493
+ }
4494
+ let metrics = inner.metrics.lock().unwrap().clone();
4495
+ assert_eq!((metrics.codec, metrics.encoder), ("h264", info.encoder));
4496
+ assert!(metrics.encoded_frames >= 2, "{metrics:?}");
4497
+ assert_eq!(
4498
+ metrics.key_frames, 1,
4499
+ "one IDR, then inter frames: {metrics:?}"
4500
+ );
4501
+ assert_eq!(metrics.refined_frames, 1, "a still desktop is refined once");
4502
+ assert_eq!(
4503
+ metrics.halved_frames, 0,
4504
+ "H.264 is always coded at full size"
4505
+ );
4506
+ pin.abort();
4507
+ inner.close("done", "closed").await;
4508
+ assert!(
4509
+ lock(&inner.encoder).is_none(),
4510
+ "closing releases the encoder"
4511
+ );
3365
4512
  }
3366
4513
  }