@desklink/host 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +26 -0
- package/README.md +134 -56
- package/bin/desklink-host.mjs +13 -5
- package/dist/bridge.d.ts +1 -0
- package/dist/bridge.d.ts.map +1 -1
- package/dist/bridge.js +12 -4
- package/dist/bridge.js.map +1 -1
- package/dist/engineProcess.d.ts +6 -1
- package/dist/engineProcess.d.ts.map +1 -1
- package/dist/engineProcess.js +35 -4
- package/dist/engineProcess.js.map +1 -1
- package/dist/protocol.d.ts +58 -0
- package/dist/protocol.d.ts.map +1 -1
- package/dist/protocol.js.map +1 -1
- package/dist/resolveEngine.d.ts +7 -1
- package/dist/resolveEngine.d.ts.map +1 -1
- package/dist/resolveEngine.js +66 -14
- package/dist/resolveEngine.js.map +1 -1
- package/docs/BOUNDARY.md +15 -7
- package/docs/PROTOCOL.md +267 -27
- package/engine/Cargo.lock +47 -2
- package/engine/Cargo.toml +18 -13
- package/engine/build.rs +86 -4
- package/engine/native/agent_overlay_mac.m +51 -4
- package/engine/native/agent_overlay_wayland.c +145 -12
- package/engine/native/agent_overlay_x11.c +72 -0
- package/engine/native/nvenc_shim.c +453 -0
- package/engine/native/vaapi_shim.c +727 -0
- package/engine/native/vpx_shim.c +54 -4
- package/engine/native/vt_shim.c +448 -0
- package/engine/src/capture.rs +167 -58
- package/engine/src/capture_mac.rs +24 -14
- package/engine/src/capture_win.rs +23 -0
- package/engine/src/clipboard_win.rs +8 -0
- package/engine/src/convert.rs +286 -61
- package/engine/src/cursor.rs +118 -0
- package/engine/src/encoder.rs +122 -13
- package/engine/src/h264.rs +188 -0
- package/engine/src/h264_nvenc.rs +308 -0
- package/engine/src/h264_openh264.rs +735 -0
- package/engine/src/h264_vaapi.rs +263 -0
- package/engine/src/h264_vt.rs +368 -0
- package/engine/src/indicator.rs +104 -3
- package/engine/src/input_win.rs +91 -0
- package/engine/src/keymap_win.rs +39 -0
- package/engine/src/mac.rs +34 -9
- package/engine/src/main.rs +186 -26
- package/engine/src/peer.rs +466 -28
- package/engine/src/portal.rs +42 -4
- package/engine/src/portal_win.rs +37 -0
- package/engine/src/protocol.rs +64 -2
- package/engine/src/session.rs +1252 -105
- package/engine/src/win.rs +110 -0
- package/engine/src/x11.rs +225 -9
- package/engine/src/x11_mac.rs +12 -0
- package/engine/vendor/libva/COPYING +19 -0
- package/engine/vendor/libva/va/va.h +5450 -0
- package/engine/vendor/libva/va/va_dec_av1.h +695 -0
- package/engine/vendor/libva/va/va_dec_hevc.h +616 -0
- package/engine/vendor/libva/va/va_dec_jpeg.h +157 -0
- package/engine/vendor/libva/va/va_dec_vp8.h +249 -0
- package/engine/vendor/libva/va/va_dec_vp9.h +312 -0
- package/engine/vendor/libva/va/va_dec_vvc.h +688 -0
- package/engine/vendor/libva/va/va_drm.h +61 -0
- package/engine/vendor/libva/va/va_enc_av1.h +1010 -0
- package/engine/vendor/libva/va/va_enc_h264.h +659 -0
- package/engine/vendor/libva/va/va_enc_hevc.h +958 -0
- package/engine/vendor/libva/va/va_enc_jpeg.h +162 -0
- package/engine/vendor/libva/va/va_enc_mpeg2.h +308 -0
- package/engine/vendor/libva/va/va_enc_vp8.h +350 -0
- package/engine/vendor/libva/va/va_enc_vp9.h +639 -0
- package/engine/vendor/libva/va/va_fei.h +170 -0
- package/engine/vendor/libva/va/va_fei_h264.h +505 -0
- package/engine/vendor/libva/va/va_fei_hevc.h +254 -0
- package/engine/vendor/libva/va/va_prot.h +525 -0
- package/engine/vendor/libva/va/va_version.h +87 -0
- package/engine/vendor/libva/va/va_vpp.h +1590 -0
- package/engine/vendor/nv-codec-headers/LICENSE +24 -0
- package/engine/vendor/nv-codec-headers/README +7 -0
- package/engine/vendor/nv-codec-headers/include/ffnvcodec/nvEncodeAPI.h +3825 -0
- package/examples/reference-client.html +1 -1
- package/package.json +4 -3
package/engine/src/session.rs
CHANGED
|
@@ -8,16 +8,18 @@
|
|
|
8
8
|
use crate::capture::{self, Capture};
|
|
9
9
|
use crate::clipboard;
|
|
10
10
|
use crate::convert::{fit, I420};
|
|
11
|
-
use crate::encoder::Encoder;
|
|
11
|
+
use crate::encoder::{Encoder, VideoEncoder};
|
|
12
12
|
use crate::input::{Button, HeldState, InputDevices};
|
|
13
13
|
use crate::keymap::{self, Layout};
|
|
14
|
-
use crate::peer::{
|
|
14
|
+
use crate::peer::{
|
|
15
|
+
h264_profile_level_id, PathReport, PeerEvent, TransportOptions, VideoCodec, VideoPeer,
|
|
16
|
+
};
|
|
15
17
|
use crate::portal::{self, SelectedSource};
|
|
16
18
|
use crate::protocol::{
|
|
17
19
|
ControlMessage, ControlReply, EncodedCodec, Permission, PointerPhase, SourceRequest,
|
|
18
20
|
};
|
|
19
21
|
use crate::x11::X11Desktop;
|
|
20
|
-
use anyhow::Result;
|
|
22
|
+
use anyhow::{Context, Result};
|
|
21
23
|
use std::sync::atomic::{AtomicBool, Ordering};
|
|
22
24
|
use std::sync::{Arc, Condvar, Mutex};
|
|
23
25
|
use std::time::{Duration, Instant};
|
|
@@ -51,14 +53,41 @@ pub struct Metrics {
|
|
|
51
53
|
pub dropped_frames: u64,
|
|
52
54
|
pub encoded_frames: u64,
|
|
53
55
|
pub encoded_bytes: u64,
|
|
54
|
-
/// Key frames sent: the first,
|
|
56
|
+
/// Key frames sent: the first, one per receiver request, and one each time
|
|
57
|
+
/// motion starts or stops being coded at half size.
|
|
55
58
|
pub key_frames: u64,
|
|
56
59
|
/// Refinement passes: a still desktop re-coded once, sharp.
|
|
57
60
|
pub refined_frames: u64,
|
|
61
|
+
/// Motion frames coded at half size because full size could not keep up.
|
|
62
|
+
pub halved_frames: u64,
|
|
63
|
+
/// Fresh captures coded at the pace of change: a motion pass on a frame
|
|
64
|
+
/// the encoder had not coded yet — the population `queue_micros` is over.
|
|
65
|
+
pub motion_frames: u64,
|
|
58
66
|
/// Time spent in the encoder, in microseconds, over every encoded frame.
|
|
59
67
|
pub encode_micros: u64,
|
|
68
|
+
/// Time spent grabbing and converting a frame on the capture thread, in
|
|
69
|
+
/// microseconds, over every frame forwarded to the encoder. On X11 this
|
|
70
|
+
/// is `GetImage` plus the downscale to I420; unchanged polls are hashed,
|
|
71
|
+
/// recognised, and never forwarded, so their grab cost stays out; the lab
|
|
72
|
+
/// harness reads it as the grab cost L1 owns.
|
|
73
|
+
pub capture_micros: u64,
|
|
74
|
+
/// Time spent hashing a captured frame and working out its damage on the
|
|
75
|
+
/// capture thread, in microseconds, over every forwarded frame; the hash
|
|
76
|
+
/// branch runs only when `local_frames` is on.
|
|
77
|
+
pub convert_micros: u64,
|
|
78
|
+
/// Time a freshly captured frame waited between the capture handoff and
|
|
79
|
+
/// the start of its encode, in microseconds, over motion frames.
|
|
80
|
+
pub queue_micros: u64,
|
|
81
|
+
/// Time spent packetizing a frame and handing it to the transport, in
|
|
82
|
+
/// microseconds, over every sent frame.
|
|
83
|
+
pub send_micros: u64,
|
|
60
84
|
/// The current rate target, after any loss back-off.
|
|
61
85
|
pub target_kbps: u32,
|
|
86
|
+
/// The codec frames go out in, `vp9` or `h264`, and what codes them
|
|
87
|
+
/// (`libvpx`, `nvenc`, `vaapi`, `openh264`, `videotoolbox`); empty for an
|
|
88
|
+
/// encoded source, whose consumer is the encoder.
|
|
89
|
+
pub codec: &'static str,
|
|
90
|
+
pub encoder: &'static str,
|
|
62
91
|
pub input_applied: u64,
|
|
63
92
|
pub input_rejected: u64,
|
|
64
93
|
/// Access units the consumer fed to an encoded source.
|
|
@@ -173,18 +202,21 @@ struct Selected {
|
|
|
173
202
|
|
|
174
203
|
/// Open an X display, start reading its root window, and hand back the same
|
|
175
204
|
/// shape the portal path produces.
|
|
205
|
+
#[allow(clippy::too_many_arguments)]
|
|
176
206
|
fn select_x11(
|
|
177
207
|
display: Option<&str>,
|
|
178
208
|
max_width: usize,
|
|
179
209
|
max_height: usize,
|
|
180
210
|
max_fps: u32,
|
|
181
211
|
metrics: Arc<Mutex<Metrics>>,
|
|
212
|
+
#[cfg(target_os = "linux")] cursor_sink: Option<crate::cursor::CursorSink>,
|
|
182
213
|
sink: capture::FrameSink,
|
|
183
214
|
on_stop: Box<dyn Fn(String) + Send>,
|
|
184
215
|
) -> Result<Selected> {
|
|
185
216
|
let desktop = Arc::new(Mutex::new(X11Desktop::connect(display)?));
|
|
186
217
|
let (width, height) = {
|
|
187
218
|
let desktop = lock(&desktop);
|
|
219
|
+
eprintln!("x11 capture path: {}", desktop.capture_path());
|
|
188
220
|
desktop.screen_size()
|
|
189
221
|
};
|
|
190
222
|
let source = SelectedSource {
|
|
@@ -207,26 +239,81 @@ fn select_x11(
|
|
|
207
239
|
.spawn(move || {
|
|
208
240
|
let interval = frame_interval(max_fps);
|
|
209
241
|
let mut sequence = 0u64;
|
|
210
|
-
|
|
211
|
-
//
|
|
212
|
-
//
|
|
242
|
+
let mut last_sent: Option<Instant> = None;
|
|
243
|
+
// XDamage wakes the loop, but pixels decide whether the root
|
|
244
|
+
// image actually changed; without XDamage, polling does so.
|
|
245
|
+
let hashed = !lock(&desktop).damage_armed();
|
|
213
246
|
let mut last_hash = None;
|
|
247
|
+
// The first frame is always grabbed; afterwards Damage wakes
|
|
248
|
+
// the loop, so a still screen costs no grab, no conversion
|
|
249
|
+
// and no hash. The fps cap stays a minimum spacing.
|
|
250
|
+
let mut first = true;
|
|
251
|
+
#[cfg(target_os = "linux")]
|
|
252
|
+
let mut cursor = cursor_sink.map(crate::cursor::Reporter::new);
|
|
214
253
|
while !stop.load(Ordering::SeqCst) {
|
|
254
|
+
#[cfg(target_os = "linux")]
|
|
255
|
+
if let Some(cursor) = cursor.as_mut() {
|
|
256
|
+
if let Ok((x, y, visible)) = lock(&desktop).cursor_position() {
|
|
257
|
+
cursor.update(x, y, visible);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
if !first {
|
|
261
|
+
let changed = lock(&desktop).take_damage();
|
|
262
|
+
if !changed {
|
|
263
|
+
std::thread::sleep(interval);
|
|
264
|
+
continue;
|
|
265
|
+
}
|
|
266
|
+
if let Some(sent) = last_sent {
|
|
267
|
+
let elapsed = sent.elapsed();
|
|
268
|
+
if elapsed < interval {
|
|
269
|
+
std::thread::sleep(interval - elapsed);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
}
|
|
215
273
|
let started = Instant::now();
|
|
274
|
+
let grabbed = Instant::now();
|
|
216
275
|
let frame = {
|
|
217
276
|
let mut desktop = lock(&desktop);
|
|
218
277
|
desktop.capture_with_pixels(max_width, max_height)
|
|
219
278
|
};
|
|
279
|
+
let grab_micros = grabbed.elapsed().as_micros() as u64;
|
|
220
280
|
match frame {
|
|
221
281
|
Ok((frame, raw)) => {
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
last_hash
|
|
228
|
-
|
|
282
|
+
let changed = {
|
|
283
|
+
use std::hash::{Hash, Hasher};
|
|
284
|
+
let mut hasher = std::collections::hash_map::DefaultHasher::new();
|
|
285
|
+
raw.hash(&mut hasher);
|
|
286
|
+
let hash = hasher.finish();
|
|
287
|
+
if last_hash == Some(hash) {
|
|
288
|
+
false
|
|
289
|
+
} else {
|
|
290
|
+
last_hash = Some(hash);
|
|
291
|
+
true
|
|
292
|
+
}
|
|
293
|
+
};
|
|
294
|
+
if changed {
|
|
295
|
+
// Handing an unchanged frame on would keep the
|
|
296
|
+
// encoder busy and a still desktop would never
|
|
297
|
+
// be refined.
|
|
298
|
+
if let Ok(mut m) = captured.lock() {
|
|
299
|
+
m.capture_micros += grab_micros;
|
|
300
|
+
}
|
|
301
|
+
let (w, h) = (frame.width, frame.height);
|
|
302
|
+
let (screen_w, screen_h) = (width as usize, height as usize);
|
|
303
|
+
sink(frame, sequence, &mut || {
|
|
304
|
+
crate::convert::to_bgrx(
|
|
305
|
+
&raw,
|
|
306
|
+
screen_w,
|
|
307
|
+
screen_h,
|
|
308
|
+
screen_w * 4,
|
|
309
|
+
crate::convert::PixelFormat::Bgrx,
|
|
310
|
+
w,
|
|
311
|
+
h,
|
|
312
|
+
)
|
|
313
|
+
});
|
|
229
314
|
sequence += 1;
|
|
315
|
+
last_sent = Some(started);
|
|
316
|
+
first = false;
|
|
230
317
|
}
|
|
231
318
|
}
|
|
232
319
|
Err(error) => {
|
|
@@ -239,11 +326,21 @@ fn select_x11(
|
|
|
239
326
|
on_stop(format!("X11 capture stopped: {error}"));
|
|
240
327
|
break;
|
|
241
328
|
}
|
|
329
|
+
if !hashed {
|
|
330
|
+
let elapsed = started.elapsed();
|
|
331
|
+
if elapsed < interval {
|
|
332
|
+
std::thread::sleep(interval - elapsed);
|
|
333
|
+
}
|
|
334
|
+
}
|
|
242
335
|
}
|
|
243
336
|
}
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
337
|
+
// With Damage the fps cap paced the grab above; without
|
|
338
|
+
// it the loop keeps its old trailing sleep.
|
|
339
|
+
if hashed {
|
|
340
|
+
let elapsed = started.elapsed();
|
|
341
|
+
if elapsed < interval {
|
|
342
|
+
std::thread::sleep(interval - elapsed);
|
|
343
|
+
}
|
|
247
344
|
}
|
|
248
345
|
}
|
|
249
346
|
})
|
|
@@ -391,10 +488,10 @@ impl InputTarget {
|
|
|
391
488
|
Ok(())
|
|
392
489
|
}
|
|
393
490
|
|
|
394
|
-
#[cfg(target_os = "macos")]
|
|
491
|
+
#[cfg(any(target_os = "macos", target_os = "windows"))]
|
|
395
492
|
fn unicode_text(&mut self, text: &str) -> Result<()> {
|
|
396
493
|
match &mut self.applier {
|
|
397
|
-
#[cfg(target_os = "macos")]
|
|
494
|
+
#[cfg(any(target_os = "macos", target_os = "windows"))]
|
|
398
495
|
Applier::Uinput(devices) => devices.unicode_text(text),
|
|
399
496
|
#[cfg(target_os = "linux")]
|
|
400
497
|
Applier::Uinput(_) => anyhow::bail!("Unicode text input is unavailable on Linux"),
|
|
@@ -493,6 +590,7 @@ impl SessionError {
|
|
|
493
590
|
}
|
|
494
591
|
|
|
495
592
|
pub struct OpenRequest {
|
|
593
|
+
pub cursor: crate::protocol::CursorMode,
|
|
496
594
|
/** Which desktop to capture; absent means the portal. */
|
|
497
595
|
pub source: Option<SourceRequest>,
|
|
498
596
|
pub permissions: Vec<Permission>,
|
|
@@ -506,11 +604,17 @@ pub struct OpenRequest {
|
|
|
506
604
|
/// Offer a passive ICE-TCP candidate on the loopback, for a forwarded client.
|
|
507
605
|
pub loopback_tcp: bool,
|
|
508
606
|
pub agent_indicator: bool,
|
|
607
|
+
/// Keep the latest frame and its tile hashes for `session.frame` and
|
|
608
|
+
/// `session.frame.changed`; off, a frame is converted once, for the encoder.
|
|
609
|
+
pub local_frames: bool,
|
|
509
610
|
}
|
|
510
611
|
|
|
511
612
|
/// Events a session raises for its consumer.
|
|
512
613
|
#[derive(Debug)]
|
|
513
614
|
pub enum SessionEvent {
|
|
615
|
+
Cursor {
|
|
616
|
+
position: crate::protocol::CursorPosition,
|
|
617
|
+
},
|
|
514
618
|
Description {
|
|
515
619
|
generation: u64,
|
|
516
620
|
sdp: String,
|
|
@@ -527,6 +631,7 @@ pub enum SessionEvent {
|
|
|
527
631
|
first_frame: bool,
|
|
528
632
|
},
|
|
529
633
|
Frame {
|
|
634
|
+
timestamp_us: u64,
|
|
530
635
|
seq: u64,
|
|
531
636
|
damage: Vec<[usize; 4]>,
|
|
532
637
|
},
|
|
@@ -546,8 +651,13 @@ pub enum SessionEvent {
|
|
|
546
651
|
CaptureStopped {
|
|
547
652
|
reason: String,
|
|
548
653
|
},
|
|
654
|
+
/// The session ended on the engine's side. `code` says whether a new
|
|
655
|
+
/// session may recover it: `transport` is the path to the consumer's peer
|
|
656
|
+
/// lost, which a fresh `session.open` can heal; `lease` and `error` are
|
|
657
|
+
/// the session's authority ending or a fault, which it cannot.
|
|
549
658
|
Revoked {
|
|
550
659
|
reason: String,
|
|
660
|
+
code: &'static str,
|
|
551
661
|
},
|
|
552
662
|
}
|
|
553
663
|
|
|
@@ -564,6 +674,13 @@ type FrameSnapshot = (u64, Instant, usize, usize, Vec<u8>, Vec<u64>, Vec<u64>);
|
|
|
564
674
|
|
|
565
675
|
/// Everything a session's background tasks need, shared rather than borrowed so
|
|
566
676
|
/// the consumer can own the `Session` handle while the pipeline runs.
|
|
677
|
+
#[cfg(target_os = "linux")]
|
|
678
|
+
enum PointDesktop {
|
|
679
|
+
X11(String),
|
|
680
|
+
Wayland,
|
|
681
|
+
Unavailable(&'static str),
|
|
682
|
+
}
|
|
683
|
+
|
|
567
684
|
struct Inner {
|
|
568
685
|
id: String,
|
|
569
686
|
generation: u64,
|
|
@@ -579,9 +696,12 @@ struct Inner {
|
|
|
579
696
|
/// offer has to name the profile the stream's own SPS declares.
|
|
580
697
|
peer: Mutex<Option<Arc<VideoPeer>>>,
|
|
581
698
|
/// Absent for an encoded source: the consumer is the encoder.
|
|
582
|
-
encoder: Mutex<Option<
|
|
699
|
+
encoder: Mutex<Option<VideoEncoder>>,
|
|
583
700
|
input: Mutex<Option<InputTarget>>,
|
|
584
701
|
indicator: Mutex<Option<crate::indicator::Indicator>>,
|
|
702
|
+
point_overlay: Mutex<Option<crate::indicator::Indicator>>,
|
|
703
|
+
#[cfg(target_os = "linux")]
|
|
704
|
+
point_desktop: PointDesktop,
|
|
585
705
|
capture: Mutex<Option<FrameSource>>,
|
|
586
706
|
/// Absent for an encoded source: there is no local keyboard to translate a
|
|
587
707
|
/// character through, because the client's keys are forwarded instead.
|
|
@@ -595,6 +715,9 @@ struct Inner {
|
|
|
595
715
|
events: tokio_mpsc::UnboundedSender<Notice>,
|
|
596
716
|
/// Present only for an encoded source.
|
|
597
717
|
encoded: Mutex<Option<Encoded>>,
|
|
718
|
+
/// `OpenRequest::local_frames`: whether `latest` is kept at all.
|
|
719
|
+
local_frames: bool,
|
|
720
|
+
cursor_info: serde_json::Value,
|
|
598
721
|
}
|
|
599
722
|
|
|
600
723
|
/// A source whose video arrives already encoded.
|
|
@@ -673,9 +796,11 @@ pub fn capabilities() -> serde_json::Value {
|
|
|
673
796
|
"backends": ["portal-screencast+pipewire", "x11-root"],
|
|
674
797
|
"formats": ["bgrx", "bgra", "rgbx", "rgba"],
|
|
675
798
|
"cursor": "embedded",
|
|
799
|
+
"cursor_modes": ["embedded", "hidden"],
|
|
800
|
+
"cursor_positions": {"x11": true, "portal": "negotiated"},
|
|
676
801
|
"audio": false,
|
|
677
802
|
},
|
|
678
|
-
"encode":
|
|
803
|
+
"encode": encode_capabilities(),
|
|
679
804
|
"input": {
|
|
680
805
|
"mechanism": "inputtino/uinput",
|
|
681
806
|
"pointer": input,
|
|
@@ -725,6 +850,7 @@ pub fn capabilities() -> serde_json::Value {
|
|
|
725
850
|
"backends": ["screencapturekit"],
|
|
726
851
|
"formats": ["bgra"],
|
|
727
852
|
"cursor": "embedded",
|
|
853
|
+
"cursor_modes": ["embedded"],
|
|
728
854
|
"audio": false,
|
|
729
855
|
"displays": crate::mac::displays(),
|
|
730
856
|
"grant": if capture_granted { "granted" } else { "missing-screen-recording" },
|
|
@@ -733,7 +859,7 @@ pub fn capabilities() -> serde_json::Value {
|
|
|
733
859
|
"remedy": format!("Allow {} in System Settings › Privacy & Security › Screen & System Audio Recording, then reconnect.", crate::mac::tcc_responsible_app_name())
|
|
734
860
|
}) }
|
|
735
861
|
},
|
|
736
|
-
"encode":
|
|
862
|
+
"encode": encode_capabilities(),
|
|
737
863
|
"input": {
|
|
738
864
|
"mechanism": "quartz-cgevent",
|
|
739
865
|
"pointer": input,
|
|
@@ -750,6 +876,34 @@ pub fn capabilities() -> serde_json::Value {
|
|
|
750
876
|
})
|
|
751
877
|
}
|
|
752
878
|
|
|
879
|
+
#[cfg(target_os = "windows")]
|
|
880
|
+
pub fn capabilities() -> serde_json::Value {
|
|
881
|
+
crate::win::capabilities()
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
#[cfg(target_os = "windows")]
|
|
885
|
+
fn wayland_clipboard_available() -> bool {
|
|
886
|
+
false
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
/// `capabilities.encode`: VP9 always, H.264 when an encoder for it starts
|
|
890
|
+
/// here now. `hardware` describes VP9, the default.
|
|
891
|
+
fn encode_capabilities() -> serde_json::Value {
|
|
892
|
+
let h264 = crate::h264::probe();
|
|
893
|
+
let mut codecs = vec!["vp9"];
|
|
894
|
+
if h264.is_some() {
|
|
895
|
+
codecs.push("h264");
|
|
896
|
+
}
|
|
897
|
+
serde_json::json!({
|
|
898
|
+
"codecs": codecs,
|
|
899
|
+
"hardware": false,
|
|
900
|
+
"h264": {
|
|
901
|
+
"encoder": h264.map(|info| info.encoder),
|
|
902
|
+
"hardware": h264.is_some_and(|info| info.hardware),
|
|
903
|
+
},
|
|
904
|
+
})
|
|
905
|
+
}
|
|
906
|
+
|
|
753
907
|
impl Session {
|
|
754
908
|
pub async fn open(
|
|
755
909
|
request: OpenRequest,
|
|
@@ -761,6 +915,26 @@ impl Session {
|
|
|
761
915
|
"session.open requires the view permission",
|
|
762
916
|
));
|
|
763
917
|
}
|
|
918
|
+
#[cfg(target_os = "windows")]
|
|
919
|
+
if request.agent_indicator {
|
|
920
|
+
return Err(SessionError::new(
|
|
921
|
+
"indicator-unavailable",
|
|
922
|
+
"Windows agent indicator is not implemented in this build.",
|
|
923
|
+
));
|
|
924
|
+
}
|
|
925
|
+
if request.cursor == crate::protocol::CursorMode::Hidden {
|
|
926
|
+
#[cfg(not(target_os = "linux"))]
|
|
927
|
+
return Err(SessionError::new(
|
|
928
|
+
"cursor-unavailable",
|
|
929
|
+
"hidden cursor capture is not implemented on this platform",
|
|
930
|
+
));
|
|
931
|
+
if matches!(request.source, Some(SourceRequest::Encoded { .. })) {
|
|
932
|
+
return Err(SessionError::new(
|
|
933
|
+
"cursor-unavailable",
|
|
934
|
+
"the engine cannot remove cursor pixels from consumer-encoded video",
|
|
935
|
+
));
|
|
936
|
+
}
|
|
937
|
+
}
|
|
764
938
|
let id = opaque_id();
|
|
765
939
|
let wants_control = request.permissions.contains(&Permission::Control);
|
|
766
940
|
let wants_x11 = matches!(request.source, Some(SourceRequest::X11 { .. }));
|
|
@@ -832,9 +1006,18 @@ impl Session {
|
|
|
832
1006
|
let capture_status = Arc::new(on_capture_status);
|
|
833
1007
|
let frame_events = events.clone();
|
|
834
1008
|
let frame_session_id = id.clone();
|
|
835
|
-
let
|
|
836
|
-
|
|
837
|
-
|
|
1009
|
+
let local_frames = request.local_frames;
|
|
1010
|
+
let sink = Box::new(move |frame: I420, _seq: u64, pixels: capture::Pixels| {
|
|
1011
|
+
// The frame's RTP time: when capture handed it over, not when it
|
|
1012
|
+
// was coded, so the receiver never sees encode jitter as cadence.
|
|
1013
|
+
// ponytail: stamped after the backend's grab and conversion; a
|
|
1014
|
+
// grab-start instant from each backend needs a FrameSink change.
|
|
1015
|
+
let captured_at = Instant::now();
|
|
1016
|
+
let raw = if local_frames { pixels() } else { None };
|
|
1017
|
+
let hashes = raw
|
|
1018
|
+
.as_ref()
|
|
1019
|
+
.map(|raw| tile_hashes(raw, frame.width, frame.height));
|
|
1020
|
+
if let (Some(raw), Some(hashes), Ok(mut held)) = (raw, hashes, observed.lock()) {
|
|
838
1021
|
let unchanged =
|
|
839
1022
|
held.as_ref()
|
|
840
1023
|
.is_some_and(|(_, _, width, height, _, current, _)| {
|
|
@@ -849,7 +1032,7 @@ impl Session {
|
|
|
849
1032
|
let damage = dirty_regions(&hashes, &previous, frame.width, frame.height);
|
|
850
1033
|
*held = Some((
|
|
851
1034
|
seq,
|
|
852
|
-
|
|
1035
|
+
captured_at,
|
|
853
1036
|
frame.width,
|
|
854
1037
|
frame.height,
|
|
855
1038
|
raw,
|
|
@@ -859,22 +1042,42 @@ impl Session {
|
|
|
859
1042
|
frame_tx_signal.send_replace(seq);
|
|
860
1043
|
let _ = frame_events.send(Notice {
|
|
861
1044
|
session_id: frame_session_id.clone(),
|
|
862
|
-
event: SessionEvent::Frame {
|
|
1045
|
+
event: SessionEvent::Frame {
|
|
1046
|
+
seq,
|
|
1047
|
+
damage,
|
|
1048
|
+
timestamp_us: crate::protocol::timestamp_us(captured_at),
|
|
1049
|
+
},
|
|
863
1050
|
});
|
|
864
1051
|
}
|
|
865
1052
|
}
|
|
866
1053
|
if let Ok(mut m) = captured.lock() {
|
|
867
1054
|
m.captured_frames += 1;
|
|
1055
|
+
m.convert_micros += captured_at.elapsed().as_micros() as u64;
|
|
868
1056
|
}
|
|
869
1057
|
// An encoder that is behind gets the newest frame, not a queue: a
|
|
870
1058
|
// desktop stream is live, and the last state is the one that counts.
|
|
871
|
-
if frame_tx.put(frame) {
|
|
1059
|
+
if frame_tx.put(frame, captured_at) {
|
|
872
1060
|
if let Ok(mut m) = captured.lock() {
|
|
873
1061
|
m.dropped_frames += 1;
|
|
874
1062
|
}
|
|
875
1063
|
}
|
|
876
1064
|
});
|
|
877
1065
|
|
|
1066
|
+
#[cfg(target_os = "linux")]
|
|
1067
|
+
let cursor_sink: Option<crate::cursor::CursorSink> =
|
|
1068
|
+
if request.cursor == crate::protocol::CursorMode::Hidden {
|
|
1069
|
+
let events = events.clone();
|
|
1070
|
+
let id = id.clone();
|
|
1071
|
+
Some(Box::new(move |position| {
|
|
1072
|
+
let _ = events.send(Notice {
|
|
1073
|
+
session_id: id.clone(),
|
|
1074
|
+
event: SessionEvent::Cursor { position },
|
|
1075
|
+
});
|
|
1076
|
+
}))
|
|
1077
|
+
} else {
|
|
1078
|
+
None
|
|
1079
|
+
};
|
|
1080
|
+
let mut cursor_info = serde_json::json!({"mode":"embedded", "positions":false});
|
|
878
1081
|
let Selected {
|
|
879
1082
|
source,
|
|
880
1083
|
capture,
|
|
@@ -886,6 +1089,8 @@ impl Session {
|
|
|
886
1089
|
request.max_height,
|
|
887
1090
|
max_fps,
|
|
888
1091
|
metrics.clone(),
|
|
1092
|
+
#[cfg(target_os = "linux")]
|
|
1093
|
+
cursor_sink,
|
|
889
1094
|
sink,
|
|
890
1095
|
Box::new({
|
|
891
1096
|
let status = capture_status.clone();
|
|
@@ -898,9 +1103,30 @@ impl Session {
|
|
|
898
1103
|
Some(SourceRequest::Display { display_id }) => {
|
|
899
1104
|
portal::open_display(*display_id).await
|
|
900
1105
|
}
|
|
901
|
-
_ =>
|
|
1106
|
+
_ => {
|
|
1107
|
+
#[cfg(target_os = "linux")]
|
|
1108
|
+
{
|
|
1109
|
+
portal::open(request.restore_token.as_deref(), request.cursor).await
|
|
1110
|
+
}
|
|
1111
|
+
#[cfg(not(target_os = "linux"))]
|
|
1112
|
+
{
|
|
1113
|
+
portal::open(request.restore_token.as_deref()).await
|
|
1114
|
+
}
|
|
1115
|
+
}
|
|
1116
|
+
}
|
|
1117
|
+
.map_err(|error| {
|
|
1118
|
+
let message = format!("{error:#}");
|
|
1119
|
+
let code = if message.contains("cursor_positions_unavailable:") {
|
|
1120
|
+
"cursor_positions_unavailable"
|
|
1121
|
+
} else {
|
|
1122
|
+
"source"
|
|
1123
|
+
};
|
|
1124
|
+
SessionError::new(code, message)
|
|
1125
|
+
})?;
|
|
1126
|
+
#[cfg(target_os = "linux")]
|
|
1127
|
+
{
|
|
1128
|
+
cursor_info = serde_json::json!({"mode":portal.cursor_mode, "positions":false});
|
|
902
1129
|
}
|
|
903
|
-
.map_err(|error| SessionError::new("source", format!("{error:#}")))?;
|
|
904
1130
|
if let Some(token) = &portal.restore_token {
|
|
905
1131
|
let _ = events.send(Notice {
|
|
906
1132
|
session_id: id.clone(),
|
|
@@ -945,12 +1171,26 @@ impl Session {
|
|
|
945
1171
|
}),
|
|
946
1172
|
)
|
|
947
1173
|
.map_err(|error| SessionError::new("source", format!("{error:#}")))?;
|
|
1174
|
+
#[cfg(target_os = "windows")]
|
|
1175
|
+
let capture = capture::start(
|
|
1176
|
+
portal,
|
|
1177
|
+
width,
|
|
1178
|
+
height,
|
|
1179
|
+
max_fps,
|
|
1180
|
+
sink,
|
|
1181
|
+
Box::new({
|
|
1182
|
+
let status = capture_status.clone();
|
|
1183
|
+
move |running, reason| status(running, reason)
|
|
1184
|
+
}),
|
|
1185
|
+
)
|
|
1186
|
+
.map_err(|error| SessionError::new("source", format!("{error:#}")))?;
|
|
948
1187
|
#[cfg(target_os = "linux")]
|
|
949
1188
|
let capture = capture::start(
|
|
950
1189
|
portal,
|
|
951
1190
|
width,
|
|
952
1191
|
height,
|
|
953
1192
|
max_fps,
|
|
1193
|
+
cursor_sink,
|
|
954
1194
|
sink,
|
|
955
1195
|
indicator
|
|
956
1196
|
.as_ref()
|
|
@@ -962,6 +1202,18 @@ impl Session {
|
|
|
962
1202
|
)
|
|
963
1203
|
.map_err(|error| SessionError::new("source", format!("{error:#}")))?;
|
|
964
1204
|
#[cfg(target_os = "linux")]
|
|
1205
|
+
{
|
|
1206
|
+
if request.cursor == crate::protocol::CursorMode::Hidden
|
|
1207
|
+
&& !capture.cursor_positions()
|
|
1208
|
+
{
|
|
1209
|
+
return Err(SessionError::new(
|
|
1210
|
+
"cursor_positions_unavailable",
|
|
1211
|
+
"portal stream did not provide SPA cursor metadata",
|
|
1212
|
+
));
|
|
1213
|
+
}
|
|
1214
|
+
cursor_info["positions"] = serde_json::json!(capture.cursor_positions());
|
|
1215
|
+
}
|
|
1216
|
+
#[cfg(target_os = "linux")]
|
|
965
1217
|
if let Some(indicator) = indicator.as_mut() {
|
|
966
1218
|
indicator.event('A', -1, -1);
|
|
967
1219
|
}
|
|
@@ -972,6 +1224,9 @@ impl Session {
|
|
|
972
1224
|
}
|
|
973
1225
|
}
|
|
974
1226
|
};
|
|
1227
|
+
if wants_x11 {
|
|
1228
|
+
cursor_info = serde_json::json!({"mode":"hidden", "positions": request.cursor == crate::protocol::CursorMode::Hidden});
|
|
1229
|
+
}
|
|
975
1230
|
|
|
976
1231
|
let source_w = source.width.max(1) as usize;
|
|
977
1232
|
let source_h = source.height.max(1) as usize;
|
|
@@ -1023,7 +1278,7 @@ impl Session {
|
|
|
1023
1278
|
applier: Applier::Uinput({
|
|
1024
1279
|
#[cfg(target_os = "linux")]
|
|
1025
1280
|
let devices = InputDevices::create(source_w as i32, source_h as i32);
|
|
1026
|
-
#[cfg(target_os = "macos")]
|
|
1281
|
+
#[cfg(any(target_os = "macos", target_os = "windows"))]
|
|
1027
1282
|
let devices = InputDevices::create_for_display(
|
|
1028
1283
|
source_w as i32,
|
|
1029
1284
|
source_h as i32,
|
|
@@ -1046,6 +1301,16 @@ impl Session {
|
|
|
1046
1301
|
SessionError::new("input", format!("no keyboard layout: {error:#}"))
|
|
1047
1302
|
})?;
|
|
1048
1303
|
|
|
1304
|
+
// H.264 is offered beside VP9 only when an encoder for it starts on
|
|
1305
|
+
// this machine; the answer then decides which one is coded.
|
|
1306
|
+
let codec = if !crate::h264::fits(width, height) {
|
|
1307
|
+
VideoCodec::Vp9
|
|
1308
|
+
} else {
|
|
1309
|
+
match tokio::task::spawn_blocking(crate::h264::prepare).await {
|
|
1310
|
+
Ok(Some(_)) => VideoCodec::Vp9OrH264,
|
|
1311
|
+
_ => VideoCodec::Vp9,
|
|
1312
|
+
}
|
|
1313
|
+
};
|
|
1049
1314
|
let (peer_events_tx, peer_events_rx) = tokio_mpsc::unbounded_channel::<PeerEvent>();
|
|
1050
1315
|
let (peer, offer) = VideoPeer::offer(
|
|
1051
1316
|
TransportOptions {
|
|
@@ -1056,7 +1321,7 @@ impl Session {
|
|
|
1056
1321
|
pace_bps: (bitrate_kbps as f64 * 3_000.0).max(20_000_000.0),
|
|
1057
1322
|
},
|
|
1058
1323
|
peer_events_tx,
|
|
1059
|
-
|
|
1324
|
+
codec,
|
|
1060
1325
|
)
|
|
1061
1326
|
.await
|
|
1062
1327
|
.map_err(|error| SessionError::new("transport", format!("{error:#}")))?;
|
|
@@ -1081,9 +1346,23 @@ impl Session {
|
|
|
1081
1346
|
capture_error,
|
|
1082
1347
|
frame_changes,
|
|
1083
1348
|
peer: Mutex::new(Some(peer.clone())),
|
|
1084
|
-
encoder: Mutex::new(Some(encoder)),
|
|
1349
|
+
encoder: Mutex::new(Some(VideoEncoder::Vp9(encoder))),
|
|
1085
1350
|
input: Mutex::new(input),
|
|
1086
1351
|
indicator: Mutex::new(indicator),
|
|
1352
|
+
point_overlay: Mutex::new(None),
|
|
1353
|
+
#[cfg(target_os = "linux")]
|
|
1354
|
+
point_desktop: if wants_x11 {
|
|
1355
|
+
match &request.source {
|
|
1356
|
+
Some(SourceRequest::X11 { display }) => display
|
|
1357
|
+
.clone()
|
|
1358
|
+
.or_else(|| std::env::var("DISPLAY").ok())
|
|
1359
|
+
.map(PointDesktop::X11)
|
|
1360
|
+
.unwrap_or(PointDesktop::Unavailable("x11-unavailable")),
|
|
1361
|
+
_ => PointDesktop::Unavailable("x11-unavailable"),
|
|
1362
|
+
}
|
|
1363
|
+
} else {
|
|
1364
|
+
PointDesktop::Wayland
|
|
1365
|
+
},
|
|
1087
1366
|
capture: Mutex::new(Some(capture)),
|
|
1088
1367
|
layout: Mutex::new(Some(layout)),
|
|
1089
1368
|
last_seq: Mutex::new(0),
|
|
@@ -1092,6 +1371,8 @@ impl Session {
|
|
|
1092
1371
|
pipeline: pipeline.clone(),
|
|
1093
1372
|
events: events.clone(),
|
|
1094
1373
|
encoded: Mutex::new(None),
|
|
1374
|
+
local_frames: request.local_frames,
|
|
1375
|
+
cursor_info,
|
|
1095
1376
|
});
|
|
1096
1377
|
|
|
1097
1378
|
inner.notify(SessionEvent::Description {
|
|
@@ -1170,6 +1451,9 @@ impl Session {
|
|
|
1170
1451
|
encoder: Mutex::new(None),
|
|
1171
1452
|
input: Mutex::new(None),
|
|
1172
1453
|
indicator: Mutex::new(None),
|
|
1454
|
+
point_overlay: Mutex::new(None),
|
|
1455
|
+
#[cfg(target_os = "linux")]
|
|
1456
|
+
point_desktop: PointDesktop::Unavailable("layer_shell_unavailable"),
|
|
1173
1457
|
capture: Mutex::new(None),
|
|
1174
1458
|
layout: Mutex::new(None),
|
|
1175
1459
|
last_seq: Mutex::new(0),
|
|
@@ -1189,6 +1473,8 @@ impl Session {
|
|
|
1189
1473
|
peer_events,
|
|
1190
1474
|
feed,
|
|
1191
1475
|
})),
|
|
1476
|
+
local_frames: false,
|
|
1477
|
+
cursor_info: serde_json::json!({"mode":"unavailable", "positions":false}),
|
|
1192
1478
|
});
|
|
1193
1479
|
inner.notify(SessionEvent::State {
|
|
1194
1480
|
capture: "consented",
|
|
@@ -1246,6 +1532,10 @@ impl Session {
|
|
|
1246
1532
|
self.inner.generation
|
|
1247
1533
|
}
|
|
1248
1534
|
|
|
1535
|
+
pub fn cursor_info(&self) -> &serde_json::Value {
|
|
1536
|
+
&self.inner.cursor_info
|
|
1537
|
+
}
|
|
1538
|
+
|
|
1249
1539
|
pub fn geometry(&self) -> &serde_json::Value {
|
|
1250
1540
|
&self.inner.geometry
|
|
1251
1541
|
}
|
|
@@ -1254,6 +1544,128 @@ impl Session {
|
|
|
1254
1544
|
&self.inner.source
|
|
1255
1545
|
}
|
|
1256
1546
|
|
|
1547
|
+
pub fn point(
|
|
1548
|
+
&self,
|
|
1549
|
+
params: &crate::protocol::PointParams,
|
|
1550
|
+
) -> std::result::Result<(), SessionError> {
|
|
1551
|
+
// Lock through validation and drawing, so a concurrent close cannot leave
|
|
1552
|
+
// a newly spawned marker alive after teardown.
|
|
1553
|
+
let mut overlay = lock(&self.inner.point_overlay);
|
|
1554
|
+
if self.inner.closed.load(Ordering::SeqCst) {
|
|
1555
|
+
return Err(SessionError::new("session", "the session is closed"));
|
|
1556
|
+
}
|
|
1557
|
+
if !self.inner.permissions.contains(&Permission::View) {
|
|
1558
|
+
return Err(SessionError::new(
|
|
1559
|
+
"permission",
|
|
1560
|
+
"point requires view permission",
|
|
1561
|
+
));
|
|
1562
|
+
}
|
|
1563
|
+
if lock(&self.inner.encoded).is_some() {
|
|
1564
|
+
return Err(SessionError::new(
|
|
1565
|
+
"operation",
|
|
1566
|
+
"an encoded source has no host desktop",
|
|
1567
|
+
));
|
|
1568
|
+
}
|
|
1569
|
+
#[cfg(target_os = "linux")]
|
|
1570
|
+
if let PointDesktop::Unavailable(reason) = &self.inner.point_desktop {
|
|
1571
|
+
return Err(SessionError::new(
|
|
1572
|
+
"point_unsupported",
|
|
1573
|
+
format!("reason: {reason}; point overlay is unavailable"),
|
|
1574
|
+
));
|
|
1575
|
+
}
|
|
1576
|
+
if params.clear {
|
|
1577
|
+
if params.x.is_some()
|
|
1578
|
+
|| params.y.is_some()
|
|
1579
|
+
|| params.timeout_ms.is_some()
|
|
1580
|
+
|| !params.label.is_empty()
|
|
1581
|
+
{
|
|
1582
|
+
return Err(SessionError::new(
|
|
1583
|
+
"malformed",
|
|
1584
|
+
"clear accepts no coordinates, label or timeout",
|
|
1585
|
+
));
|
|
1586
|
+
}
|
|
1587
|
+
#[cfg(target_os = "linux")]
|
|
1588
|
+
if overlay.is_none() && matches!(self.inner.point_desktop, PointDesktop::Wayland) {
|
|
1589
|
+
let available = crate::indicator::Indicator::wayland_layer_shell_available()
|
|
1590
|
+
.map_err(|error| {
|
|
1591
|
+
SessionError::new("indicator-unavailable", format!("{error:#}"))
|
|
1592
|
+
})?;
|
|
1593
|
+
if !available {
|
|
1594
|
+
return Err(SessionError::new(
|
|
1595
|
+
"point_unsupported",
|
|
1596
|
+
"reason: layer_shell_unavailable; compositor has no layer-shell",
|
|
1597
|
+
));
|
|
1598
|
+
}
|
|
1599
|
+
}
|
|
1600
|
+
if let Some(helper) = overlay.take() {
|
|
1601
|
+
helper.stop();
|
|
1602
|
+
}
|
|
1603
|
+
return Ok(());
|
|
1604
|
+
}
|
|
1605
|
+
let (x, y) = params
|
|
1606
|
+
.x
|
|
1607
|
+
.zip(params.y)
|
|
1608
|
+
.ok_or_else(|| SessionError::new("malformed", "point requires x and y"))?;
|
|
1609
|
+
let source = &self.inner.source;
|
|
1610
|
+
if x < 0 || y < 0 || x >= i64::from(source.width) || y >= i64::from(source.height) {
|
|
1611
|
+
return Err(SessionError::new(
|
|
1612
|
+
"coordinates",
|
|
1613
|
+
"point is outside the selected desktop",
|
|
1614
|
+
));
|
|
1615
|
+
}
|
|
1616
|
+
let timeout = params.timeout_ms.unwrap_or(3000);
|
|
1617
|
+
if !(1..=120_000).contains(&timeout)
|
|
1618
|
+
|| params.label.len() > 96
|
|
1619
|
+
|| !params.label.bytes().all(|b| (32..=126).contains(&b))
|
|
1620
|
+
{
|
|
1621
|
+
return Err(SessionError::new(
|
|
1622
|
+
"malformed",
|
|
1623
|
+
"timeout must be 1..120000 ms; label must be at most 96 printable ASCII bytes",
|
|
1624
|
+
));
|
|
1625
|
+
}
|
|
1626
|
+
#[cfg(not(target_os = "windows"))]
|
|
1627
|
+
if overlay.is_none() {
|
|
1628
|
+
#[cfg(target_os = "linux")]
|
|
1629
|
+
let helper = match &self.inner.point_desktop {
|
|
1630
|
+
PointDesktop::X11(display) => crate::indicator::Indicator::start_point(display),
|
|
1631
|
+
PointDesktop::Wayland => crate::indicator::Indicator::start_point_wayland(
|
|
1632
|
+
source.width,
|
|
1633
|
+
source.height,
|
|
1634
|
+
source.position,
|
|
1635
|
+
),
|
|
1636
|
+
PointDesktop::Unavailable(_) => unreachable!("validated point target"),
|
|
1637
|
+
};
|
|
1638
|
+
#[cfg(target_os = "macos")]
|
|
1639
|
+
let helper = crate::indicator::Indicator::start_point(
|
|
1640
|
+
source.node_id,
|
|
1641
|
+
source.width as usize,
|
|
1642
|
+
source.height as usize,
|
|
1643
|
+
);
|
|
1644
|
+
*overlay = Some(helper.map_err(|error| {
|
|
1645
|
+
let message = format!("{error:#}");
|
|
1646
|
+
if message.contains("layer_shell_unavailable") {
|
|
1647
|
+
SessionError::new(
|
|
1648
|
+
"point_unsupported",
|
|
1649
|
+
"reason: layer_shell_unavailable; compositor has no layer-shell",
|
|
1650
|
+
)
|
|
1651
|
+
} else {
|
|
1652
|
+
SessionError::new("indicator-unavailable", message)
|
|
1653
|
+
}
|
|
1654
|
+
})?);
|
|
1655
|
+
}
|
|
1656
|
+
#[cfg(not(target_os = "windows"))]
|
|
1657
|
+
return overlay
|
|
1658
|
+
.as_mut()
|
|
1659
|
+
.unwrap()
|
|
1660
|
+
.point(x, y, timeout, ¶ms.label)
|
|
1661
|
+
.map_err(|error| SessionError::new("indicator-unavailable", format!("{error:#}")));
|
|
1662
|
+
#[cfg(target_os = "windows")]
|
|
1663
|
+
return Err(SessionError::new(
|
|
1664
|
+
"point_unsupported",
|
|
1665
|
+
"Windows point overlays are unavailable in this build.",
|
|
1666
|
+
));
|
|
1667
|
+
}
|
|
1668
|
+
|
|
1257
1669
|
pub async fn wait_frame(
|
|
1258
1670
|
&self,
|
|
1259
1671
|
after_seq: Option<u64>,
|
|
@@ -1266,6 +1678,7 @@ impl Session {
|
|
|
1266
1678
|
"this session forwards an encoded stream and keeps no frame",
|
|
1267
1679
|
));
|
|
1268
1680
|
}
|
|
1681
|
+
self.keeps_frames()?;
|
|
1269
1682
|
let mut changes = self.inner.frame_changes.clone();
|
|
1270
1683
|
let deadline = tokio::time::Instant::now() + Duration::from_millis(timeout_ms.min(120_000));
|
|
1271
1684
|
loop {
|
|
@@ -1302,6 +1715,16 @@ impl Session {
|
|
|
1302
1715
|
}
|
|
1303
1716
|
}
|
|
1304
1717
|
|
|
1718
|
+
fn keeps_frames(&self) -> std::result::Result<(), SessionError> {
|
|
1719
|
+
if self.inner.local_frames {
|
|
1720
|
+
return Ok(());
|
|
1721
|
+
}
|
|
1722
|
+
Err(SessionError::new(
|
|
1723
|
+
"operation",
|
|
1724
|
+
"this session was opened with local_frames off and keeps no frame",
|
|
1725
|
+
))
|
|
1726
|
+
}
|
|
1727
|
+
|
|
1305
1728
|
/// Write only the requested lossless frame bytes; JSON carries metadata, never pixels.
|
|
1306
1729
|
pub fn frame(
|
|
1307
1730
|
&self,
|
|
@@ -1321,6 +1744,7 @@ impl Session {
|
|
|
1321
1744
|
"this session forwards an encoded stream and keeps no frame",
|
|
1322
1745
|
));
|
|
1323
1746
|
}
|
|
1747
|
+
self.keeps_frames()?;
|
|
1324
1748
|
if let Some(reason) = lock(&self.inner.capture_error).clone() {
|
|
1325
1749
|
return Err(SessionError::new("stream-stopped", reason));
|
|
1326
1750
|
}
|
|
@@ -1356,7 +1780,7 @@ impl Session {
|
|
|
1356
1780
|
}
|
|
1357
1781
|
Some(pixels)
|
|
1358
1782
|
};
|
|
1359
|
-
let result = serde_json::json!({ "seq": seq, "still_ms": at.elapsed().as_millis() as u64,
|
|
1783
|
+
let result = serde_json::json!({ "seq": seq, "timestamp_us": crate::protocol::timestamp_us(*at), "still_ms": at.elapsed().as_millis() as u64,
|
|
1360
1784
|
"width": w, "height": h, "format": "bgrx", "damage": damage, "written": !path.is_empty() });
|
|
1361
1785
|
drop(held);
|
|
1362
1786
|
if let Some(pixels) = pixels {
|
|
@@ -1449,7 +1873,7 @@ impl Session {
|
|
|
1449
1873
|
}
|
|
1450
1874
|
|
|
1451
1875
|
pub async fn close(&self, reason: &str) {
|
|
1452
|
-
self.inner.close(reason).await;
|
|
1876
|
+
self.inner.close(reason, "closed").await;
|
|
1453
1877
|
}
|
|
1454
1878
|
}
|
|
1455
1879
|
|
|
@@ -1464,16 +1888,24 @@ impl Inner {
|
|
|
1464
1888
|
|
|
1465
1889
|
/// Stop everything and release whatever input this session held. Idempotent,
|
|
1466
1890
|
/// and the one teardown both an explicit close and a lease expiry take.
|
|
1467
|
-
async fn close(&self, reason: &str) {
|
|
1891
|
+
async fn close(&self, reason: &str, code: &str) {
|
|
1468
1892
|
if self.closed.swap(true, Ordering::SeqCst) {
|
|
1469
1893
|
return;
|
|
1470
1894
|
}
|
|
1471
1895
|
self.control_open.store(false, Ordering::SeqCst);
|
|
1472
1896
|
self.pipeline.store(false, Ordering::SeqCst);
|
|
1473
1897
|
drop(lock(&self.encoded).take());
|
|
1898
|
+
// The encoder goes now, not with the last handle: a hardware session
|
|
1899
|
+
// still open when the process exits can hang the driver's exit
|
|
1900
|
+
// handlers, and `shutdown` exits right after this. The encode thread
|
|
1901
|
+
// finds it gone and stops.
|
|
1902
|
+
drop(lock(&self.encoder).take());
|
|
1474
1903
|
if let Ok(mut indicator) = self.indicator.lock() {
|
|
1475
1904
|
indicator.take();
|
|
1476
1905
|
}
|
|
1906
|
+
if let Some(helper) = lock(&self.point_overlay).take() {
|
|
1907
|
+
helper.stop();
|
|
1908
|
+
}
|
|
1477
1909
|
// Release first, then tear down: a stuck modifier is the one failure the
|
|
1478
1910
|
// user cannot undo by reconnecting.
|
|
1479
1911
|
if let Ok(mut input) = self.input.lock() {
|
|
@@ -1485,8 +1917,8 @@ impl Inner {
|
|
|
1485
1917
|
if let Some(peer) = self.peer.lock().ok().and_then(|peer| peer.clone()) {
|
|
1486
1918
|
let _ = peer
|
|
1487
1919
|
.send_control(
|
|
1488
|
-
&serde_json::to_string(&ControlReply::Revoked { reason }).unwrap_or_else(
|
|
1489
|
-
|_| String::from(r#"{"kind":"revoked","reason":"closed"}"#),
|
|
1920
|
+
&serde_json::to_string(&ControlReply::Revoked { reason, code }).unwrap_or_else(
|
|
1921
|
+
|_| String::from(r#"{"kind":"revoked","reason":"closed","code":"closed"}"#),
|
|
1490
1922
|
),
|
|
1491
1923
|
)
|
|
1492
1924
|
.await;
|
|
@@ -1820,7 +2252,7 @@ impl Inner {
|
|
|
1820
2252
|
if text.len() > 4096 {
|
|
1821
2253
|
return Err(("text-too-large", String::from("text exceeds 4096 bytes")));
|
|
1822
2254
|
}
|
|
1823
|
-
#[cfg(target_os = "macos")]
|
|
2255
|
+
#[cfg(any(target_os = "macos", target_os = "windows"))]
|
|
1824
2256
|
{
|
|
1825
2257
|
let _ = seq;
|
|
1826
2258
|
return self.with_input(|target| target.unicode_text(text));
|
|
@@ -1937,11 +2369,13 @@ fn frame_interval(max_fps: u32) -> Duration {
|
|
|
1937
2369
|
Duration::from_secs_f64(1.0 / max_fps.max(1) as f64)
|
|
1938
2370
|
}
|
|
1939
2371
|
|
|
1940
|
-
/// The newest captured frame, handed from the capture
|
|
1941
|
-
/// newer frame replaces one the encoder has not taken,
|
|
1942
|
-
/// it falls, what it codes next is the desktop as it is
|
|
2372
|
+
/// The newest captured frame and when it was captured, handed from the capture
|
|
2373
|
+
/// thread to the encoder. A newer frame replaces one the encoder has not taken,
|
|
2374
|
+
/// so however far behind it falls, what it codes next is the desktop as it is
|
|
2375
|
+
/// now. The instant travels with the picture so the pipeline can report how
|
|
2376
|
+
/// long a coded frame waited for its encode.
|
|
1943
2377
|
struct FrameSlot {
|
|
1944
|
-
frame: Mutex<(Option<I420>, bool)>,
|
|
2378
|
+
frame: Mutex<(Option<(I420, Instant)>, bool)>,
|
|
1945
2379
|
ready: Condvar,
|
|
1946
2380
|
}
|
|
1947
2381
|
|
|
@@ -1958,13 +2392,13 @@ fn latest_frame() -> (FrameSender, FrameReceiver) {
|
|
|
1958
2392
|
|
|
1959
2393
|
impl FrameSender {
|
|
1960
2394
|
/// Hand over a frame; true when it replaced one the encoder never took.
|
|
1961
|
-
fn put(&self, frame: I420) -> bool {
|
|
2395
|
+
fn put(&self, frame: I420, captured: Instant) -> bool {
|
|
1962
2396
|
let mut held = self
|
|
1963
2397
|
.0
|
|
1964
2398
|
.frame
|
|
1965
2399
|
.lock()
|
|
1966
2400
|
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
|
1967
|
-
let replaced = held.0.replace(frame).is_some();
|
|
2401
|
+
let replaced = held.0.replace((frame, captured)).is_some();
|
|
1968
2402
|
self.0.ready.notify_one();
|
|
1969
2403
|
replaced
|
|
1970
2404
|
}
|
|
@@ -1984,12 +2418,22 @@ impl Drop for FrameSender {
|
|
|
1984
2418
|
}
|
|
1985
2419
|
|
|
1986
2420
|
enum Taken {
|
|
1987
|
-
Frame(I420),
|
|
2421
|
+
Frame(I420, Instant),
|
|
1988
2422
|
Timeout,
|
|
1989
2423
|
Ended,
|
|
1990
2424
|
}
|
|
1991
2425
|
|
|
1992
2426
|
impl FrameReceiver {
|
|
2427
|
+
/// Whether a frame is waiting to be taken.
|
|
2428
|
+
fn waiting(&self) -> bool {
|
|
2429
|
+
self.0
|
|
2430
|
+
.frame
|
|
2431
|
+
.lock()
|
|
2432
|
+
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
|
2433
|
+
.0
|
|
2434
|
+
.is_some()
|
|
2435
|
+
}
|
|
2436
|
+
|
|
1993
2437
|
fn take(&self, wait: Duration) -> Taken {
|
|
1994
2438
|
let held = self
|
|
1995
2439
|
.0
|
|
@@ -2002,7 +2446,7 @@ impl FrameReceiver {
|
|
|
2002
2446
|
.wait_timeout_while(held, wait, |(frame, ended)| frame.is_none() && !*ended)
|
|
2003
2447
|
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
|
2004
2448
|
match held.0.take() {
|
|
2005
|
-
Some(frame) => Taken::Frame(frame),
|
|
2449
|
+
Some((frame, captured)) => Taken::Frame(frame, captured),
|
|
2006
2450
|
None if held.1 => Taken::Ended,
|
|
2007
2451
|
None => Taken::Timeout,
|
|
2008
2452
|
}
|
|
@@ -2028,12 +2472,32 @@ fn auto_bitrate(width: usize, height: usize) -> u32 {
|
|
|
2028
2472
|
(1500 + (width * height * 18 / 10_000) as u32).min(25_000)
|
|
2029
2473
|
}
|
|
2030
2474
|
|
|
2031
|
-
///
|
|
2032
|
-
///
|
|
2475
|
+
/// Queueing delay that means the link carries more than it drains: well under
|
|
2476
|
+
/// the latency budget, well over the noise of a busy host's scheduling.
|
|
2477
|
+
const QUEUE_HIGH: Duration = Duration::from_millis(25);
|
|
2478
|
+
/// Below this the path has room again and the target may creep back up.
|
|
2479
|
+
const QUEUE_LOW: Duration = Duration::from_millis(10);
|
|
2480
|
+
/// How often queueing delay may back the target off: long enough for the last
|
|
2481
|
+
/// back-off to reach the far end and show in its feedback, short enough that a
|
|
2482
|
+
/// queue is drained in a few steps rather than a few seconds.
|
|
2483
|
+
const DELAY_BACKOFF_EVERY: Duration = Duration::from_millis(300);
|
|
2484
|
+
|
|
2485
|
+
/// The rate target: back off when the far end reports a standing queue or
|
|
2486
|
+
/// loss, creep back when it reports neither, never above what the session
|
|
2487
|
+
/// asked for.
|
|
2488
|
+
///
|
|
2489
|
+
/// Delay leads. A link with a deep buffer (most Wi-Fi and cellular ones) never
|
|
2490
|
+
/// loses a packet it can queue, so loss alone lets queueing delay grow to
|
|
2491
|
+
/// seconds before anything reacts. A standing queue backs the target off to
|
|
2492
|
+
/// 85% of what the path delivered, which drains it; never more than halving
|
|
2493
|
+
/// per step, so a stream that was idle and sent little is not taken for a
|
|
2494
|
+
/// slow link. Receivers without transport-wide feedback leave it loss-only.
|
|
2033
2495
|
struct RateControl {
|
|
2034
2496
|
ceiling: u32,
|
|
2035
2497
|
target: u32,
|
|
2036
2498
|
last_change: Option<Instant>,
|
|
2499
|
+
/// The last queueing delay reported, while the path reports one.
|
|
2500
|
+
queue: Option<Duration>,
|
|
2037
2501
|
}
|
|
2038
2502
|
|
|
2039
2503
|
impl RateControl {
|
|
@@ -2042,26 +2506,40 @@ impl RateControl {
|
|
|
2042
2506
|
ceiling,
|
|
2043
2507
|
target: ceiling,
|
|
2044
2508
|
last_change: None,
|
|
2509
|
+
queue: None,
|
|
2045
2510
|
}
|
|
2046
2511
|
}
|
|
2047
2512
|
|
|
2048
|
-
/// The new target when these reports
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
.
|
|
2054
|
-
{
|
|
2055
|
-
return None;
|
|
2513
|
+
/// The new target when these loss reports (RFC 3550 fractions, /256) or
|
|
2514
|
+
/// this path report move it: down on a standing queue at most every
|
|
2515
|
+
/// `DELAY_BACKOFF_EVERY`, otherwise at most once a second.
|
|
2516
|
+
fn update(&mut self, reports: &[u8], path: Option<PathReport>, now: Instant) -> Option<u32> {
|
|
2517
|
+
if let Some(path) = path {
|
|
2518
|
+
self.queue = Some(path.queue_delay);
|
|
2056
2519
|
}
|
|
2057
|
-
let
|
|
2520
|
+
let since = self.last_change.map(|at| now.duration_since(at));
|
|
2521
|
+
let waited = |gap: Duration| since.is_none_or(|since| since >= gap);
|
|
2058
2522
|
let floor = (self.ceiling / 8).max(500);
|
|
2059
|
-
let next = if
|
|
2060
|
-
(
|
|
2061
|
-
|
|
2062
|
-
|
|
2523
|
+
let next = if let Some(path) = path.filter(|path| path.queue_delay > QUEUE_HIGH) {
|
|
2524
|
+
if !waited(DELAY_BACKOFF_EVERY) {
|
|
2525
|
+
return None;
|
|
2526
|
+
}
|
|
2527
|
+
let delivered = path.delivered_kbps.clamp(self.target / 2, self.target);
|
|
2528
|
+
((delivered as f64 * 0.85) as u32).max(floor)
|
|
2063
2529
|
} else {
|
|
2064
|
-
|
|
2530
|
+
let worst = *reports.iter().max()?;
|
|
2531
|
+
if !waited(Duration::from_secs(1)) {
|
|
2532
|
+
return None;
|
|
2533
|
+
}
|
|
2534
|
+
let loss = worst as f64 / 256.0;
|
|
2535
|
+
let roomy = self.queue.is_none_or(|queue| queue < QUEUE_LOW);
|
|
2536
|
+
if loss > 0.10 {
|
|
2537
|
+
((self.target as f64 * 0.7) as u32).max(floor)
|
|
2538
|
+
} else if loss < 0.02 && roomy {
|
|
2539
|
+
((self.target as f64 * 1.08) as u32 + 64).min(self.ceiling)
|
|
2540
|
+
} else {
|
|
2541
|
+
self.target
|
|
2542
|
+
}
|
|
2065
2543
|
};
|
|
2066
2544
|
if next == self.target {
|
|
2067
2545
|
return None;
|
|
@@ -2072,9 +2550,127 @@ impl RateControl {
|
|
|
2072
2550
|
}
|
|
2073
2551
|
}
|
|
2074
2552
|
|
|
2553
|
+
/// How far the stream may run ahead of its rate target before the next frame
|
|
2554
|
+
/// waits: one large frame (a key frame, a refinement) goes at once, but the
|
|
2555
|
+
/// frames behind it hold until the link has had time to carry it, instead of
|
|
2556
|
+
/// queueing behind it. The frame that goes then is the newest one.
|
|
2557
|
+
const FRAME_BURST: Duration = Duration::from_millis(60);
|
|
2558
|
+
|
|
2559
|
+
/// How much faster than the target the budget drains. A rate-controlled
|
|
2560
|
+
/// encoder averages its target but overshoots it frame to frame; drained at
|
|
2561
|
+
/// the target itself, that ordinary overshoot would add up and hold frames on
|
|
2562
|
+
/// a link with room to spare. The rate control, not this, keeps the average
|
|
2563
|
+
/// under what the path carries.
|
|
2564
|
+
const BUDGET_HEADROOM: f64 = 1.25;
|
|
2565
|
+
|
|
2566
|
+
/// What has been sent beyond the rate target, draining a little faster than
|
|
2567
|
+
/// it: a leaky bucket that bounds a burst to `FRAME_BURST` plus one frame.
|
|
2568
|
+
#[derive(Default)]
|
|
2569
|
+
struct SendBudget {
|
|
2570
|
+
bits: f64,
|
|
2571
|
+
at: Option<Instant>,
|
|
2572
|
+
}
|
|
2573
|
+
|
|
2574
|
+
impl SendBudget {
|
|
2575
|
+
/// The drain rate, in bits per second.
|
|
2576
|
+
fn rate(kbps: u32) -> f64 {
|
|
2577
|
+
kbps.max(1) as f64 * 1000.0 * BUDGET_HEADROOM
|
|
2578
|
+
}
|
|
2579
|
+
|
|
2580
|
+
fn drained(&self, now: Instant, kbps: u32) -> f64 {
|
|
2581
|
+
let elapsed = self
|
|
2582
|
+
.at
|
|
2583
|
+
.map_or(0.0, |at| now.saturating_duration_since(at).as_secs_f64());
|
|
2584
|
+
(self.bits - elapsed * Self::rate(kbps)).max(0.0)
|
|
2585
|
+
}
|
|
2586
|
+
|
|
2587
|
+
/// How long until a frame may go; zero when it may go now.
|
|
2588
|
+
fn wait(&self, now: Instant, kbps: u32) -> Duration {
|
|
2589
|
+
let rate = Self::rate(kbps);
|
|
2590
|
+
let over = self.drained(now, kbps) - FRAME_BURST.as_secs_f64() * rate;
|
|
2591
|
+
Duration::from_secs_f64((over / rate).max(0.0))
|
|
2592
|
+
}
|
|
2593
|
+
|
|
2594
|
+
fn sent(&mut self, now: Instant, bytes: usize, kbps: u32) {
|
|
2595
|
+
self.bits = self.drained(now, kbps) + bytes as f64 * 8.0;
|
|
2596
|
+
self.at = Some(now);
|
|
2597
|
+
}
|
|
2598
|
+
}
|
|
2599
|
+
|
|
2600
|
+
/// Whether motion is coded at half size. A full-size frame is slow when it
|
|
2601
|
+
/// took longer to code than a frame interval and the source already had a
|
|
2602
|
+
/// newer frame waiting: the encoder, not the source, set the pace. Content that
|
|
2603
|
+
/// arrives slower than the frame cap (30 fps into a 60 fps session) is never
|
|
2604
|
+
/// slow however long a frame takes, as long as it is done before the next one.
|
|
2605
|
+
/// Several slow frames that each change much of the picture (a packet over a
|
|
2606
|
+
/// quarter of the per-frame rate budget) mean full size cannot reach the frame
|
|
2607
|
+
/// rate, so the rest of the motion is coded at half size, until it stops (the
|
|
2608
|
+
/// refinement pass is always full size) or a run of frames that are both small
|
|
2609
|
+
/// and not slow shows full size would keep up again. Packet size or coding time
|
|
2610
|
+
/// alone never returns it to full size, so heavy motion does not flap between
|
|
2611
|
+
/// sizes; each change costs a key frame. Both directions need a run of frames
|
|
2612
|
+
/// rather than one, so a slow key frame or a busy moment on the host does not
|
|
2613
|
+
/// flip it, and typing never halves.
|
|
2614
|
+
#[derive(Default)]
|
|
2615
|
+
struct MotionSize {
|
|
2616
|
+
half: bool,
|
|
2617
|
+
run: u32,
|
|
2618
|
+
}
|
|
2619
|
+
|
|
2620
|
+
impl MotionSize {
|
|
2621
|
+
const SLOW_RUN: u32 = 5;
|
|
2622
|
+
const SMALL_RUN: u32 = 30;
|
|
2623
|
+
|
|
2624
|
+
/// Account for a motion frame coded at the current size in `took`; `large`
|
|
2625
|
+
/// when its packet was over a quarter of the per-frame rate budget, `behind`
|
|
2626
|
+
/// when a newer frame was already waiting once it was coded.
|
|
2627
|
+
fn coded(&mut self, took: Duration, large: bool, behind: bool, interval: Duration) {
|
|
2628
|
+
let slow = took > interval && behind;
|
|
2629
|
+
let (counts, needed) = if self.half {
|
|
2630
|
+
(!large && !slow, Self::SMALL_RUN)
|
|
2631
|
+
} else {
|
|
2632
|
+
(large && slow, Self::SLOW_RUN)
|
|
2633
|
+
};
|
|
2634
|
+
self.run = if counts { self.run + 1 } else { 0 };
|
|
2635
|
+
if self.run >= needed {
|
|
2636
|
+
self.half = !self.half;
|
|
2637
|
+
self.run = 0;
|
|
2638
|
+
}
|
|
2639
|
+
}
|
|
2640
|
+
}
|
|
2641
|
+
|
|
2642
|
+
/// The rate cap as a deadline rather than a minimum gap. `due` is where the
|
|
2643
|
+
/// next frame falls on the `max_fps` cadence the stream has kept so far, and a
|
|
2644
|
+
/// frame may go up to one interval before it, so a late frame's lost time is
|
|
2645
|
+
/// made up by the next. A capture clock that jitters around the cap is then
|
|
2646
|
+
/// coded the moment each frame arrives, instead of beating against a second
|
|
2647
|
+
/// clock that delays a frame or lets the next one supersede it; the sustained
|
|
2648
|
+
/// rate still never exceeds the cap, and no window holds more than one frame
|
|
2649
|
+
/// over it.
|
|
2650
|
+
struct Cadence {
|
|
2651
|
+
interval: Duration,
|
|
2652
|
+
due: Instant,
|
|
2653
|
+
}
|
|
2654
|
+
|
|
2655
|
+
impl Cadence {
|
|
2656
|
+
fn new(interval: Duration, now: Instant) -> Self {
|
|
2657
|
+
Self { interval, due: now }
|
|
2658
|
+
}
|
|
2659
|
+
|
|
2660
|
+
/// How long until a frame may go; zero when it may go now.
|
|
2661
|
+
fn wait(&self, now: Instant) -> Duration {
|
|
2662
|
+
self.due.saturating_duration_since(now + self.interval)
|
|
2663
|
+
}
|
|
2664
|
+
|
|
2665
|
+
fn sent(&mut self, now: Instant) {
|
|
2666
|
+
self.due = self.due.max(now) + self.interval;
|
|
2667
|
+
}
|
|
2668
|
+
}
|
|
2669
|
+
|
|
2075
2670
|
enum Pass {
|
|
2076
2671
|
Motion {
|
|
2077
2672
|
keyframe: bool,
|
|
2673
|
+
half: bool,
|
|
2078
2674
|
},
|
|
2079
2675
|
Refine,
|
|
2080
2676
|
/// The same picture again, so a still stream never looks stalled.
|
|
@@ -2107,8 +2703,11 @@ fn spawn_pipeline(
|
|
|
2107
2703
|
let mut rate = RateControl::new(bitrate_kbps);
|
|
2108
2704
|
if let Ok(mut m) = inner.metrics.lock() {
|
|
2109
2705
|
m.target_kbps = bitrate_kbps;
|
|
2706
|
+
(m.codec, m.encoder) = ("vp9", "libvpx");
|
|
2110
2707
|
}
|
|
2111
|
-
|
|
2708
|
+
// Whether motion may be coded at half size: VP9's alone.
|
|
2709
|
+
let mut halves = true;
|
|
2710
|
+
let mut latest: Option<(I420, Instant)> = None;
|
|
2112
2711
|
// `pending`: the latest frame has not been coded. `refined`: it has
|
|
2113
2712
|
// had its refinement pass, or there is nothing to refine.
|
|
2114
2713
|
let mut pending = false;
|
|
@@ -2116,11 +2715,15 @@ fn spawn_pipeline(
|
|
|
2116
2715
|
let mut keyframe = true;
|
|
2117
2716
|
let mut still_since = Instant::now();
|
|
2118
2717
|
let mut last_sent: Option<Instant> = None;
|
|
2718
|
+
let mut size = MotionSize::default();
|
|
2719
|
+
let mut cadence = Cadence::new(interval, Instant::now());
|
|
2720
|
+
let mut budget = SendBudget::default();
|
|
2721
|
+
// What the last frame was stamped with: the RTP clock never runs
|
|
2722
|
+
// backwards, even for a re-coded picture.
|
|
2723
|
+
let mut last_stamp: Option<Instant> = None;
|
|
2119
2724
|
loop {
|
|
2120
2725
|
let now = Instant::now();
|
|
2121
|
-
let slot =
|
|
2122
|
-
interval.saturating_sub(now.duration_since(at))
|
|
2123
|
-
});
|
|
2726
|
+
let slot = cadence.wait(now).max(budget.wait(now, rate.target));
|
|
2124
2727
|
let connected = peer.is_connected();
|
|
2125
2728
|
let wait = if !connected {
|
|
2126
2729
|
// Nothing sent before the transport connects arrives; the
|
|
@@ -2140,13 +2743,13 @@ fn spawn_pipeline(
|
|
|
2140
2743
|
.clamp(Duration::from_millis(10), Duration::from_millis(100))
|
|
2141
2744
|
};
|
|
2142
2745
|
match frame_rx.take(wait) {
|
|
2143
|
-
Taken::Frame(frame) => {
|
|
2746
|
+
Taken::Frame(frame, captured) => {
|
|
2144
2747
|
if pending {
|
|
2145
2748
|
if let Ok(mut m) = inner.metrics.lock() {
|
|
2146
2749
|
m.dropped_frames += 1;
|
|
2147
2750
|
}
|
|
2148
2751
|
}
|
|
2149
|
-
latest = Some(frame);
|
|
2752
|
+
latest = Some((frame, captured));
|
|
2150
2753
|
pending = true;
|
|
2151
2754
|
refined = false;
|
|
2152
2755
|
still_since = Instant::now();
|
|
@@ -2161,7 +2764,8 @@ fn spawn_pipeline(
|
|
|
2161
2764
|
keyframe = true;
|
|
2162
2765
|
}
|
|
2163
2766
|
let reports = peer.take_loss_reports();
|
|
2164
|
-
|
|
2767
|
+
let path = peer.take_path_report();
|
|
2768
|
+
if let Some(kbps) = rate.update(&reports, path, Instant::now()) {
|
|
2165
2769
|
let applied = inner.encoder.lock().map(|mut encoder| {
|
|
2166
2770
|
encoder
|
|
2167
2771
|
.as_mut()
|
|
@@ -2174,18 +2778,45 @@ fn spawn_pipeline(
|
|
|
2174
2778
|
}
|
|
2175
2779
|
}
|
|
2176
2780
|
}
|
|
2177
|
-
let Some(frame) = latest.as_ref() else {
|
|
2781
|
+
let Some((frame, captured)) = latest.as_ref() else {
|
|
2178
2782
|
continue;
|
|
2179
2783
|
};
|
|
2180
2784
|
if !connected {
|
|
2181
2785
|
continue;
|
|
2182
2786
|
}
|
|
2183
2787
|
let now = Instant::now();
|
|
2184
|
-
if
|
|
2788
|
+
if !cadence.wait(now).is_zero() || !budget.wait(now, rate.target).is_zero() {
|
|
2185
2789
|
continue;
|
|
2186
2790
|
}
|
|
2791
|
+
if halves && peer.sends_h264() {
|
|
2792
|
+
// The answer asked for H.264: nothing has been sent yet,
|
|
2793
|
+
// since frames wait for the transport, which waits for it.
|
|
2794
|
+
match start_h264(&inner, rate.target, max_fps) {
|
|
2795
|
+
Ok(()) => {
|
|
2796
|
+
halves = false;
|
|
2797
|
+
size = MotionSize::default();
|
|
2798
|
+
keyframe = true;
|
|
2799
|
+
}
|
|
2800
|
+
Err(error) => {
|
|
2801
|
+
eprintln!("{error:#}");
|
|
2802
|
+
let reason = String::from("no H.264 encoder could start");
|
|
2803
|
+
inner.notify(SessionEvent::Revoked {
|
|
2804
|
+
reason: reason.clone(),
|
|
2805
|
+
code: "error",
|
|
2806
|
+
});
|
|
2807
|
+
let target = inner.clone();
|
|
2808
|
+
handle.spawn(async move {
|
|
2809
|
+
target.close(&reason, "error").await;
|
|
2810
|
+
});
|
|
2811
|
+
break;
|
|
2812
|
+
}
|
|
2813
|
+
}
|
|
2814
|
+
}
|
|
2187
2815
|
let pass = if pending || keyframe {
|
|
2188
|
-
Pass::Motion {
|
|
2816
|
+
Pass::Motion {
|
|
2817
|
+
keyframe,
|
|
2818
|
+
half: size.half && halves,
|
|
2819
|
+
}
|
|
2189
2820
|
} else if !refined && now.duration_since(still_since) >= REFINE_AFTER {
|
|
2190
2821
|
Pass::Refine
|
|
2191
2822
|
} else if refined
|
|
@@ -2206,9 +2837,9 @@ fn spawn_pipeline(
|
|
|
2206
2837
|
break;
|
|
2207
2838
|
};
|
|
2208
2839
|
match pass {
|
|
2209
|
-
Pass::Motion { keyframe } => encoder.encode(frame, keyframe),
|
|
2840
|
+
Pass::Motion { keyframe, half } => encoder.encode(frame, keyframe, half),
|
|
2210
2841
|
Pass::Refine => encoder.refine(frame),
|
|
2211
|
-
Pass::Keepalive => encoder.encode(frame, false),
|
|
2842
|
+
Pass::Keepalive => encoder.encode(frame, false, false),
|
|
2212
2843
|
}
|
|
2213
2844
|
};
|
|
2214
2845
|
let packet = match packet {
|
|
@@ -2226,22 +2857,29 @@ fn spawn_pipeline(
|
|
|
2226
2857
|
let reason = String::from("the encoder rejected a frame");
|
|
2227
2858
|
inner.notify(SessionEvent::Revoked {
|
|
2228
2859
|
reason: reason.clone(),
|
|
2860
|
+
code: "error",
|
|
2229
2861
|
});
|
|
2230
2862
|
let target = inner.clone();
|
|
2231
2863
|
handle.spawn(async move {
|
|
2232
|
-
target.close(&reason).await;
|
|
2864
|
+
target.close(&reason, "error").await;
|
|
2233
2865
|
});
|
|
2234
2866
|
break;
|
|
2235
2867
|
}
|
|
2236
2868
|
};
|
|
2869
|
+
let took = started.elapsed();
|
|
2237
2870
|
if let Ok(mut m) = inner.metrics.lock() {
|
|
2238
2871
|
m.encoded_frames += 1;
|
|
2239
2872
|
m.encoded_bytes += packet.data.len() as u64;
|
|
2240
|
-
m.encode_micros +=
|
|
2873
|
+
m.encode_micros += took.as_micros() as u64;
|
|
2874
|
+
m.key_frames += packet.keyframe as u64;
|
|
2875
|
+
if matches!(pass, Pass::Motion { .. }) && pending {
|
|
2876
|
+
m.queue_micros += started.duration_since(*captured).as_micros() as u64;
|
|
2877
|
+
m.motion_frames += 1;
|
|
2878
|
+
}
|
|
2241
2879
|
match pass {
|
|
2242
|
-
Pass::Motion {
|
|
2880
|
+
Pass::Motion { half: true, .. } => m.halved_frames += 1,
|
|
2243
2881
|
Pass::Refine => m.refined_frames += 1,
|
|
2244
|
-
Pass::Motion {
|
|
2882
|
+
Pass::Motion { half: false, .. } | Pass::Keepalive => {}
|
|
2245
2883
|
}
|
|
2246
2884
|
}
|
|
2247
2885
|
match pass {
|
|
@@ -2250,13 +2888,34 @@ fn spawn_pipeline(
|
|
|
2250
2888
|
keyframe = false;
|
|
2251
2889
|
refined = false;
|
|
2252
2890
|
still_since = now;
|
|
2891
|
+
let budget = rate.target as u64 * 1000 / 8 / max_fps.max(1) as u64;
|
|
2892
|
+
let large = packet.data.len() as u64 * 4 > budget;
|
|
2893
|
+
if halves {
|
|
2894
|
+
size.coded(took, large, frame_rx.waiting(), interval);
|
|
2895
|
+
}
|
|
2896
|
+
}
|
|
2897
|
+
Pass::Refine => {
|
|
2898
|
+
refined = true;
|
|
2899
|
+
size = MotionSize::default();
|
|
2253
2900
|
}
|
|
2254
|
-
Pass::Refine => refined = true,
|
|
2255
2901
|
Pass::Keepalive => {}
|
|
2256
2902
|
}
|
|
2257
2903
|
last_sent = Some(now);
|
|
2904
|
+
cadence.sent(now);
|
|
2905
|
+
budget.sent(now, packet.data.len(), rate.target);
|
|
2906
|
+
// A new picture carries the moment it was captured, so the
|
|
2907
|
+
// receiver paces playout by the desktop's clock rather than by
|
|
2908
|
+
// encode time; a re-coded one (refinement, keepalive) is new
|
|
2909
|
+
// content now.
|
|
2910
|
+
let stamp = match pass {
|
|
2911
|
+
Pass::Motion { .. } => *captured,
|
|
2912
|
+
Pass::Refine | Pass::Keepalive => now,
|
|
2913
|
+
};
|
|
2914
|
+
let stamp = last_stamp.map_or(stamp, |at| stamp.max(at + Duration::from_millis(1)));
|
|
2915
|
+
last_stamp = Some(stamp);
|
|
2916
|
+
let sending = Instant::now();
|
|
2258
2917
|
if let Err(error) =
|
|
2259
|
-
handle.block_on(peer.send_frame(&packet.data, packet.keyframe,
|
|
2918
|
+
handle.block_on(peer.send_frame(&packet.data, packet.keyframe, stamp))
|
|
2260
2919
|
{
|
|
2261
2920
|
// The transport is gone. End the session rather than leave
|
|
2262
2921
|
// the desktop captured behind a picture that stopped.
|
|
@@ -2264,18 +2923,41 @@ fn spawn_pipeline(
|
|
|
2264
2923
|
let reason = String::from("the connection to the phone was lost");
|
|
2265
2924
|
inner.notify(SessionEvent::Revoked {
|
|
2266
2925
|
reason: reason.clone(),
|
|
2926
|
+
code: "transport",
|
|
2267
2927
|
});
|
|
2268
2928
|
let target = inner.clone();
|
|
2269
2929
|
handle.spawn(async move {
|
|
2270
|
-
target.close(&reason).await;
|
|
2930
|
+
target.close(&reason, "transport").await;
|
|
2271
2931
|
});
|
|
2272
2932
|
break;
|
|
2273
2933
|
}
|
|
2934
|
+
if let Ok(mut m) = inner.metrics.lock() {
|
|
2935
|
+
m.send_micros += sending.elapsed().as_micros() as u64;
|
|
2936
|
+
}
|
|
2274
2937
|
}
|
|
2275
2938
|
})
|
|
2276
2939
|
.ok();
|
|
2277
2940
|
}
|
|
2278
2941
|
|
|
2942
|
+
/// Replace the session's VP9 encoder with an H.264 one of the same size and
|
|
2943
|
+
/// rate target, on the encode thread that will drive it.
|
|
2944
|
+
fn start_h264(inner: &Arc<Inner>, bitrate_kbps: u32, fps: u32) -> Result<()> {
|
|
2945
|
+
let mut held = lock(&inner.encoder);
|
|
2946
|
+
let (width, height) = held
|
|
2947
|
+
.as_ref()
|
|
2948
|
+
.map(VideoEncoder::size)
|
|
2949
|
+
.context("the session has no encoder")?;
|
|
2950
|
+
let (backend, info) = crate::h264::open(width, height, bitrate_kbps, fps)?;
|
|
2951
|
+
*held = Some(VideoEncoder::H264 {
|
|
2952
|
+
backend,
|
|
2953
|
+
size: (width, height),
|
|
2954
|
+
});
|
|
2955
|
+
if let Ok(mut m) = inner.metrics.lock() {
|
|
2956
|
+
(m.codec, m.encoder) = ("h264", info.encoder);
|
|
2957
|
+
}
|
|
2958
|
+
Ok(())
|
|
2959
|
+
}
|
|
2960
|
+
|
|
2279
2961
|
/// Packetize what a consumer feeds an encoded source, building the peer on the
|
|
2280
2962
|
/// first access unit that carries an SPS.
|
|
2281
2963
|
///
|
|
@@ -2338,6 +3020,7 @@ fn spawn_encoded(inner: &Arc<Inner>, mut feed: tokio_mpsc::Receiver<FedAccessUni
|
|
|
2338
3020
|
revoke(
|
|
2339
3021
|
&inner,
|
|
2340
3022
|
format!("the transport refused the encoded stream: {error:#}"),
|
|
3023
|
+
"error",
|
|
2341
3024
|
)
|
|
2342
3025
|
.await;
|
|
2343
3026
|
return;
|
|
@@ -2362,7 +3045,12 @@ fn spawn_encoded(inner: &Arc<Inner>, mut feed: tokio_mpsc::Receiver<FedAccessUni
|
|
|
2362
3045
|
}
|
|
2363
3046
|
Err(error) => {
|
|
2364
3047
|
eprintln!("the video track refused a fed access unit: {error:#}");
|
|
2365
|
-
revoke(
|
|
3048
|
+
revoke(
|
|
3049
|
+
&inner,
|
|
3050
|
+
String::from("the connection to the phone was lost"),
|
|
3051
|
+
"transport",
|
|
3052
|
+
)
|
|
3053
|
+
.await;
|
|
2366
3054
|
return;
|
|
2367
3055
|
}
|
|
2368
3056
|
}
|
|
@@ -2376,11 +3064,12 @@ fn current_peer(inner: &Arc<Inner>) -> Option<Arc<VideoPeer>> {
|
|
|
2376
3064
|
|
|
2377
3065
|
/// End the session with a reason the consumer can show, exactly as the encode
|
|
2378
3066
|
/// pipeline does when its transport goes away.
|
|
2379
|
-
async fn revoke(inner: &Arc<Inner>, reason: String) {
|
|
3067
|
+
async fn revoke(inner: &Arc<Inner>, reason: String, code: &'static str) {
|
|
2380
3068
|
inner.notify(SessionEvent::Revoked {
|
|
2381
3069
|
reason: reason.clone(),
|
|
3070
|
+
code,
|
|
2382
3071
|
});
|
|
2383
|
-
inner.close(&reason).await;
|
|
3072
|
+
inner.close(&reason, code).await;
|
|
2384
3073
|
}
|
|
2385
3074
|
|
|
2386
3075
|
fn spawn_peer_events(inner: &Arc<Inner>, mut events: tokio_mpsc::UnboundedReceiver<PeerEvent>) {
|
|
@@ -2428,8 +3117,9 @@ fn spawn_peer_events(inner: &Arc<Inner>, mut events: tokio_mpsc::UnboundedReceiv
|
|
|
2428
3117
|
let reason = String::from("the connection to the phone was lost");
|
|
2429
3118
|
inner.notify(SessionEvent::Revoked {
|
|
2430
3119
|
reason: reason.clone(),
|
|
3120
|
+
code: "transport",
|
|
2431
3121
|
});
|
|
2432
|
-
inner.close(&reason).await;
|
|
3122
|
+
inner.close(&reason, "transport").await;
|
|
2433
3123
|
continue;
|
|
2434
3124
|
}
|
|
2435
3125
|
inner.notify(SessionEvent::State {
|
|
@@ -2487,10 +3177,11 @@ fn spawn_lease(inner: &Arc<Inner>, ttl: Duration) {
|
|
|
2487
3177
|
let reason = String::from("the session lease expired");
|
|
2488
3178
|
inner.notify(SessionEvent::Revoked {
|
|
2489
3179
|
reason: reason.clone(),
|
|
3180
|
+
code: "lease",
|
|
2490
3181
|
});
|
|
2491
3182
|
// An expiry ends the session exactly as an explicit close does; the
|
|
2492
3183
|
// notification above is not a substitute for stopping the desktop.
|
|
2493
|
-
inner.close(&reason).await;
|
|
3184
|
+
inner.close(&reason, "lease").await;
|
|
2494
3185
|
});
|
|
2495
3186
|
}
|
|
2496
3187
|
|
|
@@ -2558,6 +3249,9 @@ mod tests {
|
|
|
2558
3249
|
encoder: Mutex::new(None),
|
|
2559
3250
|
input: Mutex::new(None),
|
|
2560
3251
|
indicator: Mutex::new(None),
|
|
3252
|
+
point_overlay: Mutex::new(None),
|
|
3253
|
+
#[cfg(target_os = "linux")]
|
|
3254
|
+
point_desktop: PointDesktop::Unavailable("layer_shell_unavailable"),
|
|
2561
3255
|
capture: Mutex::new(None),
|
|
2562
3256
|
layout: Mutex::new(None),
|
|
2563
3257
|
last_seq: Mutex::new(0),
|
|
@@ -2575,10 +3269,12 @@ mod tests {
|
|
|
2575
3269
|
peer_events,
|
|
2576
3270
|
feed,
|
|
2577
3271
|
})),
|
|
3272
|
+
local_frames: false,
|
|
3273
|
+
cursor_info: serde_json::json!({"mode":"unavailable", "positions":false}),
|
|
2578
3274
|
});
|
|
2579
3275
|
spawn_peer_events(&inner, peer_events_rx);
|
|
2580
3276
|
spawn_encoded(&inner, feed_rx);
|
|
2581
|
-
inner.close("test").await;
|
|
3277
|
+
inner.close("test", "closed").await;
|
|
2582
3278
|
for _ in 0..100 {
|
|
2583
3279
|
if Arc::strong_count(&inner) == 1 {
|
|
2584
3280
|
break;
|
|
@@ -2600,6 +3296,41 @@ mod tests {
|
|
|
2600
3296
|
);
|
|
2601
3297
|
}
|
|
2602
3298
|
|
|
3299
|
+
#[test]
|
|
3300
|
+
fn frames_that_jitter_around_the_cap_go_at_once_and_a_flood_stays_capped() {
|
|
3301
|
+
let interval = Duration::from_millis(20);
|
|
3302
|
+
let start = Instant::now();
|
|
3303
|
+
let mut cadence = Cadence::new(interval, start);
|
|
3304
|
+
// Captured at the cap, each a few ms early or late: none waits, so
|
|
3305
|
+
// none is delayed or superseded by the next.
|
|
3306
|
+
for (n, jitter) in [0i64, -6, 5, -8, 7, -3, 0, -9, 9, -5].iter().enumerate() {
|
|
3307
|
+
let at = start + interval * n as u32 + Duration::from_millis(9);
|
|
3308
|
+
let at = if *jitter < 0 {
|
|
3309
|
+
at - Duration::from_millis(jitter.unsigned_abs())
|
|
3310
|
+
} else {
|
|
3311
|
+
at + Duration::from_millis(*jitter as u64)
|
|
3312
|
+
};
|
|
3313
|
+
assert_eq!(cadence.wait(at), Duration::ZERO, "frame {n} waited");
|
|
3314
|
+
cadence.sent(at);
|
|
3315
|
+
}
|
|
3316
|
+
|
|
3317
|
+
// A source far above the cap: one second of 1 ms frames sends at most
|
|
3318
|
+
// one frame over the cap.
|
|
3319
|
+
let start = start + Duration::from_secs(10);
|
|
3320
|
+
let mut cadence = Cadence::new(interval, start);
|
|
3321
|
+
let sent = (0..1000)
|
|
3322
|
+
.map(|ms| start + Duration::from_millis(ms))
|
|
3323
|
+
.filter(|&at| {
|
|
3324
|
+
let go = cadence.wait(at).is_zero();
|
|
3325
|
+
if go {
|
|
3326
|
+
cadence.sent(at);
|
|
3327
|
+
}
|
|
3328
|
+
go
|
|
3329
|
+
})
|
|
3330
|
+
.count();
|
|
3331
|
+
assert!((50..=51).contains(&sent), "sent {sent} at a 50 fps cap");
|
|
3332
|
+
}
|
|
3333
|
+
|
|
2603
3334
|
#[test]
|
|
2604
3335
|
fn a_small_source_is_never_upscaled() {
|
|
2605
3336
|
assert_eq!(fit(1000, 600, 1280, 800), (1000, 600));
|
|
@@ -2658,6 +3389,149 @@ mod tests {
|
|
|
2658
3389
|
)
|
|
2659
3390
|
.await
|
|
2660
3391
|
.expect("a peer connection builds without a network");
|
|
3392
|
+
test_inner_with(events, peer, 64, 64)
|
|
3393
|
+
}
|
|
3394
|
+
|
|
3395
|
+
#[cfg(target_os = "linux")]
|
|
3396
|
+
#[tokio::test]
|
|
3397
|
+
async fn a_wayland_portal_session_reports_typed_point_unsupported() {
|
|
3398
|
+
let (events, _) = tokio_mpsc::unbounded_channel();
|
|
3399
|
+
let (mut inner, _) = test_inner(events).await;
|
|
3400
|
+
Arc::get_mut(&mut inner)
|
|
3401
|
+
.expect("test session has one owner")
|
|
3402
|
+
.permissions = vec![Permission::View];
|
|
3403
|
+
let session = Session { inner };
|
|
3404
|
+
|
|
3405
|
+
let error = session
|
|
3406
|
+
.point(&crate::protocol::PointParams {
|
|
3407
|
+
session_id: String::from("test-session"),
|
|
3408
|
+
x: Some(10),
|
|
3409
|
+
y: Some(20),
|
|
3410
|
+
label: String::new(),
|
|
3411
|
+
timeout_ms: None,
|
|
3412
|
+
clear: false,
|
|
3413
|
+
})
|
|
3414
|
+
.expect_err("compositors without layer-shell do not support point overlays");
|
|
3415
|
+
|
|
3416
|
+
assert_eq!(error.code, "point_unsupported");
|
|
3417
|
+
assert!(error.message.contains("reason: layer_shell_unavailable"));
|
|
3418
|
+
}
|
|
3419
|
+
|
|
3420
|
+
// Driven only on the private headless compositor created by point-wayland-flow.py.
|
|
3421
|
+
#[cfg(target_os = "linux")]
|
|
3422
|
+
#[tokio::test]
|
|
3423
|
+
#[ignore = "requires the task-owned compositor and screenshot driver"]
|
|
3424
|
+
async fn point_wayland_private_flow() {
|
|
3425
|
+
let evidence = std::path::PathBuf::from(
|
|
3426
|
+
std::env::var_os("DESKLINK_POINT_TEST_DIR").expect("private evidence"),
|
|
3427
|
+
);
|
|
3428
|
+
let socket =
|
|
3429
|
+
std::path::PathBuf::from(std::env::var_os("WAYLAND_DISPLAY").expect("private socket"));
|
|
3430
|
+
assert!(
|
|
3431
|
+
socket.is_absolute() && socket.starts_with(evidence.join("runtime")),
|
|
3432
|
+
"never use an ambient desktop"
|
|
3433
|
+
);
|
|
3434
|
+
fn checkpoint(dir: &std::path::Path, name: &str) {
|
|
3435
|
+
std::fs::write(dir.join(format!("{name}.ready")), b"").unwrap();
|
|
3436
|
+
let deadline = Instant::now() + Duration::from_secs(15);
|
|
3437
|
+
while !dir.join(format!("{name}.continue")).exists() {
|
|
3438
|
+
assert!(
|
|
3439
|
+
Instant::now() < deadline,
|
|
3440
|
+
"screenshot driver timed out at {name}"
|
|
3441
|
+
);
|
|
3442
|
+
std::thread::sleep(Duration::from_millis(20));
|
|
3443
|
+
}
|
|
3444
|
+
}
|
|
3445
|
+
let (events, _) = tokio_mpsc::unbounded_channel();
|
|
3446
|
+
let (mut inner, _) = test_inner(events).await;
|
|
3447
|
+
let state = Arc::get_mut(&mut inner).unwrap();
|
|
3448
|
+
state.permissions = vec![Permission::View];
|
|
3449
|
+
state.point_desktop = PointDesktop::Wayland;
|
|
3450
|
+
state.source.width = 1280;
|
|
3451
|
+
state.source.height = 720;
|
|
3452
|
+
state.source.position = Some((1280, 0));
|
|
3453
|
+
let mut session = Session { inner };
|
|
3454
|
+
let mut params = crate::protocol::PointParams {
|
|
3455
|
+
session_id: String::from("test-session"),
|
|
3456
|
+
x: Some(400),
|
|
3457
|
+
y: Some(300),
|
|
3458
|
+
label: String::from("Here"),
|
|
3459
|
+
timeout_ms: Some(3000),
|
|
3460
|
+
clear: false,
|
|
3461
|
+
};
|
|
3462
|
+
let unsupported_socket = std::env::var_os("DESKLINK_POINT_UNSUPPORTED_SOCKET").unwrap();
|
|
3463
|
+
assert!(std::path::Path::new(&unsupported_socket).starts_with(evidence.join("runtime")));
|
|
3464
|
+
// This test runs alone, in a separate process. Only child helper startup
|
|
3465
|
+
// and the read-only clear probe read this private socket environment.
|
|
3466
|
+
std::env::set_var("WAYLAND_DISPLAY", &unsupported_socket);
|
|
3467
|
+
let refusal = session.point(¶ms).unwrap_err();
|
|
3468
|
+
assert_eq!(refusal.code, "point_unsupported");
|
|
3469
|
+
assert!(refusal.message.contains("reason: layer_shell_unavailable"));
|
|
3470
|
+
let clear = crate::protocol::PointParams {
|
|
3471
|
+
session_id: params.session_id.clone(),
|
|
3472
|
+
clear: true,
|
|
3473
|
+
x: None,
|
|
3474
|
+
y: None,
|
|
3475
|
+
label: String::new(),
|
|
3476
|
+
timeout_ms: None,
|
|
3477
|
+
};
|
|
3478
|
+
let refusal = session.point(&clear).unwrap_err();
|
|
3479
|
+
assert_eq!(refusal.code, "point_unsupported");
|
|
3480
|
+
std::env::set_var("WAYLAND_DISPLAY", &socket);
|
|
3481
|
+
Arc::get_mut(&mut session.inner).unwrap().source.position = None;
|
|
3482
|
+
let ambiguous = session.point(¶ms).unwrap_err();
|
|
3483
|
+
assert_eq!(ambiguous.code, "indicator-unavailable");
|
|
3484
|
+
assert!(ambiguous.message.contains("selected_output_unavailable"));
|
|
3485
|
+
Arc::get_mut(&mut session.inner).unwrap().source.position = Some((1280, 0));
|
|
3486
|
+
checkpoint(&evidence, "before");
|
|
3487
|
+
session.point(¶ms).unwrap();
|
|
3488
|
+
checkpoint(&evidence, "after");
|
|
3489
|
+
params.x = Some(1280);
|
|
3490
|
+
assert_eq!(session.point(¶ms).unwrap_err().code, "coordinates");
|
|
3491
|
+
params.x = Some(400);
|
|
3492
|
+
params.label = String::from("bad\nlabel");
|
|
3493
|
+
assert_eq!(session.point(¶ms).unwrap_err().code, "malformed");
|
|
3494
|
+
checkpoint(&evidence, "invalid");
|
|
3495
|
+
tokio::time::sleep(Duration::from_millis(3100)).await;
|
|
3496
|
+
checkpoint(&evidence, "expired");
|
|
3497
|
+
params.x = Some(700);
|
|
3498
|
+
params.y = Some(500);
|
|
3499
|
+
params.label.clear();
|
|
3500
|
+
params.timeout_ms = Some(10000);
|
|
3501
|
+
session.point(¶ms).unwrap();
|
|
3502
|
+
checkpoint(&evidence, "replaced");
|
|
3503
|
+
params.clear = true;
|
|
3504
|
+
params.x = None;
|
|
3505
|
+
params.y = None;
|
|
3506
|
+
params.timeout_ms = None;
|
|
3507
|
+
session.point(¶ms).unwrap();
|
|
3508
|
+
session.point(¶ms).unwrap();
|
|
3509
|
+
assert!(lock(&session.inner.point_overlay).is_none());
|
|
3510
|
+
checkpoint(&evidence, "cleared");
|
|
3511
|
+
params.clear = false;
|
|
3512
|
+
params.x = Some(400);
|
|
3513
|
+
params.y = Some(300);
|
|
3514
|
+
session.point(¶ms).unwrap();
|
|
3515
|
+
let pid = lock(&session.inner.point_overlay)
|
|
3516
|
+
.as_ref()
|
|
3517
|
+
.unwrap()
|
|
3518
|
+
.test_pid();
|
|
3519
|
+
checkpoint(&evidence, "clickthrough");
|
|
3520
|
+
session.close("test complete").await;
|
|
3521
|
+
assert_eq!(
|
|
3522
|
+
unsafe { libc::kill(pid as i32, 0) },
|
|
3523
|
+
-1,
|
|
3524
|
+
"helper reaped on session close"
|
|
3525
|
+
);
|
|
3526
|
+
checkpoint(&evidence, "closed");
|
|
3527
|
+
}
|
|
3528
|
+
|
|
3529
|
+
fn test_inner_with(
|
|
3530
|
+
events: tokio_mpsc::UnboundedSender<Notice>,
|
|
3531
|
+
peer: VideoPeer,
|
|
3532
|
+
width: usize,
|
|
3533
|
+
height: usize,
|
|
3534
|
+
) -> (Arc<Inner>, Arc<Mutex<Vec<(i16, bool)>>>) {
|
|
2661
3535
|
// The pipeline codes nothing before the transport connects.
|
|
2662
3536
|
peer.assume_connected();
|
|
2663
3537
|
let recorded = Arc::new(Mutex::new(Vec::new()));
|
|
@@ -2684,7 +3558,9 @@ mod tests {
|
|
|
2684
3558
|
capture_error: Arc::new(Mutex::new(None)),
|
|
2685
3559
|
frame_changes: tokio::sync::watch::channel(0).1,
|
|
2686
3560
|
peer: Mutex::new(Some(Arc::new(peer))),
|
|
2687
|
-
encoder: Mutex::new(Some(
|
|
3561
|
+
encoder: Mutex::new(Some(VideoEncoder::Vp9(
|
|
3562
|
+
Encoder::new(width, height, 1000, 30, 1).expect("an encoder"),
|
|
3563
|
+
))),
|
|
2688
3564
|
input: Mutex::new(Some(InputTarget {
|
|
2689
3565
|
applier: Applier::Recording(recorded.clone()),
|
|
2690
3566
|
held: HeldState::default(),
|
|
@@ -2695,6 +3571,9 @@ mod tests {
|
|
|
2695
3571
|
})),
|
|
2696
3572
|
capture: Mutex::new(None),
|
|
2697
3573
|
indicator: Mutex::new(None),
|
|
3574
|
+
point_overlay: Mutex::new(None),
|
|
3575
|
+
#[cfg(target_os = "linux")]
|
|
3576
|
+
point_desktop: PointDesktop::Unavailable("layer_shell_unavailable"),
|
|
2698
3577
|
layout: Mutex::new(Some(Layout::from_environment().expect("a keymap"))),
|
|
2699
3578
|
last_seq: Mutex::new(0),
|
|
2700
3579
|
control_open: AtomicBool::new(true),
|
|
@@ -2702,6 +3581,8 @@ mod tests {
|
|
|
2702
3581
|
pipeline: Arc::new(AtomicBool::new(true)),
|
|
2703
3582
|
events,
|
|
2704
3583
|
encoded: Mutex::new(None),
|
|
3584
|
+
local_frames: true,
|
|
3585
|
+
cursor_info: serde_json::json!({"mode":"embedded", "positions":false}),
|
|
2705
3586
|
});
|
|
2706
3587
|
(inner, recorded)
|
|
2707
3588
|
}
|
|
@@ -2711,6 +3592,11 @@ mod tests {
|
|
|
2711
3592
|
let (events, mut received) = tokio_mpsc::unbounded_channel();
|
|
2712
3593
|
let (inner, recorded) = test_inner(events).await;
|
|
2713
3594
|
|
|
3595
|
+
#[cfg(target_os = "windows")]
|
|
3596
|
+
inner
|
|
3597
|
+
.with_input(|target| target.chord(1, Vec::new(), true))
|
|
3598
|
+
.unwrap();
|
|
3599
|
+
#[cfg(not(target_os = "windows"))]
|
|
2714
3600
|
inner.apply(ControlMessage::Key {
|
|
2715
3601
|
name: None,
|
|
2716
3602
|
character: Some(String::from("a")),
|
|
@@ -2742,11 +3628,11 @@ mod tests {
|
|
|
2742
3628
|
matches!(
|
|
2743
3629
|
received.try_recv(),
|
|
2744
3630
|
Ok(Notice {
|
|
2745
|
-
event: SessionEvent::Revoked { .. },
|
|
3631
|
+
event: SessionEvent::Revoked { code: "lease", .. },
|
|
2746
3632
|
..
|
|
2747
3633
|
})
|
|
2748
3634
|
),
|
|
2749
|
-
"the consumer is still told
|
|
3635
|
+
"the consumer is still told the session's authority ended",
|
|
2750
3636
|
);
|
|
2751
3637
|
|
|
2752
3638
|
let rejected = inner.metrics.lock().unwrap().input_rejected;
|
|
@@ -2764,6 +3650,7 @@ mod tests {
|
|
|
2764
3650
|
);
|
|
2765
3651
|
}
|
|
2766
3652
|
|
|
3653
|
+
#[cfg(not(target_os = "windows"))]
|
|
2767
3654
|
#[tokio::test]
|
|
2768
3655
|
async fn a_held_modifier_survives_character_named_and_text_chords() {
|
|
2769
3656
|
use crate::input::keycode;
|
|
@@ -3020,20 +3907,24 @@ mod tests {
|
|
|
3020
3907
|
|
|
3021
3908
|
// The encoder is 64x64; a 32x32 frame is the dimension mismatch that
|
|
3022
3909
|
// used to be counted and dropped behind a permanently black picture.
|
|
3023
|
-
frame_tx.put(
|
|
3024
|
-
|
|
3025
|
-
|
|
3026
|
-
|
|
3027
|
-
|
|
3910
|
+
frame_tx.put(
|
|
3911
|
+
I420 {
|
|
3912
|
+
width: 32,
|
|
3913
|
+
height: 32,
|
|
3914
|
+
data: vec![128u8; 32 * 32 + 2 * 16 * 16],
|
|
3915
|
+
},
|
|
3916
|
+
Instant::now(),
|
|
3917
|
+
);
|
|
3028
3918
|
tokio::time::sleep(Duration::from_millis(150)).await;
|
|
3029
3919
|
|
|
3030
3920
|
let mut reason = None;
|
|
3031
3921
|
while let Ok(event) = received.try_recv() {
|
|
3032
3922
|
if let Notice {
|
|
3033
|
-
event: SessionEvent::Revoked { reason: why },
|
|
3923
|
+
event: SessionEvent::Revoked { reason: why, code },
|
|
3034
3924
|
..
|
|
3035
3925
|
} = event
|
|
3036
3926
|
{
|
|
3927
|
+
assert_eq!(code, "error", "a refused frame is a fault, not a lost path");
|
|
3037
3928
|
reason = Some(why);
|
|
3038
3929
|
}
|
|
3039
3930
|
}
|
|
@@ -3059,6 +3950,7 @@ mod tests {
|
|
|
3059
3950
|
|
|
3060
3951
|
fn encoded_open() -> OpenRequest {
|
|
3061
3952
|
OpenRequest {
|
|
3953
|
+
cursor: crate::protocol::CursorMode::Embedded,
|
|
3062
3954
|
source: Some(SourceRequest::Encoded {
|
|
3063
3955
|
codec: EncodedCodec::H264,
|
|
3064
3956
|
width: 486,
|
|
@@ -3074,6 +3966,7 @@ mod tests {
|
|
|
3074
3966
|
ttl: Some(Duration::from_secs(30)),
|
|
3075
3967
|
loopback_tcp: false,
|
|
3076
3968
|
agent_indicator: false,
|
|
3969
|
+
local_frames: true,
|
|
3077
3970
|
}
|
|
3078
3971
|
}
|
|
3079
3972
|
|
|
@@ -3084,6 +3977,18 @@ mod tests {
|
|
|
3084
3977
|
.expect("the channel stays open")
|
|
3085
3978
|
}
|
|
3086
3979
|
|
|
3980
|
+
#[tokio::test]
|
|
3981
|
+
async fn hidden_cursor_is_refused_for_consumer_encoded_pixels() {
|
|
3982
|
+
let (events, _) = tokio_mpsc::unbounded_channel();
|
|
3983
|
+
let mut request = encoded_open();
|
|
3984
|
+
request.cursor = crate::protocol::CursorMode::Hidden;
|
|
3985
|
+
let error = Session::open(request, events)
|
|
3986
|
+
.await
|
|
3987
|
+
.err()
|
|
3988
|
+
.expect("cannot alter encoded pixels");
|
|
3989
|
+
assert_eq!(error.code, "cursor-unavailable");
|
|
3990
|
+
}
|
|
3991
|
+
|
|
3087
3992
|
#[tokio::test]
|
|
3088
3993
|
async fn an_encoded_session_opens_without_capture_encoder_or_input() {
|
|
3089
3994
|
let (events, _received) = tokio_mpsc::unbounded_channel();
|
|
@@ -3230,6 +4135,7 @@ mod tests {
|
|
|
3230
4135
|
let (events, _received) = tokio_mpsc::unbounded_channel();
|
|
3231
4136
|
let result = Session::open(
|
|
3232
4137
|
OpenRequest {
|
|
4138
|
+
cursor: crate::protocol::CursorMode::Embedded,
|
|
3233
4139
|
source: None,
|
|
3234
4140
|
permissions: vec![Permission::Control],
|
|
3235
4141
|
max_width: 640,
|
|
@@ -3241,6 +4147,7 @@ mod tests {
|
|
|
3241
4147
|
ttl: None,
|
|
3242
4148
|
loopback_tcp: false,
|
|
3243
4149
|
agent_indicator: false,
|
|
4150
|
+
local_frames: true,
|
|
3244
4151
|
},
|
|
3245
4152
|
events,
|
|
3246
4153
|
)
|
|
@@ -3257,6 +4164,11 @@ mod tests {
|
|
|
3257
4164
|
use webrtc::peer_connection::RTCPeerConnectionState as State;
|
|
3258
4165
|
let (events, mut received) = tokio_mpsc::unbounded_channel();
|
|
3259
4166
|
let (inner, recorded) = test_inner(events).await;
|
|
4167
|
+
#[cfg(target_os = "windows")]
|
|
4168
|
+
inner
|
|
4169
|
+
.with_input(|target| target.chord(1, Vec::new(), true))
|
|
4170
|
+
.unwrap();
|
|
4171
|
+
#[cfg(not(target_os = "windows"))]
|
|
3260
4172
|
inner.apply(ControlMessage::Key {
|
|
3261
4173
|
name: Some(String::from("Control")),
|
|
3262
4174
|
character: None,
|
|
@@ -3282,8 +4194,17 @@ mod tests {
|
|
|
3282
4194
|
);
|
|
3283
4195
|
|
|
3284
4196
|
peer_tx.send(PeerEvent::State(State::Failed)).unwrap();
|
|
4197
|
+
let mut transport_lost = false;
|
|
3285
4198
|
tokio::time::timeout(Duration::from_secs(2), async {
|
|
3286
4199
|
while let Some(notice) = received.recv().await {
|
|
4200
|
+
// A lost path is the one revocation a new session can heal.
|
|
4201
|
+
transport_lost |= matches!(
|
|
4202
|
+
notice.event,
|
|
4203
|
+
SessionEvent::Revoked {
|
|
4204
|
+
code: "transport",
|
|
4205
|
+
..
|
|
4206
|
+
}
|
|
4207
|
+
);
|
|
3287
4208
|
if matches!(
|
|
3288
4209
|
notice.event,
|
|
3289
4210
|
SessionEvent::State {
|
|
@@ -3298,11 +4219,158 @@ mod tests {
|
|
|
3298
4219
|
})
|
|
3299
4220
|
.await
|
|
3300
4221
|
.unwrap();
|
|
4222
|
+
assert!(
|
|
4223
|
+
transport_lost,
|
|
4224
|
+
"a failed transport must revoke with code transport"
|
|
4225
|
+
);
|
|
3301
4226
|
assert!(inner.revoked_reason().is_some());
|
|
3302
4227
|
assert!(!inner.pipeline.load(Ordering::SeqCst));
|
|
3303
4228
|
assert!(inner.input.lock().unwrap().is_none());
|
|
3304
4229
|
}
|
|
3305
4230
|
|
|
4231
|
+
#[test]
|
|
4232
|
+
fn a_standing_queue_backs_the_rate_off_to_what_the_path_delivers() {
|
|
4233
|
+
let start = Instant::now();
|
|
4234
|
+
let queued = |ms: u64, delivered_kbps: u32| {
|
|
4235
|
+
Some(PathReport {
|
|
4236
|
+
queue_delay: Duration::from_millis(ms),
|
|
4237
|
+
delivered_kbps,
|
|
4238
|
+
})
|
|
4239
|
+
};
|
|
4240
|
+
let mut rate = RateControl::new(8000);
|
|
4241
|
+
// A burst's worth of queue and no loss moves nothing.
|
|
4242
|
+
assert_eq!(rate.update(&[], queued(15, 3000), start), None);
|
|
4243
|
+
// A standing queue on a 4 Mbps link: 85% of what got through, at once,
|
|
4244
|
+
// with no loss reported at all.
|
|
4245
|
+
assert_eq!(rate.update(&[], queued(80, 4000), start), Some(3400));
|
|
4246
|
+
// Not again until the back-off has had time to show in the feedback.
|
|
4247
|
+
assert_eq!(
|
|
4248
|
+
rate.update(&[], queued(80, 3400), start + Duration::from_millis(100)),
|
|
4249
|
+
None
|
|
4250
|
+
);
|
|
4251
|
+
assert_eq!(
|
|
4252
|
+
rate.update(&[], queued(60, 3400), start + Duration::from_millis(300)),
|
|
4253
|
+
Some(2890)
|
|
4254
|
+
);
|
|
4255
|
+
// A stream that sent little is not taken for a slow link: at most half.
|
|
4256
|
+
assert_eq!(
|
|
4257
|
+
rate.update(&[], queued(60, 100), start + Duration::from_millis(600)),
|
|
4258
|
+
Some(1228)
|
|
4259
|
+
);
|
|
4260
|
+
// Clean loss reports do not raise it while the queue has not drained...
|
|
4261
|
+
assert_eq!(
|
|
4262
|
+
rate.update(&[0], queued(15, 1200), start + Duration::from_secs(2)),
|
|
4263
|
+
None
|
|
4264
|
+
);
|
|
4265
|
+
// ...and do once it has.
|
|
4266
|
+
assert_eq!(
|
|
4267
|
+
rate.update(&[0], queued(2, 1200), start + Duration::from_secs(3)),
|
|
4268
|
+
Some(1390)
|
|
4269
|
+
);
|
|
4270
|
+
// Nothing drops below the floor.
|
|
4271
|
+
let mut rate = RateControl::new(8000);
|
|
4272
|
+
for step in 0..20u64 {
|
|
4273
|
+
rate.update(
|
|
4274
|
+
&[],
|
|
4275
|
+
queued(200, 1),
|
|
4276
|
+
start + DELAY_BACKOFF_EVERY * step as u32,
|
|
4277
|
+
);
|
|
4278
|
+
}
|
|
4279
|
+
assert_eq!(rate.target, 1000);
|
|
4280
|
+
}
|
|
4281
|
+
|
|
4282
|
+
#[test]
|
|
4283
|
+
fn without_transport_feedback_the_rate_follows_loss_alone() {
|
|
4284
|
+
let start = Instant::now();
|
|
4285
|
+
let mut rate = RateControl::new(8000);
|
|
4286
|
+
assert_eq!(rate.update(&[], None, start), None);
|
|
4287
|
+
assert_eq!(rate.update(&[64], None, start), Some(5600));
|
|
4288
|
+
assert_eq!(
|
|
4289
|
+
rate.update(&[64], None, start + Duration::from_millis(500)),
|
|
4290
|
+
None
|
|
4291
|
+
);
|
|
4292
|
+
assert_eq!(
|
|
4293
|
+
rate.update(&[10], None, start + Duration::from_secs(1)),
|
|
4294
|
+
None
|
|
4295
|
+
);
|
|
4296
|
+
assert_eq!(
|
|
4297
|
+
rate.update(&[0], None, start + Duration::from_secs(2)),
|
|
4298
|
+
Some(6112)
|
|
4299
|
+
);
|
|
4300
|
+
}
|
|
4301
|
+
|
|
4302
|
+
#[test]
|
|
4303
|
+
fn a_large_frame_holds_the_next_until_the_link_has_carried_it() {
|
|
4304
|
+
let start = Instant::now();
|
|
4305
|
+
let mut budget = SendBudget::default();
|
|
4306
|
+
// Frames at the target, one per 20 ms at 4000 kbps, never wait, and
|
|
4307
|
+
// neither does the ordinary overshoot of a frame at twice the budget.
|
|
4308
|
+
for n in 0..50u32 {
|
|
4309
|
+
let at = start + Duration::from_millis(20) * n;
|
|
4310
|
+
assert_eq!(budget.wait(at, 4000), Duration::ZERO, "frame {n} waited");
|
|
4311
|
+
budget.sent(at, if n % 5 == 0 { 20_000 } else { 10_000 }, 4000);
|
|
4312
|
+
}
|
|
4313
|
+
// A 100 kB key frame drains in 160 ms (4000 kbps and a quarter): it
|
|
4314
|
+
// goes at once, and the next waits out all of it past the 60 ms burst
|
|
4315
|
+
// allowance.
|
|
4316
|
+
let at = start + Duration::from_secs(1);
|
|
4317
|
+
assert_eq!(budget.wait(at, 4000), Duration::ZERO);
|
|
4318
|
+
budget.sent(at, 100_000, 4000);
|
|
4319
|
+
let wait = budget.wait(at, 4000);
|
|
4320
|
+
assert!(
|
|
4321
|
+
(Duration::from_millis(99)..=Duration::from_millis(101)).contains(&wait),
|
|
4322
|
+
"waited {wait:?}"
|
|
4323
|
+
);
|
|
4324
|
+
assert_eq!(budget.wait(at + wait, 4000), Duration::ZERO);
|
|
4325
|
+
}
|
|
4326
|
+
|
|
4327
|
+
#[test]
|
|
4328
|
+
fn motion_halves_only_when_the_encoder_sets_the_pace_and_does_not_flap() {
|
|
4329
|
+
let interval = Duration::from_millis(16);
|
|
4330
|
+
let (slow, fast) = (Duration::from_millis(30), Duration::from_millis(2));
|
|
4331
|
+
let mut size = MotionSize::default();
|
|
4332
|
+
for _ in 0..MotionSize::SLOW_RUN * 4 {
|
|
4333
|
+
size.coded(slow, false, true, interval);
|
|
4334
|
+
}
|
|
4335
|
+
assert!(
|
|
4336
|
+
!size.half,
|
|
4337
|
+
"slow small frames (typing on a busy host) keep full size"
|
|
4338
|
+
);
|
|
4339
|
+
for _ in 0..MotionSize::SLOW_RUN * 4 {
|
|
4340
|
+
size.coded(slow, true, false, interval);
|
|
4341
|
+
}
|
|
4342
|
+
assert!(
|
|
4343
|
+
!size.half,
|
|
4344
|
+
"a source slower than the cap (30 fps, 30 ms encode) never halves"
|
|
4345
|
+
);
|
|
4346
|
+
for _ in 0..MotionSize::SLOW_RUN - 1 {
|
|
4347
|
+
size.coded(slow, true, true, interval);
|
|
4348
|
+
}
|
|
4349
|
+
size.coded(fast, true, true, interval);
|
|
4350
|
+
assert!(!size.half, "a broken run of slow frames keeps full size");
|
|
4351
|
+
for _ in 0..MotionSize::SLOW_RUN {
|
|
4352
|
+
size.coded(slow, true, true, interval);
|
|
4353
|
+
}
|
|
4354
|
+
assert!(size.half, "a run of slow large frames halves");
|
|
4355
|
+
// The marginal band: half-size frames that are small but still slow,
|
|
4356
|
+
// between large ones. Neither may send it back to full size.
|
|
4357
|
+
for i in 0..MotionSize::SMALL_RUN * 10 {
|
|
4358
|
+
size.coded(slow, i % 7 == 0, true, interval);
|
|
4359
|
+
assert!(size.half, "marginal half-size motion must not flap back");
|
|
4360
|
+
}
|
|
4361
|
+
for _ in 0..MotionSize::SMALL_RUN * 2 {
|
|
4362
|
+
size.coded(fast, true, true, interval);
|
|
4363
|
+
}
|
|
4364
|
+
assert!(size.half, "large half-size frames stay half, however fast");
|
|
4365
|
+
for _ in 0..MotionSize::SMALL_RUN {
|
|
4366
|
+
size.coded(fast, false, true, interval);
|
|
4367
|
+
}
|
|
4368
|
+
assert!(
|
|
4369
|
+
!size.half,
|
|
4370
|
+
"a run of small, fast half-size frames returns to full size"
|
|
4371
|
+
);
|
|
4372
|
+
}
|
|
4373
|
+
|
|
3306
4374
|
#[tokio::test]
|
|
3307
4375
|
async fn the_pipeline_caps_the_rate_codes_the_last_frame_and_refines_a_still_desktop_once() {
|
|
3308
4376
|
let (events, _events_rx) = tokio_mpsc::unbounded_channel();
|
|
@@ -3317,19 +4385,29 @@ mod tests {
|
|
|
3317
4385
|
1000,
|
|
3318
4386
|
);
|
|
3319
4387
|
|
|
3320
|
-
// 40 frames over 200 ms
|
|
4388
|
+
// 40 frames nominally over 200 ms; Windows timers can round each 5 ms
|
|
4389
|
+
// sleep upward, so its cap assertion uses the actual burst duration.
|
|
4390
|
+
#[cfg(target_os = "windows")]
|
|
4391
|
+
let burst_started = Instant::now();
|
|
3321
4392
|
for _ in 0..40 {
|
|
3322
|
-
frame_tx.put(
|
|
3323
|
-
|
|
3324
|
-
|
|
3325
|
-
|
|
3326
|
-
|
|
4393
|
+
frame_tx.put(
|
|
4394
|
+
I420 {
|
|
4395
|
+
width: 64,
|
|
4396
|
+
height: 64,
|
|
4397
|
+
data: vec![128u8; 64 * 64 + 2 * 32 * 32],
|
|
4398
|
+
},
|
|
4399
|
+
Instant::now(),
|
|
4400
|
+
);
|
|
3327
4401
|
tokio::time::sleep(Duration::from_millis(5)).await;
|
|
3328
4402
|
}
|
|
3329
4403
|
let motion = inner.metrics.lock().unwrap().encoded_frames;
|
|
4404
|
+
#[cfg(target_os = "windows")]
|
|
4405
|
+
let cap = (burst_started.elapsed().as_secs_f64() * 20.0).ceil() as u64 + 3;
|
|
4406
|
+
#[cfg(not(target_os = "windows"))]
|
|
4407
|
+
let cap = 7;
|
|
3330
4408
|
assert!(
|
|
3331
|
-
motion <=
|
|
3332
|
-
"the requested rate is a cap, got {motion} of 40"
|
|
4409
|
+
motion <= cap,
|
|
4410
|
+
"the requested rate is a cap, got {motion} of 40 (cap {cap})"
|
|
3333
4411
|
);
|
|
3334
4412
|
assert!(
|
|
3335
4413
|
motion >= 2,
|
|
@@ -3362,5 +4440,74 @@ mod tests {
|
|
|
3362
4440
|
"a still stream still sends"
|
|
3363
4441
|
);
|
|
3364
4442
|
assert_eq!((later.key_frames, later.refined_frames), (1, 1));
|
|
4443
|
+
assert_eq!((later.codec, later.encoder), ("vp9", "libvpx"));
|
|
4444
|
+
}
|
|
4445
|
+
|
|
4446
|
+
#[tokio::test]
|
|
4447
|
+
async fn a_receiver_that_asks_for_h264_is_sent_h264_from_the_first_frame() {
|
|
4448
|
+
let Some(info) = tokio::task::spawn_blocking(crate::h264::probe)
|
|
4449
|
+
.await
|
|
4450
|
+
.unwrap()
|
|
4451
|
+
else {
|
|
4452
|
+
eprintln!("skipped: no H.264 encoder starts on this machine");
|
|
4453
|
+
return;
|
|
4454
|
+
};
|
|
4455
|
+
let (peer, _answer) =
|
|
4456
|
+
crate::peer::tests::answered(vec![crate::peer::tests::h264_receiver_codec()]).await;
|
|
4457
|
+
let (events, _events_rx) = tokio_mpsc::unbounded_channel();
|
|
4458
|
+
let (inner, _recorded) = test_inner_with(events, peer, 320, 240);
|
|
4459
|
+
// The answer is real, so ICE runs and, with no candidates, never
|
|
4460
|
+
// connects; each state it reports would take back `assume_connected`.
|
|
4461
|
+
let pinned = current_peer(&inner).expect("the test's peer");
|
|
4462
|
+
let pin = tokio::spawn(async move {
|
|
4463
|
+
loop {
|
|
4464
|
+
pinned.assume_connected();
|
|
4465
|
+
tokio::time::sleep(Duration::from_millis(5)).await;
|
|
4466
|
+
}
|
|
4467
|
+
});
|
|
4468
|
+
let (frame_tx, frame_rx) = latest_frame();
|
|
4469
|
+
spawn_pipeline(
|
|
4470
|
+
&inner,
|
|
4471
|
+
current_peer(&inner).expect("the test's peer"),
|
|
4472
|
+
frame_rx,
|
|
4473
|
+
Arc::new(AtomicBool::new(true)),
|
|
4474
|
+
20,
|
|
4475
|
+
1000,
|
|
4476
|
+
);
|
|
4477
|
+
for shade in 0..10u8 {
|
|
4478
|
+
frame_tx.put(
|
|
4479
|
+
I420 {
|
|
4480
|
+
width: 320,
|
|
4481
|
+
height: 240,
|
|
4482
|
+
data: vec![shade * 20; 320 * 240 * 3 / 2],
|
|
4483
|
+
},
|
|
4484
|
+
Instant::now(),
|
|
4485
|
+
);
|
|
4486
|
+
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
4487
|
+
}
|
|
4488
|
+
// Opening a hardware session can take a good part of a second, more
|
|
4489
|
+
// with other tests holding the GPU: wait for the still to be refined.
|
|
4490
|
+
let deadline = Instant::now() + Duration::from_secs(10);
|
|
4491
|
+
while inner.metrics.lock().unwrap().refined_frames == 0 && Instant::now() < deadline {
|
|
4492
|
+
tokio::time::sleep(Duration::from_millis(50)).await;
|
|
4493
|
+
}
|
|
4494
|
+
let metrics = inner.metrics.lock().unwrap().clone();
|
|
4495
|
+
assert_eq!((metrics.codec, metrics.encoder), ("h264", info.encoder));
|
|
4496
|
+
assert!(metrics.encoded_frames >= 2, "{metrics:?}");
|
|
4497
|
+
assert_eq!(
|
|
4498
|
+
metrics.key_frames, 1,
|
|
4499
|
+
"one IDR, then inter frames: {metrics:?}"
|
|
4500
|
+
);
|
|
4501
|
+
assert_eq!(metrics.refined_frames, 1, "a still desktop is refined once");
|
|
4502
|
+
assert_eq!(
|
|
4503
|
+
metrics.halved_frames, 0,
|
|
4504
|
+
"H.264 is always coded at full size"
|
|
4505
|
+
);
|
|
4506
|
+
pin.abort();
|
|
4507
|
+
inner.close("done", "closed").await;
|
|
4508
|
+
assert!(
|
|
4509
|
+
lock(&inner.encoder).is_none(),
|
|
4510
|
+
"closing releases the encoder"
|
|
4511
|
+
);
|
|
3365
4512
|
}
|
|
3366
4513
|
}
|