@desklink/host 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/LICENSE +202 -0
  2. package/NOTICE +44 -0
  3. package/README.md +266 -0
  4. package/bin/desklink-host.mjs +113 -0
  5. package/dist/bridge.d.ts +57 -0
  6. package/dist/bridge.d.ts.map +1 -0
  7. package/dist/bridge.js +208 -0
  8. package/dist/bridge.js.map +1 -0
  9. package/dist/engineProcess.d.ts +82 -0
  10. package/dist/engineProcess.d.ts.map +1 -0
  11. package/dist/engineProcess.js +243 -0
  12. package/dist/engineProcess.js.map +1 -0
  13. package/dist/index.d.ts +12 -0
  14. package/dist/index.d.ts.map +1 -0
  15. package/dist/index.js +12 -0
  16. package/dist/index.js.map +1 -0
  17. package/dist/protocol.d.ts +169 -0
  18. package/dist/protocol.d.ts.map +1 -0
  19. package/dist/protocol.js +18 -0
  20. package/dist/protocol.js.map +1 -0
  21. package/dist/resolveEngine.d.ts +34 -0
  22. package/dist/resolveEngine.d.ts.map +1 -0
  23. package/dist/resolveEngine.js +103 -0
  24. package/dist/resolveEngine.js.map +1 -0
  25. package/docs/BOUNDARY.md +145 -0
  26. package/docs/PROTOCOL.md +363 -0
  27. package/engine/Cargo.lock +3077 -0
  28. package/engine/Cargo.toml +37 -0
  29. package/engine/build.rs +107 -0
  30. package/engine/native/inputtino_shim.cpp +11 -0
  31. package/engine/native/vpx_shim.c +184 -0
  32. package/engine/native/wrapper.h +7 -0
  33. package/engine/src/capture.rs +510 -0
  34. package/engine/src/clipboard.rs +132 -0
  35. package/engine/src/convert.rs +299 -0
  36. package/engine/src/encoder.rs +204 -0
  37. package/engine/src/input.rs +355 -0
  38. package/engine/src/keymap.rs +338 -0
  39. package/engine/src/main.rs +614 -0
  40. package/engine/src/peer.rs +775 -0
  41. package/engine/src/portal.rs +115 -0
  42. package/engine/src/protocol.rs +331 -0
  43. package/engine/src/session.rs +2159 -0
  44. package/engine/src/x11.rs +244 -0
  45. package/engine/vendor/inputtino/.clang-format +166 -0
  46. package/engine/vendor/inputtino/.clang-tidy +30 -0
  47. package/engine/vendor/inputtino/.dockerignore +11 -0
  48. package/engine/vendor/inputtino/.github/workflows/mirror-repo.yml +14 -0
  49. package/engine/vendor/inputtino/.github/workflows/python-lint-format-test-typecheck.yml +124 -0
  50. package/engine/vendor/inputtino/.github/workflows/python-test-install.yaml +49 -0
  51. package/engine/vendor/inputtino/.github/workflows/rust.yml +40 -0
  52. package/engine/vendor/inputtino/CMakeLists.txt +187 -0
  53. package/engine/vendor/inputtino/LICENSE +21 -0
  54. package/engine/vendor/inputtino/README.md +110 -0
  55. package/engine/vendor/inputtino/bindings/python/.devcontainer/Dockerfile +23 -0
  56. package/engine/vendor/inputtino/bindings/python/.devcontainer/devcontainer.json +28 -0
  57. package/engine/vendor/inputtino/bindings/python/.python-version +1 -0
  58. package/engine/vendor/inputtino/bindings/python/CMakeLists.txt +23 -0
  59. package/engine/vendor/inputtino/bindings/python/CPPLINT.cfg +2 -0
  60. package/engine/vendor/inputtino/bindings/python/Makefile +49 -0
  61. package/engine/vendor/inputtino/bindings/python/README.md +215 -0
  62. package/engine/vendor/inputtino/bindings/python/pyproject.toml +178 -0
  63. package/engine/vendor/inputtino/bindings/python/src/inputtino/__init__.py +43 -0
  64. package/engine/vendor/inputtino/bindings/python/src/inputtino/_core.pyi +533 -0
  65. package/engine/vendor/inputtino/bindings/python/src/inputtino/base.py +63 -0
  66. package/engine/vendor/inputtino/bindings/python/src/inputtino/joypad.py +248 -0
  67. package/engine/vendor/inputtino/bindings/python/src/inputtino/keyboard.py +256 -0
  68. package/engine/vendor/inputtino/bindings/python/src/inputtino/mouse.py +106 -0
  69. package/engine/vendor/inputtino/bindings/python/src/inputtino/pentablet.py +73 -0
  70. package/engine/vendor/inputtino/bindings/python/src/inputtino/py.typed +0 -0
  71. package/engine/vendor/inputtino/bindings/python/src/inputtino/touchscreen.py +61 -0
  72. package/engine/vendor/inputtino/bindings/python/src/inputtino/trackpad.py +74 -0
  73. package/engine/vendor/inputtino/bindings/python/src/main.cpp +217 -0
  74. package/engine/vendor/inputtino/bindings/python/tests/__init__.py +0 -0
  75. package/engine/vendor/inputtino/bindings/python/tests/helpers.py +3 -0
  76. package/engine/vendor/inputtino/bindings/python/tests/test_base.py +60 -0
  77. package/engine/vendor/inputtino/bindings/python/tests/test_inputtino.py +5 -0
  78. package/engine/vendor/inputtino/bindings/python/tests/test_joypad.py +332 -0
  79. package/engine/vendor/inputtino/bindings/python/tests/test_keyboard.py +164 -0
  80. package/engine/vendor/inputtino/bindings/python/tests/test_mouse.py +185 -0
  81. package/engine/vendor/inputtino/bindings/python/tests/test_pentablet.py +198 -0
  82. package/engine/vendor/inputtino/bindings/python/tests/test_touchscreen.py +144 -0
  83. package/engine/vendor/inputtino/bindings/python/tests/test_trackpad.py +146 -0
  84. package/engine/vendor/inputtino/bindings/python/uv.lock +390 -0
  85. package/engine/vendor/inputtino/bindings/rust/Cargo.toml +6 -0
  86. package/engine/vendor/inputtino/bindings/rust/inputtino/.dockerignore +1 -0
  87. package/engine/vendor/inputtino/bindings/rust/inputtino/Cargo.toml +18 -0
  88. package/engine/vendor/inputtino/bindings/rust/inputtino/examples/keyboard.rs +33 -0
  89. package/engine/vendor/inputtino/bindings/rust/inputtino/examples/mouse.rs +17 -0
  90. package/engine/vendor/inputtino/bindings/rust/inputtino/src/common.rs +116 -0
  91. package/engine/vendor/inputtino/bindings/rust/inputtino/src/error.rs +17 -0
  92. package/engine/vendor/inputtino/bindings/rust/inputtino/src/joypad.rs +74 -0
  93. package/engine/vendor/inputtino/bindings/rust/inputtino/src/joypad_nintendo.rs +135 -0
  94. package/engine/vendor/inputtino/bindings/rust/inputtino/src/joypad_ps5.rs +280 -0
  95. package/engine/vendor/inputtino/bindings/rust/inputtino/src/joypad_xbox.rs +134 -0
  96. package/engine/vendor/inputtino/bindings/rust/inputtino/src/keyboard.rs +76 -0
  97. package/engine/vendor/inputtino/bindings/rust/inputtino/src/lib.rs +32 -0
  98. package/engine/vendor/inputtino/bindings/rust/inputtino/src/mouse.rs +178 -0
  99. package/engine/vendor/inputtino/bindings/rust/inputtino/tests/common.rs +38 -0
  100. package/engine/vendor/inputtino/bindings/rust/inputtino/tests/joypads.rs +150 -0
  101. package/engine/vendor/inputtino/bindings/rust/inputtino/tests/keyboard.rs +73 -0
  102. package/engine/vendor/inputtino/bindings/rust/inputtino/tests/mouse.rs +139 -0
  103. package/engine/vendor/inputtino/bindings/rust/inputtino-sys/.dockerignore +1 -0
  104. package/engine/vendor/inputtino/bindings/rust/inputtino-sys/Cargo.toml +12 -0
  105. package/engine/vendor/inputtino/bindings/rust/inputtino-sys/build.rs +48 -0
  106. package/engine/vendor/inputtino/bindings/rust/inputtino-sys/src/lib.rs +5 -0
  107. package/engine/vendor/inputtino/bindings/rust/inputtino-sys/wrapper.h +1 -0
  108. package/engine/vendor/inputtino/cmake/utils.cmake +9 -0
  109. package/engine/vendor/inputtino/docker/Dockerfile +70 -0
  110. package/engine/vendor/inputtino/include/inputtino/input.h +324 -0
  111. package/engine/vendor/inputtino/include/inputtino/input.hpp +466 -0
  112. package/engine/vendor/inputtino/include/inputtino/result.hpp +37 -0
  113. package/engine/vendor/inputtino/share/pkgconfig/libinputtino.pc.in +17 -0
  114. package/engine/vendor/inputtino/src/c-bindings/helpers.hpp +19 -0
  115. package/engine/vendor/inputtino/src/c-bindings/joypad_ps5.cpp +115 -0
  116. package/engine/vendor/inputtino/src/c-bindings/joypad_switch.cpp +63 -0
  117. package/engine/vendor/inputtino/src/c-bindings/joypad_xone.cpp +62 -0
  118. package/engine/vendor/inputtino/src/c-bindings/keyboard.cpp +42 -0
  119. package/engine/vendor/inputtino/src/c-bindings/mouse.cpp +67 -0
  120. package/engine/vendor/inputtino/src/c-bindings/pen_tablet.cpp +50 -0
  121. package/engine/vendor/inputtino/src/c-bindings/touchscreen.cpp +43 -0
  122. package/engine/vendor/inputtino/src/c-bindings/trackpad.cpp +49 -0
  123. package/engine/vendor/inputtino/src/uhid/README.adoc +63 -0
  124. package/engine/vendor/inputtino/src/uhid/include/crc32.hpp +38 -0
  125. package/engine/vendor/inputtino/src/uhid/include/uhid/protected_types.hpp +36 -0
  126. package/engine/vendor/inputtino/src/uhid/include/uhid/ps5.hpp +550 -0
  127. package/engine/vendor/inputtino/src/uhid/include/uhid/uhid.hpp +168 -0
  128. package/engine/vendor/inputtino/src/uhid/joypad_ps5.cpp +580 -0
  129. package/engine/vendor/inputtino/src/uinput/include/inputtino/keyboard.hpp +84 -0
  130. package/engine/vendor/inputtino/src/uinput/include/inputtino/protected_ps5_types.hpp +7 -0
  131. package/engine/vendor/inputtino/src/uinput/include/inputtino/protected_types.hpp +110 -0
  132. package/engine/vendor/inputtino/src/uinput/joypad_nintendo.cpp +190 -0
  133. package/engine/vendor/inputtino/src/uinput/joypad_ps.cpp +203 -0
  134. package/engine/vendor/inputtino/src/uinput/joypad_utils.hpp +246 -0
  135. package/engine/vendor/inputtino/src/uinput/joypad_xbox.cpp +190 -0
  136. package/engine/vendor/inputtino/src/uinput/keyboard.cpp +120 -0
  137. package/engine/vendor/inputtino/src/uinput/mouse.cpp +199 -0
  138. package/engine/vendor/inputtino/src/uinput/pentablet.cpp +162 -0
  139. package/engine/vendor/inputtino/src/uinput/touchscreen.cpp +140 -0
  140. package/engine/vendor/inputtino/src/uinput/trackpad.cpp +192 -0
  141. package/examples/reference-client.html +428 -0
  142. package/package.json +38 -0
@@ -0,0 +1,2159 @@
1
+ //! One authorized desktop session: capture in, VP9 out, control channel in,
2
+ //! desktop input out, and a bounded teardown.
3
+ //!
4
+ //! The engine trusts the consumer's local permission decision and enforces the
5
+ //! resulting scope. It has no second identity system, no account and no notion
6
+ //! of which application asked.
7
+
8
+ use crate::capture::{self, Capture};
9
+ use crate::clipboard;
10
+ use crate::convert::{fit, I420};
11
+ use crate::encoder::Encoder;
12
+ use crate::input::{Button, HeldState, InputDevices};
13
+ use crate::keymap::{self, Layout};
14
+ use crate::peer::{PeerEvent, TransportOptions, VideoPeer};
15
+ use crate::portal::{self, SelectedSource};
16
+ use crate::protocol::{ControlMessage, ControlReply, Permission, PointerPhase, SourceRequest};
17
+ use crate::x11::X11Desktop;
18
+ use anyhow::Result;
19
+ use std::sync::atomic::{AtomicBool, Ordering};
20
+ use std::sync::{Arc, Condvar, Mutex};
21
+ use std::time::{Duration, Instant};
22
+ use tokio::sync::mpsc as tokio_mpsc;
23
+
24
+ /// How long a clipboard operation may take before it is reported as unanswered.
25
+ ///
26
+ /// A compositor with no clipboard service must produce an error the user can
27
+ /// read, not a request that never returns.
28
+ const CLIPBOARD_TIMEOUT: Duration = Duration::from_secs(3);
29
+
30
+ /// Run a clipboard operation off the async runtime and bounded.
31
+ ///
32
+ /// Both halves matter. The clipboard backends block, and running one inline on
33
+ /// a runtime thread stalls the session's own input and lifecycle events; and a
34
+ /// desktop whose clipboard cannot answer must fail visibly rather than hang.
35
+ async fn clipboard_task<T: Send + 'static>(
36
+ operation: impl FnOnce() -> std::result::Result<T, String> + Send + 'static,
37
+ ) -> std::result::Result<T, String> {
38
+ match tokio::time::timeout(CLIPBOARD_TIMEOUT, tokio::task::spawn_blocking(operation)).await {
39
+ Ok(Ok(result)) => result,
40
+ Ok(Err(join)) => Err(format!("the clipboard operation failed: {join}")),
41
+ Err(_) => Err(String::from("the desktop clipboard did not answer")),
42
+ }
43
+ }
44
+
45
+ /// How the capture → encode → send path is doing, reported to the consumer.
46
+ #[derive(Debug, Default, Clone, serde::Serialize)]
47
+ pub struct Metrics {
48
+ pub captured_frames: u64,
49
+ pub dropped_frames: u64,
50
+ pub encoded_frames: u64,
51
+ pub encoded_bytes: u64,
52
+ /// Key frames sent: the first, and one per receiver request.
53
+ pub key_frames: u64,
54
+ /// Refinement passes: a still desktop re-coded once, sharp.
55
+ pub refined_frames: u64,
56
+ /// Time spent in the encoder, in microseconds, over every encoded frame.
57
+ pub encode_micros: u64,
58
+ /// The current rate target, after any loss back-off.
59
+ pub target_kbps: u32,
60
+ pub input_applied: u64,
61
+ pub input_rejected: u64,
62
+ }
63
+
64
+ /// Which desktop is being captured.
65
+ ///
66
+ /// Dropping either variant stops its capture: the portal variant drops the
67
+ /// PipeWire stream, which is what releases the compositor's consent, and the X11
68
+ /// variant stops reading the server.
69
+ enum FrameSource {
70
+ Portal(Capture),
71
+ X11(X11Capture),
72
+ }
73
+
74
+ /// Root-window capture on an X display this engine was pointed at.
75
+ struct X11Capture {
76
+ stop: Arc<AtomicBool>,
77
+ thread: Option<std::thread::JoinHandle<()>>,
78
+ }
79
+
80
+ impl Drop for X11Capture {
81
+ fn drop(&mut self) {
82
+ self.stop.store(true, Ordering::SeqCst);
83
+ if let Some(thread) = self.thread.take() {
84
+ let _ = thread.join();
85
+ }
86
+ }
87
+ }
88
+
89
+ /// What a chosen capture backend gives the session.
90
+ struct Selected {
91
+ source: SelectedSource,
92
+ capture: FrameSource,
93
+ /// Present only on the X11 path, where the same connection applies input.
94
+ x11: Option<Arc<Mutex<X11Desktop>>>,
95
+ }
96
+
97
+ /// Open an X display, start reading its root window, and hand back the same
98
+ /// shape the portal path produces.
99
+ fn select_x11(
100
+ display: Option<&str>,
101
+ max_width: usize,
102
+ max_height: usize,
103
+ max_fps: u32,
104
+ metrics: Arc<Mutex<Metrics>>,
105
+ sink: capture::FrameSink,
106
+ ) -> Result<Selected> {
107
+ let desktop = Arc::new(Mutex::new(X11Desktop::connect(display)?));
108
+ let (width, height) = {
109
+ let desktop = lock(&desktop);
110
+ desktop.screen_size()
111
+ };
112
+ let source = SelectedSource {
113
+ node_id: 0,
114
+ width,
115
+ height,
116
+ position: Some((0, 0)),
117
+ source_type: Some(String::from("x11-root")),
118
+ origin_x: 0,
119
+ origin_y: 0,
120
+ };
121
+
122
+ let stop = Arc::new(AtomicBool::new(false));
123
+ let captured = metrics.clone();
124
+ let thread = {
125
+ let desktop = desktop.clone();
126
+ let stop = stop.clone();
127
+ std::thread::Builder::new()
128
+ .name("desklink-x11-capture".into())
129
+ .spawn(move || {
130
+ let interval = frame_interval(max_fps);
131
+ let mut sequence = 0u64;
132
+ // An X server has no damage signal here, so an unchanged screen
133
+ // is recognised by its pixels: handing it on would keep the
134
+ // encoder busy and a still desktop would never be refined.
135
+ let mut last_hash = None;
136
+ while !stop.load(Ordering::SeqCst) {
137
+ let started = Instant::now();
138
+ let frame = {
139
+ let mut desktop = lock(&desktop);
140
+ desktop.capture(max_width, max_height)
141
+ };
142
+ match frame {
143
+ Ok(frame) => {
144
+ use std::hash::{Hash, Hasher};
145
+ let mut hasher = std::collections::hash_map::DefaultHasher::new();
146
+ frame.data.hash(&mut hasher);
147
+ let hash = hasher.finish();
148
+ if last_hash != Some(hash) {
149
+ last_hash = Some(hash);
150
+ sink(frame, sequence);
151
+ sequence += 1;
152
+ }
153
+ }
154
+ Err(error) => {
155
+ if let Ok(mut m) = captured.lock() {
156
+ m.dropped_frames += 1;
157
+ }
158
+ // A display that has gone away is not recoverable by
159
+ // retrying, so stop rather than spin on the error.
160
+ if error.to_string().contains("the X11 connection dropped") {
161
+ break;
162
+ }
163
+ }
164
+ }
165
+ let elapsed = started.elapsed();
166
+ if elapsed < interval {
167
+ std::thread::sleep(interval - elapsed);
168
+ }
169
+ }
170
+ })
171
+ .ok()
172
+ };
173
+
174
+ Ok(Selected {
175
+ source,
176
+ capture: FrameSource::X11(X11Capture { stop, thread }),
177
+ x11: Some(desktop),
178
+ })
179
+ }
180
+
181
+ /// Where input goes. One controller, one applier, one held-state record.
182
+ struct InputTarget {
183
+ applier: Applier,
184
+ held: HeldState,
185
+ explicit_modifiers: Vec<i16>,
186
+ chord_modifiers: Vec<i16>,
187
+ chord_keys: Vec<(i16, Vec<i16>)>,
188
+ /// Fractions of a detent an X server, which only knows whole wheel clicks,
189
+ /// has not been sent yet.
190
+ wheel_rest: (f64, f64),
191
+ }
192
+
193
+ enum Applier {
194
+ Uinput(InputDevices),
195
+ X11(Arc<Mutex<X11Desktop>>),
196
+ #[cfg(test)]
197
+ Recording(Arc<Mutex<Vec<(i16, bool)>>>),
198
+ }
199
+
200
+ impl InputTarget {
201
+ fn move_absolute(&mut self, x: i64, y: i64) -> Result<()> {
202
+ match &mut self.applier {
203
+ Applier::Uinput(devices) => {
204
+ devices.move_absolute(x, y);
205
+ Ok(())
206
+ }
207
+ Applier::X11(desktop) => lock(desktop).move_pointer(x, y),
208
+ #[cfg(test)]
209
+ Applier::Recording(_) => Ok(()),
210
+ }
211
+ }
212
+
213
+ fn button(&mut self, button: Button, down: bool) -> Result<()> {
214
+ self.held.button(button, down);
215
+ match &mut self.applier {
216
+ Applier::Uinput(devices) => {
217
+ devices.button(button, down);
218
+ Ok(())
219
+ }
220
+ Applier::X11(desktop) => lock(desktop).button(x11_button(button), down),
221
+ #[cfg(test)]
222
+ Applier::Recording(log) => {
223
+ if let Ok(mut log) = log.lock() {
224
+ log.push((-1, down));
225
+ }
226
+ Ok(())
227
+ }
228
+ }
229
+ }
230
+
231
+ fn scroll(&mut self, dx: f64, dy: f64) -> Result<()> {
232
+ match &mut self.applier {
233
+ Applier::Uinput(devices) => {
234
+ devices.scroll(dx, dy);
235
+ Ok(())
236
+ }
237
+ Applier::X11(desktop) => {
238
+ let (rest_x, rest_y) = (self.wheel_rest.0 + dx, self.wheel_rest.1 + dy);
239
+ let (whole_x, whole_y) = (rest_x.trunc(), rest_y.trunc());
240
+ self.wheel_rest = (rest_x - whole_x, rest_y - whole_y);
241
+ lock(desktop).scroll(whole_x as i64, whole_y as i64)
242
+ }
243
+ #[cfg(test)]
244
+ Applier::Recording(_) => Ok(()),
245
+ }
246
+ }
247
+
248
+ fn check_keys(&self, codes: impl IntoIterator<Item = i16>) -> Result<()> {
249
+ if matches!(&self.applier, Applier::Uinput(_)) {
250
+ for code in codes {
251
+ crate::input::native_keycode(code)?;
252
+ }
253
+ }
254
+ Ok(())
255
+ }
256
+
257
+ fn key(&mut self, code: i16, down: bool) -> Result<()> {
258
+ match &mut self.applier {
259
+ Applier::Uinput(devices) => devices.key(code, down),
260
+ Applier::X11(desktop) => lock(desktop).key(code, down),
261
+ #[cfg(test)]
262
+ Applier::Recording(log) => {
263
+ if let Ok(mut log) = log.lock() {
264
+ log.push((code, down));
265
+ }
266
+ Ok(())
267
+ }
268
+ }?;
269
+ self.held.key(code, down);
270
+ Ok(())
271
+ }
272
+
273
+ fn modifier(&mut self, code: i16, down: bool, explicit: bool) -> Result<()> {
274
+ let own = if explicit {
275
+ &self.explicit_modifiers
276
+ } else {
277
+ &self.chord_modifiers
278
+ };
279
+ let other = if explicit {
280
+ &self.chord_modifiers
281
+ } else {
282
+ &self.explicit_modifiers
283
+ };
284
+ if down {
285
+ if explicit && own.contains(&code) {
286
+ return Ok(());
287
+ }
288
+ if !own.contains(&code) && !other.contains(&code) {
289
+ self.key(code, true)?;
290
+ }
291
+ } else {
292
+ let Some(index) = own.iter().position(|held| *held == code) else {
293
+ return Ok(());
294
+ };
295
+ if own.iter().filter(|held| **held == code).count() == 1 && !other.contains(&code) {
296
+ self.key(code, false)?;
297
+ }
298
+ let own = if explicit {
299
+ &mut self.explicit_modifiers
300
+ } else {
301
+ &mut self.chord_modifiers
302
+ };
303
+ own.remove(index);
304
+ return Ok(());
305
+ }
306
+ let own = if explicit {
307
+ &mut self.explicit_modifiers
308
+ } else {
309
+ &mut self.chord_modifiers
310
+ };
311
+ own.push(code);
312
+ Ok(())
313
+ }
314
+
315
+ fn chord(&mut self, code: i16, modifiers: Vec<i16>, down: bool) -> Result<()> {
316
+ if down {
317
+ if self.chord_keys.iter().any(|(held, _)| *held == code) {
318
+ return Ok(());
319
+ }
320
+ self.check_keys(modifiers.iter().copied().chain(std::iter::once(code)))?;
321
+ for modifier in &modifiers {
322
+ self.modifier(*modifier, true, false)?;
323
+ }
324
+ self.key(code, true)?;
325
+ self.chord_keys.push((code, modifiers));
326
+ } else if let Some(index) = self.chord_keys.iter().position(|(held, _)| *held == code) {
327
+ self.key(code, false)?;
328
+ let (_, modifiers) = self.chord_keys.remove(index);
329
+ for modifier in modifiers.iter().rev() {
330
+ self.modifier(*modifier, false, false)?;
331
+ }
332
+ }
333
+ Ok(())
334
+ }
335
+
336
+ /// Release exactly what this session pressed, once, buttons before keys.
337
+ fn release_all(&mut self) -> Result<()> {
338
+ self.explicit_modifiers.clear();
339
+ self.chord_modifiers.clear();
340
+ self.chord_keys.clear();
341
+ let (buttons, keys) = self.held.release_plan();
342
+ for button in buttons {
343
+ match &mut self.applier {
344
+ Applier::Uinput(devices) => devices.button(button, false),
345
+ Applier::X11(desktop) => lock(desktop).button(x11_button(button), false)?,
346
+ #[cfg(test)]
347
+ Applier::Recording(log) => {
348
+ if let Ok(mut log) = log.lock() {
349
+ log.push((-1, false));
350
+ }
351
+ }
352
+ }
353
+ }
354
+ for code in keys {
355
+ match &mut self.applier {
356
+ Applier::Uinput(devices) => devices.key(code, false)?,
357
+ Applier::X11(desktop) => lock(desktop).key(code, false)?,
358
+ #[cfg(test)]
359
+ Applier::Recording(log) => {
360
+ if let Ok(mut log) = log.lock() {
361
+ log.push((code, false));
362
+ }
363
+ }
364
+ }
365
+ }
366
+ Ok(())
367
+ }
368
+ }
369
+
370
+ fn x11_button(button: Button) -> u8 {
371
+ match button {
372
+ Button::Left => crate::x11::button::LEFT,
373
+ Button::Middle => crate::x11::button::MIDDLE,
374
+ Button::Right => crate::x11::button::RIGHT,
375
+ }
376
+ }
377
+
378
+ fn lock<T>(mutex: &Arc<Mutex<T>>) -> std::sync::MutexGuard<'_, T> {
379
+ // A poisoned lock means a previous input call panicked while applying; the
380
+ // desktop is then in unknown state, so the session is closed rather than
381
+ // continuing to drive it.
382
+ mutex
383
+ .lock()
384
+ .unwrap_or_else(|poisoned| poisoned.into_inner())
385
+ }
386
+
387
+ /// Why a session could not do what was asked, in the protocol's own vocabulary.
388
+ pub struct SessionError {
389
+ pub code: &'static str,
390
+ pub message: String,
391
+ }
392
+
393
+ impl SessionError {
394
+ fn new(code: &'static str, message: impl Into<String>) -> Self {
395
+ Self {
396
+ code,
397
+ message: message.into(),
398
+ }
399
+ }
400
+ }
401
+
402
+ pub struct OpenRequest {
403
+ /** Which desktop to capture; absent means the portal. */
404
+ pub source: Option<SourceRequest>,
405
+ pub permissions: Vec<Permission>,
406
+ pub max_width: usize,
407
+ pub max_height: usize,
408
+ pub bitrate_kbps: u32,
409
+ pub max_fps: u32,
410
+ pub ice_servers: Vec<(String, Option<String>, Option<String>)>,
411
+ pub restore_token: Option<String>,
412
+ pub ttl: Option<Duration>,
413
+ /// Offer a passive ICE-TCP candidate on the loopback, for a forwarded client.
414
+ pub loopback_tcp: bool,
415
+ }
416
+
417
+ /// Events a session raises for its consumer.
418
+ #[derive(Debug)]
419
+ pub enum SessionEvent {
420
+ Description {
421
+ generation: u64,
422
+ sdp: String,
423
+ },
424
+ Candidate {
425
+ generation: u64,
426
+ candidate: String,
427
+ sdp_mid: Option<String>,
428
+ sdp_m_line_index: Option<u16>,
429
+ },
430
+ State {
431
+ capture: &'static str,
432
+ transport: String,
433
+ first_frame: bool,
434
+ },
435
+ RestoreToken(String),
436
+ Revoked {
437
+ reason: String,
438
+ },
439
+ }
440
+
441
+ /// One notification plus the session it belongs to. The engine serves one
442
+ /// session at a time, but its queue outlives a session, so a consumer needs the
443
+ /// identity to attribute an event to the record that asked for it.
444
+ #[derive(Debug)]
445
+ pub struct Notice {
446
+ pub session_id: String,
447
+ pub event: SessionEvent,
448
+ }
449
+
450
+ /// Everything a session's background tasks need, shared rather than borrowed so
451
+ /// the consumer can own the `Session` handle while the pipeline runs.
452
+ struct Inner {
453
+ id: String,
454
+ generation: u64,
455
+ permissions: Vec<Permission>,
456
+ source: SelectedSource,
457
+ geometry: serde_json::Value,
458
+ metrics: Arc<Mutex<Metrics>>,
459
+ peer: Arc<VideoPeer>,
460
+ encoder: Mutex<Encoder>,
461
+ input: Mutex<Option<InputTarget>>,
462
+ capture: Mutex<Option<FrameSource>>,
463
+ layout: Mutex<Layout>,
464
+ last_seq: Mutex<u64>,
465
+ control_open: AtomicBool,
466
+ closed: AtomicBool,
467
+ /// Set false to stop the encode loop; owned here so closing is reachable
468
+ /// from the lease as well as from the consumer.
469
+ pipeline: Arc<AtomicBool>,
470
+ events: tokio_mpsc::UnboundedSender<Notice>,
471
+ }
472
+
473
+ pub struct Session {
474
+ inner: Arc<Inner>,
475
+ }
476
+
477
+ fn wayland_clipboard_available() -> bool {
478
+ std::env::var("XDG_SESSION_TYPE").as_deref() == Ok("wayland")
479
+ && std::env::var_os("WAYLAND_DISPLAY").is_some()
480
+ }
481
+
482
+ /// What the engine can actually do on this machine right now.
483
+ pub fn capabilities() -> serde_json::Value {
484
+ let session_kind = if std::env::var_os("WAYLAND_DISPLAY").is_some() {
485
+ "wayland"
486
+ } else if std::env::var_os("DISPLAY").is_some() {
487
+ "x11"
488
+ } else {
489
+ "none"
490
+ };
491
+ // Clipboard transfer needs Wayland; writes also need a selection server
492
+ // that can outlive this engine. Do not offer it when that tool is absent.
493
+ let clipboard = wayland_clipboard_available();
494
+ let x11 = crate::x11::X11Desktop::connect(None)
495
+ .map(|desktop| serde_json::json!([desktop.screen_size().0, desktop.screen_size().1]))
496
+ .unwrap_or(serde_json::Value::Null);
497
+ let (input, grant_state, unavailable) = match crate::input::probe() {
498
+ Ok(()) => (serde_json::json!(true), "granted", serde_json::Value::Null),
499
+ Err(unavailable) => (
500
+ serde_json::json!(false),
501
+ "missing-device-access",
502
+ serde_json::json!({
503
+ "reason": unavailable.reason,
504
+ "remedy": unavailable.remedy,
505
+ }),
506
+ ),
507
+ };
508
+ serde_json::json!({
509
+ "protocol": crate::protocol::PROTOCOL_VERSION,
510
+ "engine": format!("desklink-host/{}", env!("CARGO_PKG_VERSION")),
511
+ "platform": "linux",
512
+ "session": { "kind": session_kind },
513
+ "x11": { "available": x11.is_array(), "size": x11 },
514
+ "capture": {
515
+ "mechanism": "portal-screencast+pipewire",
516
+ // Backends this build has, not a claim that both are usable here; the
517
+ // portal is preferred and needs the user's consent, the X display is
518
+ // available whenever DISPLAY points at a server.
519
+ "backends": ["portal-screencast+pipewire", "x11-root"],
520
+ "formats": ["bgrx", "bgra", "rgbx", "rgba"],
521
+ "cursor": "embedded",
522
+ "audio": false,
523
+ },
524
+ "encode": { "codecs": ["vp9"], "hardware": false },
525
+ "input": {
526
+ "mechanism": "inputtino/uinput",
527
+ "pointer": input,
528
+ "wheel": input,
529
+ "keyboard": input,
530
+ "text": ["layout-reachable"],
531
+ "layout": keymap::LayoutNames::from_environment().identity(),
532
+ "unavailable_reason": unavailable,
533
+ "grant": grant_state,
534
+ },
535
+ "clipboard": { "read": clipboard, "write": clipboard && clipboard::writer_available(), "mime": ["text/plain;charset=utf-8"],
536
+ "maxBytes": clipboard::MAX_CLIPBOARD_BYTES },
537
+ })
538
+ }
539
+
540
+ impl Session {
541
+ pub async fn open(
542
+ request: OpenRequest,
543
+ events: tokio_mpsc::UnboundedSender<Notice>,
544
+ ) -> std::result::Result<Self, SessionError> {
545
+ let id = opaque_id();
546
+ let wants_control = request.permissions.contains(&Permission::Control);
547
+ let wants_x11 = matches!(request.source, Some(SourceRequest::X11 { .. }));
548
+ // Which desktop decides which input path is even available: an X display
549
+ // takes XTest, which cannot reach any other session, while a portal
550
+ // desktop needs kernel input access.
551
+ if wants_control && !wants_x11 {
552
+ // Refuse up front rather than presenting a control surface that
553
+ // silently does nothing.
554
+ if let Err(unavailable) = crate::input::probe() {
555
+ return Err(SessionError::new(
556
+ "input-unavailable",
557
+ format!("{}; {}", unavailable.reason, unavailable.remedy),
558
+ ));
559
+ }
560
+ }
561
+
562
+ let metrics = Arc::new(Mutex::new(Metrics::default()));
563
+ let (frame_tx, frame_rx) = latest_frame();
564
+ let captured = metrics.clone();
565
+ let sink = Box::new(move |frame: I420, _seq: u64| {
566
+ if let Ok(mut m) = captured.lock() {
567
+ m.captured_frames += 1;
568
+ }
569
+ // An encoder that is behind gets the newest frame, not a queue: a
570
+ // desktop stream is live, and the last state is the one that counts.
571
+ if frame_tx.put(frame) {
572
+ if let Ok(mut m) = captured.lock() {
573
+ m.dropped_frames += 1;
574
+ }
575
+ }
576
+ });
577
+
578
+ let Selected {
579
+ source,
580
+ capture,
581
+ x11,
582
+ } = match request.source.clone() {
583
+ Some(SourceRequest::X11 { display }) => select_x11(
584
+ display.as_deref(),
585
+ request.max_width,
586
+ request.max_height,
587
+ request.max_fps,
588
+ metrics.clone(),
589
+ sink,
590
+ )
591
+ .map_err(|error| SessionError::new("source", format!("{error:#}")))?,
592
+ _ => {
593
+ let portal = portal::open(request.restore_token.as_deref())
594
+ .await
595
+ .map_err(|error| SessionError::new("source", format!("{error:#}")))?;
596
+ if let Some(token) = &portal.restore_token {
597
+ let _ = events.send(Notice {
598
+ session_id: id.clone(),
599
+ event: SessionEvent::RestoreToken(token.clone()),
600
+ });
601
+ }
602
+ let source = portal.source.clone();
603
+ let source_w = source.width.max(1) as usize;
604
+ let source_h = source.height.max(1) as usize;
605
+ let (width, height) =
606
+ fit(source_w, source_h, request.max_width, request.max_height);
607
+ let capture = capture::start(portal, width, height, request.max_fps, sink)
608
+ .map_err(|error| SessionError::new("source", format!("{error:#}")))?;
609
+ Selected {
610
+ source,
611
+ capture: FrameSource::Portal(capture),
612
+ x11: None,
613
+ }
614
+ }
615
+ };
616
+
617
+ let source_w = source.width.max(1) as usize;
618
+ let source_h = source.height.max(1) as usize;
619
+ let (width, height) = fit(source_w, source_h, request.max_width, request.max_height);
620
+
621
+ let bitrate_kbps = if request.bitrate_kbps == 0 {
622
+ auto_bitrate(width, height)
623
+ } else {
624
+ request.bitrate_kbps
625
+ };
626
+ let encoder = Encoder::new(
627
+ width,
628
+ height,
629
+ bitrate_kbps,
630
+ request.max_fps,
631
+ available_parallelism().min(8) as u32,
632
+ )
633
+ .map_err(|error| SessionError::new("encode", format!("{error:#}")))?;
634
+
635
+ // Everything that can still refuse the session comes before the peer:
636
+ // a peer created and then abandoned keeps its socket and tasks alive.
637
+ let input = if !wants_control {
638
+ None
639
+ } else {
640
+ Some(match &x11 {
641
+ Some(desktop) => InputTarget {
642
+ applier: Applier::X11(desktop.clone()),
643
+ held: HeldState::default(),
644
+ explicit_modifiers: Vec::new(),
645
+ chord_modifiers: Vec::new(),
646
+ chord_keys: Vec::new(),
647
+ wheel_rest: (0.0, 0.0),
648
+ },
649
+ None => InputTarget {
650
+ applier: Applier::Uinput(
651
+ InputDevices::create(source_w as i32, source_h as i32).map_err(
652
+ |error| SessionError::new("input-unavailable", format!("{error:#}")),
653
+ )?,
654
+ ),
655
+ held: HeldState::default(),
656
+ explicit_modifiers: Vec::new(),
657
+ chord_modifiers: Vec::new(),
658
+ chord_keys: Vec::new(),
659
+ wheel_rest: (0.0, 0.0),
660
+ },
661
+ })
662
+ };
663
+
664
+ let layout = Layout::from_environment().map_err(|error| {
665
+ SessionError::new("input", format!("no keyboard layout: {error:#}"))
666
+ })?;
667
+
668
+ let (peer_events_tx, peer_events_rx) = tokio_mpsc::unbounded_channel::<PeerEvent>();
669
+ let (peer, offer) = VideoPeer::offer(
670
+ TransportOptions {
671
+ ice_servers: request.ice_servers.clone(),
672
+ loopback_tcp: request.loopback_tcp,
673
+ // Well above the rate target, so pacing only spreads a large
674
+ // frame over a few tens of milliseconds and never queues.
675
+ pace_bps: (bitrate_kbps as f64 * 3_000.0).max(20_000_000.0),
676
+ },
677
+ peer_events_tx,
678
+ )
679
+ .await
680
+ .map_err(|error| SessionError::new("transport", format!("{error:#}")))?;
681
+
682
+ let generation = 1u64;
683
+ let geometry = serde_json::json!({
684
+ "source": { "width": source_w, "height": source_h },
685
+ "encoded": { "width": width, "height": height },
686
+ "origin": { "x": source.origin_x, "y": source.origin_y },
687
+ });
688
+
689
+ let pipeline = Arc::new(AtomicBool::new(true));
690
+ let inner = Arc::new(Inner {
691
+ id,
692
+ generation,
693
+ permissions: request.permissions,
694
+ source,
695
+ geometry,
696
+ metrics,
697
+ peer: Arc::new(peer),
698
+ encoder: Mutex::new(encoder),
699
+ input: Mutex::new(input),
700
+ capture: Mutex::new(Some(capture)),
701
+ layout: Mutex::new(layout),
702
+ last_seq: Mutex::new(0),
703
+ control_open: AtomicBool::new(false),
704
+ closed: AtomicBool::new(false),
705
+ pipeline: pipeline.clone(),
706
+ events: events.clone(),
707
+ });
708
+
709
+ inner.notify(SessionEvent::Description {
710
+ generation,
711
+ sdp: offer,
712
+ });
713
+ inner.notify(SessionEvent::State {
714
+ capture: "consented",
715
+ transport: String::from("new"),
716
+ first_frame: false,
717
+ });
718
+
719
+ spawn_pipeline(&inner, frame_rx, pipeline, request.max_fps, bitrate_kbps);
720
+ spawn_peer_events(&inner, peer_events_rx);
721
+ spawn_lease(&inner, request.ttl.unwrap_or(Duration::from_secs(3600)));
722
+
723
+ Ok(Self { inner })
724
+ }
725
+
726
+ pub fn id(&self) -> &str {
727
+ &self.inner.id
728
+ }
729
+
730
+ pub fn generation(&self) -> u64 {
731
+ self.inner.generation
732
+ }
733
+
734
+ pub fn geometry(&self) -> &serde_json::Value {
735
+ &self.inner.geometry
736
+ }
737
+
738
+ pub fn source(&self) -> &SelectedSource {
739
+ &self.inner.source
740
+ }
741
+
742
+ pub fn metrics(&self) -> Metrics {
743
+ self.inner
744
+ .metrics
745
+ .lock()
746
+ .map(|m| m.clone())
747
+ .unwrap_or_default()
748
+ }
749
+
750
+ pub async fn accept_answer(&self, sdp: String) -> Result<()> {
751
+ self.inner.peer.accept_answer(sdp).await.map(|_applied| ())
752
+ }
753
+
754
+ pub async fn add_candidate(
755
+ &self,
756
+ candidate: String,
757
+ sdp_mid: Option<String>,
758
+ sdp_m_line_index: Option<u16>,
759
+ ) -> Result<()> {
760
+ self.inner
761
+ .peer
762
+ .add_candidate(candidate, sdp_mid, sdp_m_line_index)
763
+ .await
764
+ }
765
+
766
+ /// Read the desktop clipboard for the consumer. Explicit, never polled.
767
+ pub async fn read_clipboard(&self) -> std::result::Result<(String, bool), String> {
768
+ if let Some((_, reason)) = self.inner.clipboard_refusal() {
769
+ return Err(reason.to_owned());
770
+ }
771
+ clipboard_task(move || clipboard::read_or_explain()).await
772
+ }
773
+
774
+ pub async fn write_clipboard(&self, text: String) -> std::result::Result<(), String> {
775
+ if let Some((_, reason)) = self.inner.clipboard_refusal() {
776
+ return Err(reason.to_owned());
777
+ }
778
+ clipboard_task(move || {
779
+ clipboard::write(&text)
780
+ .map(|()| String::new())
781
+ .map_err(|error| format!("{error:#}"))
782
+ })
783
+ .await
784
+ .map(|_written| ())
785
+ }
786
+
787
+ pub fn mark_closed_reason(&self) -> Option<String> {
788
+ self.inner.revoked_reason()
789
+ }
790
+
791
+ /// The source kind this session captures, e.g. `monitor`.
792
+ pub fn restore_source(&self) -> String {
793
+ self.inner
794
+ .source
795
+ .source_type
796
+ .clone()
797
+ .unwrap_or_else(|| String::from("monitor"))
798
+ }
799
+
800
+ pub async fn close(&self, reason: &str) {
801
+ self.inner.close(reason).await;
802
+ }
803
+ }
804
+
805
+ impl Inner {
806
+ /// Send one notification, stamped with the session it belongs to.
807
+ fn notify(&self, event: SessionEvent) {
808
+ let _ = self.events.send(Notice {
809
+ session_id: self.id.clone(),
810
+ event,
811
+ });
812
+ }
813
+
814
+ /// Stop everything and release whatever input this session held. Idempotent,
815
+ /// and the one teardown both an explicit close and a lease expiry take.
816
+ async fn close(&self, reason: &str) {
817
+ if self.closed.swap(true, Ordering::SeqCst) {
818
+ return;
819
+ }
820
+ self.control_open.store(false, Ordering::SeqCst);
821
+ self.pipeline.store(false, Ordering::SeqCst);
822
+ // Release first, then tear down: a stuck modifier is the one failure the
823
+ // user cannot undo by reconnecting.
824
+ if let Ok(mut input) = self.input.lock() {
825
+ if let Some(target) = input.as_mut() {
826
+ let _ = target.release_all();
827
+ }
828
+ *input = None;
829
+ }
830
+ let _ = self
831
+ .peer
832
+ .send_control(
833
+ &serde_json::to_string(&ControlReply::Revoked { reason })
834
+ .unwrap_or_else(|_| String::from(r#"{"kind":"revoked","reason":"closed"}"#)),
835
+ )
836
+ .await;
837
+ self.peer.close().await;
838
+ // Dropping the capture stops the PipeWire stream and joins its thread,
839
+ // which is what releases the compositor's consent for this session. It
840
+ // has to happen synchronously, not when the last handle happens to fall
841
+ // out of scope, so the consumer can truthfully say the desktop stopped.
842
+ let capture = self.capture.lock().ok().and_then(|mut held| held.take());
843
+ drop(capture);
844
+ self.notify(SessionEvent::State {
845
+ capture: "ended",
846
+ transport: String::from("closed"),
847
+ first_frame: false,
848
+ });
849
+ }
850
+
851
+ fn revoked_reason(&self) -> Option<String> {
852
+ if self.closed.load(Ordering::Relaxed) {
853
+ Some(String::from("session closed"))
854
+ } else {
855
+ None
856
+ }
857
+ }
858
+
859
+ fn has(&self, permission: Permission) -> bool {
860
+ self.permissions.contains(&permission)
861
+ }
862
+
863
+ fn clipboard_refusal(&self) -> Option<(&'static str, &'static str)> {
864
+ if self.source.source_type.as_deref() == Some("x11-root") || !wayland_clipboard_available()
865
+ {
866
+ return Some((
867
+ "clipboard-unsupported",
868
+ "clipboard is unavailable for this desktop source",
869
+ ));
870
+ }
871
+ if !self.has(Permission::Clipboard) {
872
+ return Some(("permission", "this session has no clipboard permission"));
873
+ }
874
+ None
875
+ }
876
+
877
+ fn reject(&self, seq: u64, code: &'static str, message: &str) {
878
+ if let Ok(mut m) = self.metrics.lock() {
879
+ m.input_rejected += 1;
880
+ }
881
+ let payload = ControlReply::Rejected { seq, code, message };
882
+ let _ = self.reply(&payload);
883
+ }
884
+
885
+ fn reply(&self, payload: &ControlReply<'_>) -> Result<()> {
886
+ let text = serde_json::to_string(payload)?;
887
+ // The channel is asynchronous; a failure here means the peer is gone,
888
+ // which the connection-state handler closes anyway.
889
+ let peer = self.peer.clone();
890
+ tokio::spawn(async move {
891
+ let _ = peer.send_control(&text).await;
892
+ });
893
+ Ok(())
894
+ }
895
+
896
+ /// Admit one client action. Validation, permission and ordering checks all
897
+ /// happen before any physical effect, so a refused action leaves the desktop
898
+ /// exactly as it was.
899
+ fn apply(self: &Arc<Self>, message: ControlMessage) {
900
+ let seq = message.seq();
901
+ if self.closed.load(Ordering::Relaxed) {
902
+ self.reject(seq, "session", "the session has ended");
903
+ return;
904
+ }
905
+ if !self.control_open.load(Ordering::Relaxed) {
906
+ self.reject(seq, "session", "the control channel is not open");
907
+ return;
908
+ }
909
+ if !self.has(Permission::Control) {
910
+ self.reject(seq, "permission", "this session is view-only");
911
+ return;
912
+ }
913
+ if seq != 0 {
914
+ let mut last = match self.last_seq.lock() {
915
+ Ok(last) => last,
916
+ Err(_) => return,
917
+ };
918
+ if seq <= *last {
919
+ drop(last);
920
+ self.reject(seq, "input-replay", "sequence already applied");
921
+ return;
922
+ }
923
+ *last = seq;
924
+ }
925
+ let outcome = match message {
926
+ ControlMessage::Pointer {
927
+ phase,
928
+ x,
929
+ y,
930
+ button,
931
+ ..
932
+ } => self.pointer(phase, x, y, button, seq),
933
+ ControlMessage::Wheel { dx, dy, .. } => self.wheel(dx, dy, seq),
934
+ ControlMessage::Key {
935
+ name,
936
+ character,
937
+ down,
938
+ modifiers,
939
+ ..
940
+ } => self.key(name, character, down, modifiers, seq),
941
+ ControlMessage::Text { text, .. } => self.text(&text, seq),
942
+ ControlMessage::ReleaseAll { .. } => {
943
+ if let Ok(mut input) = self.input.lock() {
944
+ if let Some(target) = input.as_mut() {
945
+ let _ = target.release_all();
946
+ }
947
+ }
948
+ Ok(())
949
+ }
950
+ ControlMessage::ClipboardRead { request, .. } => {
951
+ self.clipboard_read(request);
952
+ return;
953
+ }
954
+ ControlMessage::ClipboardWrite { request, text, .. } => {
955
+ self.clipboard_write(request, text);
956
+ return;
957
+ }
958
+ };
959
+ match outcome {
960
+ Ok(()) => {
961
+ if let Ok(mut m) = self.metrics.lock() {
962
+ m.input_applied += 1;
963
+ }
964
+ let _ = self.reply(&ControlReply::Ack { seq });
965
+ }
966
+ Err((code, message)) => self.reject(seq, code, &message),
967
+ }
968
+ }
969
+
970
+ fn with_input<T>(
971
+ &self,
972
+ action: impl FnOnce(&mut InputTarget) -> Result<T>,
973
+ ) -> std::result::Result<T, (&'static str, String)> {
974
+ let mut input = self
975
+ .input
976
+ .lock()
977
+ .map_err(|_| ("session", String::from("the input device is unavailable")))?;
978
+ match input.as_mut() {
979
+ Some(target) => action(target).map_err(|error| ("input", format!("{error:#}"))),
980
+ None => Err(("input-unavailable", String::from("no input backend"))),
981
+ }
982
+ }
983
+
984
+ fn pointer(
985
+ &self,
986
+ phase: PointerPhase,
987
+ x: i64,
988
+ y: i64,
989
+ button: i64,
990
+ _seq: u64,
991
+ ) -> std::result::Result<(), (&'static str, String)> {
992
+ let (width, height) = self.encoded_size();
993
+ if !(0..width as i64).contains(&x) || !(0..height as i64).contains(&y) {
994
+ return Err((
995
+ "coordinates",
996
+ format!("({x},{y}) is outside the {width}x{height} surface"),
997
+ ));
998
+ }
999
+ // The client sends the encoded surface's own pixels; the applier is
1000
+ // created for, and clamps to, the source's own pixels. One conversion
1001
+ // here keeps every caller honest.
1002
+ let (x, y) = to_source_pixels(x, y, (width, height), &self.source);
1003
+ self.with_input(|target| match phase {
1004
+ PointerPhase::Move => target.move_absolute(x, y),
1005
+ PointerPhase::Down => {
1006
+ target.move_absolute(x, y)?;
1007
+ target.button(Button::from_number(button), true)
1008
+ }
1009
+ PointerPhase::Up => {
1010
+ target.move_absolute(x, y)?;
1011
+ target.button(Button::from_number(button), false)
1012
+ }
1013
+ PointerPhase::Cancel => target.release_all(),
1014
+ })
1015
+ }
1016
+
1017
+ fn wheel(
1018
+ &self,
1019
+ dx: f64,
1020
+ dy: f64,
1021
+ _seq: u64,
1022
+ ) -> std::result::Result<(), (&'static str, String)> {
1023
+ if !dx.is_finite() || !dy.is_finite() || dx.abs() > 100.0 || dy.abs() > 100.0 {
1024
+ return Err(("coordinates", String::from("scroll delta is out of range")));
1025
+ }
1026
+ self.with_input(|target| target.scroll(dx, dy))
1027
+ }
1028
+
1029
+ fn key(
1030
+ &self,
1031
+ name: Option<String>,
1032
+ character: Option<String>,
1033
+ down: bool,
1034
+ modifiers: Vec<String>,
1035
+ _seq: u64,
1036
+ ) -> std::result::Result<(), (&'static str, String)> {
1037
+ // A named modifier is applied directly: it is the same key the user
1038
+ // would press, and it does not depend on the layout.
1039
+ if let Some(name) = &name {
1040
+ if let Some(code) = keymap::modifier_key(name) {
1041
+ return self.with_input(|target| target.modifier(code, down, true));
1042
+ }
1043
+ }
1044
+
1045
+ let stroke = {
1046
+ let layout = self
1047
+ .layout
1048
+ .lock()
1049
+ .map_err(|_| ("session", String::from("no keyboard layout")))?;
1050
+ if let Some(name) = name {
1051
+ keymap::named_key(&name)
1052
+ .ok_or(("text-unsupported", format!("unknown key {name}")))?
1053
+ } else if let Some(character) = character {
1054
+ let wanted = character
1055
+ .chars()
1056
+ .next()
1057
+ .ok_or(("text-unsupported", String::from("empty character")))?;
1058
+ layout.keystroke_for_char(wanted).ok_or((
1059
+ "text-unsupported",
1060
+ format!("the active layout cannot produce {wanted:?}"),
1061
+ ))?
1062
+ } else {
1063
+ return Err(("text-unsupported", String::from("no key given")));
1064
+ }
1065
+ };
1066
+
1067
+ // Explicitly requested modifiers travel with this key only.
1068
+ let mut requested: Vec<i16> = modifiers
1069
+ .iter()
1070
+ .filter_map(|name| keymap::modifier_key(name))
1071
+ .collect();
1072
+ requested.extend(stroke.modifiers());
1073
+ requested.sort_unstable();
1074
+ requested.dedup();
1075
+
1076
+ self.with_input(|target| target.chord(stroke.code, requested, down))
1077
+ }
1078
+
1079
+ fn text(&self, text: &str, seq: u64) -> std::result::Result<(), (&'static str, String)> {
1080
+ if text.len() > 4096 {
1081
+ return Err(("text-too-large", String::from("text exceeds 4096 bytes")));
1082
+ }
1083
+ let layout = self
1084
+ .layout
1085
+ .lock()
1086
+ .map_err(|_| ("session", String::from("no keyboard layout")))?;
1087
+ let (plan, unreachable) = layout.plan_text(text);
1088
+ drop(layout);
1089
+ if !unreachable.is_empty() {
1090
+ return Err((
1091
+ "text-unsupported",
1092
+ format!("the active layout cannot produce {unreachable:?}; use the clipboard"),
1093
+ ));
1094
+ }
1095
+ let _ = seq;
1096
+ self.with_input(|target| {
1097
+ // Refuse an unsupported physical key before typing any prefix or
1098
+ // holding a modifier. X11 already consumes evdev identities.
1099
+ target.check_keys(
1100
+ plan.iter()
1101
+ .flatten()
1102
+ .flat_map(|stroke| std::iter::once(stroke.code).chain(stroke.modifiers())),
1103
+ )?;
1104
+ for keystroke in plan {
1105
+ for stroke in keystroke {
1106
+ let modifiers = stroke.modifiers();
1107
+ for modifier in &modifiers {
1108
+ target.modifier(*modifier, true, false)?;
1109
+ }
1110
+ target.key(stroke.code, true)?;
1111
+ target.key(stroke.code, false)?;
1112
+ for modifier in modifiers.iter().rev() {
1113
+ target.modifier(*modifier, false, false)?;
1114
+ }
1115
+ }
1116
+ }
1117
+ Ok(())
1118
+ })
1119
+ }
1120
+
1121
+ fn clipboard_read(self: &Arc<Self>, request: String) {
1122
+ if let Some((code, reason)) = self.clipboard_refusal() {
1123
+ self.reject(0, code, reason);
1124
+ return;
1125
+ }
1126
+ let inner = Arc::clone(self);
1127
+ tokio::spawn(async move {
1128
+ let (text, truncated, error) = match clipboard_task(clipboard::read_or_explain).await {
1129
+ Ok((text, truncated)) => (text, truncated, None),
1130
+ Err(reason) => (String::new(), false, Some(reason)),
1131
+ };
1132
+ let _ = inner.reply(&ControlReply::Clipboard {
1133
+ request: &request,
1134
+ text,
1135
+ truncated,
1136
+ error: error.as_deref(),
1137
+ });
1138
+ });
1139
+ }
1140
+
1141
+ fn clipboard_write(self: &Arc<Self>, request: String, text: String) {
1142
+ if let Some((code, reason)) = self.clipboard_refusal() {
1143
+ self.reject(0, code, reason);
1144
+ return;
1145
+ }
1146
+ let inner = Arc::clone(self);
1147
+ tokio::spawn(async move {
1148
+ let error = clipboard_task(move || {
1149
+ clipboard::write(&text)
1150
+ .map(|()| String::new())
1151
+ .map_err(|error| format!("{error:#}"))
1152
+ })
1153
+ .await
1154
+ .err();
1155
+ let _ = inner.reply(&ControlReply::Clipboard {
1156
+ request: &request,
1157
+ text: String::new(),
1158
+ truncated: false,
1159
+ error: error.as_deref(),
1160
+ });
1161
+ });
1162
+ }
1163
+
1164
+ fn encoded_size(&self) -> (usize, usize) {
1165
+ let encoded = &self.geometry["encoded"];
1166
+ (
1167
+ encoded["width"].as_u64().unwrap_or(0) as usize,
1168
+ encoded["height"].as_u64().unwrap_or(0) as usize,
1169
+ )
1170
+ }
1171
+ }
1172
+
1173
+ /// The interval between frames at `max_fps`. The requested rate is a cap, so the
1174
+ /// pipeline and the X11 capture loop derive their cadence from the same rule.
1175
+ fn frame_interval(max_fps: u32) -> Duration {
1176
+ Duration::from_secs_f64(1.0 / max_fps.max(1) as f64)
1177
+ }
1178
+
1179
+ /// The newest captured frame, handed from the capture thread to the encoder. A
1180
+ /// newer frame replaces one the encoder has not taken, so however far behind
1181
+ /// it falls, what it codes next is the desktop as it is now.
1182
+ struct FrameSlot {
1183
+ frame: Mutex<(Option<I420>, bool)>,
1184
+ ready: Condvar,
1185
+ }
1186
+
1187
+ struct FrameSender(Arc<FrameSlot>);
1188
+ struct FrameReceiver(Arc<FrameSlot>);
1189
+
1190
+ fn latest_frame() -> (FrameSender, FrameReceiver) {
1191
+ let slot = Arc::new(FrameSlot {
1192
+ frame: Mutex::new((None, false)),
1193
+ ready: Condvar::new(),
1194
+ });
1195
+ (FrameSender(slot.clone()), FrameReceiver(slot))
1196
+ }
1197
+
1198
+ impl FrameSender {
1199
+ /// Hand over a frame; true when it replaced one the encoder never took.
1200
+ fn put(&self, frame: I420) -> bool {
1201
+ let mut held = self
1202
+ .0
1203
+ .frame
1204
+ .lock()
1205
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
1206
+ let replaced = held.0.replace(frame).is_some();
1207
+ self.0.ready.notify_one();
1208
+ replaced
1209
+ }
1210
+ }
1211
+
1212
+ impl Drop for FrameSender {
1213
+ /// The capture stopped: the encoder ends once it has taken what is left.
1214
+ fn drop(&mut self) {
1215
+ let mut held = self
1216
+ .0
1217
+ .frame
1218
+ .lock()
1219
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
1220
+ held.1 = true;
1221
+ self.0.ready.notify_one();
1222
+ }
1223
+ }
1224
+
1225
+ enum Taken {
1226
+ Frame(I420),
1227
+ Timeout,
1228
+ Ended,
1229
+ }
1230
+
1231
+ impl FrameReceiver {
1232
+ fn take(&self, wait: Duration) -> Taken {
1233
+ let held = self
1234
+ .0
1235
+ .frame
1236
+ .lock()
1237
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
1238
+ let (mut held, _) = self
1239
+ .0
1240
+ .ready
1241
+ .wait_timeout_while(held, wait, |(frame, ended)| frame.is_none() && !*ended)
1242
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
1243
+ match held.0.take() {
1244
+ Some(frame) => Taken::Frame(frame),
1245
+ None if held.1 => Taken::Ended,
1246
+ None => Taken::Timeout,
1247
+ }
1248
+ }
1249
+ }
1250
+
1251
+ /// How long the desktop has to stay still before its last frame is refined.
1252
+ /// Long enough that a pause between keystrokes does not pay for a refinement
1253
+ /// the next key makes stale; short enough that reading does not wait for it.
1254
+ const REFINE_AFTER: Duration = Duration::from_millis(150);
1255
+
1256
+ /// The longest the stream goes without a frame while the desktop is still. A
1257
+ /// WebRTC receiver that has had no decodable frame for a few seconds asks for
1258
+ /// a key frame (a 4K one is hundreds of kilobytes), then asks again after the
1259
+ /// next quiet spell. An unchanged frame costs a few hundred bytes and keeps it
1260
+ /// satisfied.
1261
+ const KEEPALIVE_AFTER: Duration = Duration::from_millis(1000);
1262
+
1263
+ /// The rate target for a surface when the consumer did not choose one: enough
1264
+ /// for a scroll across a full 4K desktop to stay legible. A still desktop costs
1265
+ /// almost nothing whatever this says.
1266
+ fn auto_bitrate(width: usize, height: usize) -> u32 {
1267
+ (1500 + (width * height * 18 / 10_000) as u32).min(25_000)
1268
+ }
1269
+
1270
+ /// Loss-based rate target: back off quickly when the far end reports loss,
1271
+ /// creep back when it reports none, never above what the session asked for.
1272
+ struct RateControl {
1273
+ ceiling: u32,
1274
+ target: u32,
1275
+ last_change: Option<Instant>,
1276
+ }
1277
+
1278
+ impl RateControl {
1279
+ fn new(ceiling: u32) -> Self {
1280
+ Self {
1281
+ ceiling,
1282
+ target: ceiling,
1283
+ last_change: None,
1284
+ }
1285
+ }
1286
+
1287
+ /// The new target when these reports move it, at most once a second.
1288
+ fn update(&mut self, reports: &[u8], now: Instant) -> Option<u32> {
1289
+ let worst = *reports.iter().max()?;
1290
+ if self
1291
+ .last_change
1292
+ .is_some_and(|at| now.duration_since(at) < Duration::from_secs(1))
1293
+ {
1294
+ return None;
1295
+ }
1296
+ let loss = worst as f64 / 256.0;
1297
+ let floor = (self.ceiling / 8).max(500);
1298
+ let next = if loss > 0.10 {
1299
+ ((self.target as f64 * 0.7) as u32).max(floor)
1300
+ } else if loss < 0.02 {
1301
+ ((self.target as f64 * 1.08) as u32 + 64).min(self.ceiling)
1302
+ } else {
1303
+ self.target
1304
+ };
1305
+ if next == self.target {
1306
+ return None;
1307
+ }
1308
+ self.target = next;
1309
+ self.last_change = Some(now);
1310
+ Some(next)
1311
+ }
1312
+ }
1313
+
1314
+ enum Pass {
1315
+ Motion {
1316
+ keyframe: bool,
1317
+ },
1318
+ Refine,
1319
+ /// The same picture again, so a still stream never looks stalled.
1320
+ Keepalive,
1321
+ }
1322
+
1323
+ /// Encode and send frames off the async runtime: libvpx blocks for the duration
1324
+ /// of a frame and the send is asynchronous, so a plain thread driving the
1325
+ /// runtime handle keeps both honest.
1326
+ ///
1327
+ /// PipeWire delivers a frame only when the desktop changes. Every frame that
1328
+ /// arrives is either coded or superseded by a newer one within one frame
1329
+ /// interval, so the phone never shows a state older than the desktop's; the
1330
+ /// rate cap delays a frame rather than dropping the last one of a burst. When
1331
+ /// the desktop goes still the last frame is coded once more, sharp.
1332
+ fn spawn_pipeline(
1333
+ inner: &Arc<Inner>,
1334
+ frame_rx: FrameReceiver,
1335
+ running: Arc<AtomicBool>,
1336
+ max_fps: u32,
1337
+ bitrate_kbps: u32,
1338
+ ) {
1339
+ let inner = inner.clone();
1340
+ let handle = tokio::runtime::Handle::current();
1341
+ std::thread::Builder::new()
1342
+ .name("desklink-encode".into())
1343
+ .spawn(move || {
1344
+ let interval = frame_interval(max_fps);
1345
+ let mut rate = RateControl::new(bitrate_kbps);
1346
+ if let Ok(mut m) = inner.metrics.lock() {
1347
+ m.target_kbps = bitrate_kbps;
1348
+ }
1349
+ let mut latest: Option<I420> = None;
1350
+ // `pending`: the latest frame has not been coded. `refined`: it has
1351
+ // had its refinement pass, or there is nothing to refine.
1352
+ let mut pending = false;
1353
+ let mut refined = true;
1354
+ let mut keyframe = true;
1355
+ let mut still_since = Instant::now();
1356
+ let mut last_sent: Option<Instant> = None;
1357
+ loop {
1358
+ let now = Instant::now();
1359
+ let slot = last_sent.map_or(Duration::ZERO, |at| {
1360
+ interval.saturating_sub(now.duration_since(at))
1361
+ });
1362
+ let connected = inner.peer.is_connected();
1363
+ let wait = if !connected {
1364
+ // Nothing sent before the transport connects arrives; the
1365
+ // first frame after it is a key frame.
1366
+ Duration::from_millis(50)
1367
+ } else if pending {
1368
+ slot
1369
+ } else if !refined {
1370
+ REFINE_AFTER
1371
+ .saturating_sub(now.duration_since(still_since))
1372
+ .max(slot)
1373
+ } else {
1374
+ last_sent
1375
+ .map_or(Duration::from_millis(100), |at| {
1376
+ KEEPALIVE_AFTER.saturating_sub(now.duration_since(at))
1377
+ })
1378
+ .clamp(Duration::from_millis(10), Duration::from_millis(100))
1379
+ };
1380
+ match frame_rx.take(wait) {
1381
+ Taken::Frame(frame) => {
1382
+ if pending {
1383
+ if let Ok(mut m) = inner.metrics.lock() {
1384
+ m.dropped_frames += 1;
1385
+ }
1386
+ }
1387
+ latest = Some(frame);
1388
+ pending = true;
1389
+ refined = false;
1390
+ still_since = Instant::now();
1391
+ }
1392
+ Taken::Timeout => {}
1393
+ Taken::Ended => break,
1394
+ }
1395
+ if !running.load(Ordering::Relaxed) {
1396
+ break;
1397
+ }
1398
+ if inner.peer.take_keyframe_request() || !connected {
1399
+ keyframe = true;
1400
+ }
1401
+ let reports = inner.peer.take_loss_reports();
1402
+ if let Some(kbps) = rate.update(&reports, Instant::now()) {
1403
+ let applied = inner
1404
+ .encoder
1405
+ .lock()
1406
+ .map(|mut encoder| encoder.set_bitrate(kbps));
1407
+ if matches!(applied, Ok(Ok(()))) {
1408
+ if let Ok(mut m) = inner.metrics.lock() {
1409
+ m.target_kbps = kbps;
1410
+ }
1411
+ }
1412
+ }
1413
+ let Some(frame) = latest.as_ref() else {
1414
+ continue;
1415
+ };
1416
+ if !connected {
1417
+ continue;
1418
+ }
1419
+ let now = Instant::now();
1420
+ if last_sent.is_some_and(|at| now.duration_since(at) < interval) {
1421
+ continue;
1422
+ }
1423
+ let pass = if pending || keyframe {
1424
+ Pass::Motion { keyframe }
1425
+ } else if !refined && now.duration_since(still_since) >= REFINE_AFTER {
1426
+ Pass::Refine
1427
+ } else if refined
1428
+ && last_sent.is_some_and(|at| now.duration_since(at) >= KEEPALIVE_AFTER)
1429
+ {
1430
+ Pass::Keepalive
1431
+ } else {
1432
+ continue;
1433
+ };
1434
+
1435
+ let started = Instant::now();
1436
+ let packet = {
1437
+ let mut encoder = match inner.encoder.lock() {
1438
+ Ok(encoder) => encoder,
1439
+ Err(_) => break,
1440
+ };
1441
+ match pass {
1442
+ Pass::Motion { keyframe } => encoder.encode(frame, keyframe),
1443
+ Pass::Refine => encoder.refine(frame),
1444
+ Pass::Keepalive => encoder.encode(frame, false),
1445
+ }
1446
+ };
1447
+ let packet = match packet {
1448
+ Ok(packet) => packet,
1449
+ Err(error) => {
1450
+ if let Ok(mut m) = inner.metrics.lock() {
1451
+ m.dropped_frames += 1;
1452
+ }
1453
+ // The encoder refusing a frame means it is not the size
1454
+ // it was built for, or libvpx rejected it. Both leave a
1455
+ // permanently black desktop, so end the session instead
1456
+ // of dropping frames in silence; the client turns this
1457
+ // reason into what it shows the user.
1458
+ eprintln!("the encoder rejected a frame: {error:#}");
1459
+ let reason = String::from("the encoder rejected a frame");
1460
+ inner.notify(SessionEvent::Revoked {
1461
+ reason: reason.clone(),
1462
+ });
1463
+ let target = inner.clone();
1464
+ handle.spawn(async move {
1465
+ target.close(&reason).await;
1466
+ });
1467
+ break;
1468
+ }
1469
+ };
1470
+ if let Ok(mut m) = inner.metrics.lock() {
1471
+ m.encoded_frames += 1;
1472
+ m.encoded_bytes += packet.data.len() as u64;
1473
+ m.encode_micros += started.elapsed().as_micros() as u64;
1474
+ match pass {
1475
+ Pass::Motion { keyframe: true } => m.key_frames += 1,
1476
+ Pass::Refine => m.refined_frames += 1,
1477
+ Pass::Motion { keyframe: false } | Pass::Keepalive => {}
1478
+ }
1479
+ }
1480
+ match pass {
1481
+ Pass::Motion { .. } => {
1482
+ pending = false;
1483
+ keyframe = false;
1484
+ refined = false;
1485
+ still_since = now;
1486
+ }
1487
+ Pass::Refine => refined = true,
1488
+ Pass::Keepalive => {}
1489
+ }
1490
+ last_sent = Some(now);
1491
+ let peer = inner.peer.clone();
1492
+ if let Err(error) =
1493
+ handle.block_on(peer.send_frame(&packet.data, packet.keyframe, now))
1494
+ {
1495
+ // The transport is gone. End the session rather than leave
1496
+ // the desktop captured behind a picture that stopped.
1497
+ eprintln!("the video track refused an encoded frame: {error:#}");
1498
+ let reason = String::from("the connection to the phone was lost");
1499
+ inner.notify(SessionEvent::Revoked {
1500
+ reason: reason.clone(),
1501
+ });
1502
+ let target = inner.clone();
1503
+ handle.spawn(async move {
1504
+ target.close(&reason).await;
1505
+ });
1506
+ break;
1507
+ }
1508
+ }
1509
+ })
1510
+ .ok();
1511
+ }
1512
+
1513
+ fn spawn_peer_events(inner: &Arc<Inner>, mut events: tokio_mpsc::UnboundedReceiver<PeerEvent>) {
1514
+ let inner = inner.clone();
1515
+ tokio::spawn(async move {
1516
+ while let Some(event) = events.recv().await {
1517
+ if inner.closed.load(Ordering::SeqCst) {
1518
+ break;
1519
+ }
1520
+ match event {
1521
+ PeerEvent::Candidate {
1522
+ candidate,
1523
+ sdp_mid,
1524
+ sdp_m_line_index,
1525
+ } => {
1526
+ inner.notify(SessionEvent::Candidate {
1527
+ generation: inner.generation,
1528
+ candidate,
1529
+ sdp_mid,
1530
+ sdp_m_line_index,
1531
+ });
1532
+ }
1533
+ PeerEvent::State(state) => {
1534
+ use webrtc::peer_connection::RTCPeerConnectionState as State;
1535
+ // ICE loss need not close SCTP. Never leave a drag or chord
1536
+ // held while waiting for that independent notification.
1537
+ if matches!(state, State::Disconnected | State::Failed | State::Closed) {
1538
+ if let Ok(mut input) = inner.input.lock() {
1539
+ if let Some(target) = input.as_mut() {
1540
+ let _ = target.release_all();
1541
+ }
1542
+ }
1543
+ }
1544
+ if matches!(state, State::Failed | State::Closed) {
1545
+ let reason = String::from("the connection to the phone was lost");
1546
+ inner.notify(SessionEvent::Revoked {
1547
+ reason: reason.clone(),
1548
+ });
1549
+ inner.close(&reason).await;
1550
+ continue;
1551
+ }
1552
+ inner.notify(SessionEvent::State {
1553
+ capture: if inner.metrics.lock().map(|m| m.encoded_frames).unwrap_or(0) > 0
1554
+ {
1555
+ "streaming"
1556
+ } else {
1557
+ "consented"
1558
+ },
1559
+ transport: format!("{state}").to_lowercase(),
1560
+ first_frame: inner.metrics.lock().map(|m| m.encoded_frames).unwrap_or(0)
1561
+ > 0,
1562
+ });
1563
+ }
1564
+ PeerEvent::ControlOpen => {
1565
+ inner.control_open.store(true, Ordering::SeqCst);
1566
+ let payload = ControlReply::Hello {
1567
+ protocol: crate::protocol::PROTOCOL_VERSION,
1568
+ geometry: inner.geometry.clone(),
1569
+ };
1570
+ let _ = inner.reply(&payload);
1571
+ }
1572
+ PeerEvent::ControlClosed => {
1573
+ inner.control_open.store(false, Ordering::SeqCst);
1574
+ // A client that vanished must not leave a button held.
1575
+ if let Ok(mut input) = inner.input.lock() {
1576
+ if let Some(target) = input.as_mut() {
1577
+ let _ = target.release_all();
1578
+ }
1579
+ }
1580
+ }
1581
+ PeerEvent::ControlMessage(text) => {
1582
+ match serde_json::from_str::<ControlMessage>(&text) {
1583
+ Ok(message) => inner.apply(message),
1584
+ Err(_) => inner.reject(0, "operation", "unrecognised control message"),
1585
+ }
1586
+ }
1587
+ }
1588
+ }
1589
+ });
1590
+ }
1591
+
1592
+ fn spawn_lease(inner: &Arc<Inner>, ttl: Duration) {
1593
+ let inner = inner.clone();
1594
+ tokio::spawn(async move {
1595
+ tokio::time::sleep(ttl).await;
1596
+ if inner.closed.load(Ordering::SeqCst) {
1597
+ return;
1598
+ }
1599
+ let reason = String::from("the session lease expired");
1600
+ inner.notify(SessionEvent::Revoked {
1601
+ reason: reason.clone(),
1602
+ });
1603
+ // An expiry ends the session exactly as an explicit close does; the
1604
+ // notification above is not a substitute for stopping the desktop.
1605
+ inner.close(&reason).await;
1606
+ });
1607
+ }
1608
+
1609
+ fn available_parallelism() -> usize {
1610
+ std::thread::available_parallelism()
1611
+ .map(|n| n.get())
1612
+ .unwrap_or(4)
1613
+ }
1614
+
1615
+ /// Encoded-surface pixels to the source's own pixels. The client aims at what it
1616
+ /// sees; the applier is sized and clamped to the source, so the layout origin
1617
+ /// must not be added here.
1618
+ fn to_source_pixels(
1619
+ x: i64,
1620
+ y: i64,
1621
+ encoded: (usize, usize),
1622
+ source: &SelectedSource,
1623
+ ) -> (i64, i64) {
1624
+ let (width, height) = encoded;
1625
+ (
1626
+ x * source.width as i64 / width as i64,
1627
+ y * source.height as i64 / height as i64,
1628
+ )
1629
+ }
1630
+
1631
+ fn opaque_id() -> String {
1632
+ use rand::Rng;
1633
+ let mut rng = rand::thread_rng();
1634
+ let bytes: [u8; 16] = rng.gen();
1635
+ bytes.iter().map(|byte| format!("{byte:02x}")).collect()
1636
+ }
1637
+
1638
+ #[cfg(test)]
1639
+ mod tests {
1640
+ use super::*;
1641
+
1642
+ #[test]
1643
+ fn a_small_source_is_never_upscaled() {
1644
+ assert_eq!(fit(1000, 600, 1280, 800), (1000, 600));
1645
+ }
1646
+
1647
+ #[test]
1648
+ fn a_large_source_is_fitted_into_the_box_with_even_dimensions() {
1649
+ let (w, h) = fit(2560, 1440, 1280, 800);
1650
+ assert_eq!((w, h), (1280, 720));
1651
+ assert_eq!((w % 2, h % 2), (0, 0));
1652
+ }
1653
+
1654
+ #[test]
1655
+ fn fit_keeps_the_aspect_ratio_when_the_box_is_the_constraint() {
1656
+ assert_eq!(fit(2560, 1440, 1200, 600), (1066, 600));
1657
+ }
1658
+
1659
+ #[test]
1660
+ fn an_encoded_surface_pixel_maps_to_the_sources_own_pixel() {
1661
+ let source = SelectedSource {
1662
+ node_id: 0,
1663
+ width: 2560,
1664
+ height: 1440,
1665
+ position: Some((1920, 0)),
1666
+ source_type: None,
1667
+ origin_x: 1920,
1668
+ origin_y: 0,
1669
+ };
1670
+ // A tap at the centre of a 1280x720 surface must reach the centre of the
1671
+ // 2560x1440 source, not two thirds of the way there; a tap in the right
1672
+ // half must stay source-local rather than land on a layout origin the
1673
+ // applier would clamp away.
1674
+ assert_eq!(
1675
+ to_source_pixels(640, 360, (1280, 720), &source),
1676
+ (1280, 720)
1677
+ );
1678
+ assert_eq!(
1679
+ to_source_pixels(1279, 719, (1280, 720), &source),
1680
+ (2558, 1438)
1681
+ );
1682
+ assert_eq!(to_source_pixels(0, 0, (1280, 720), &source), (0, 0));
1683
+ }
1684
+
1685
+ async fn test_inner(
1686
+ events: tokio_mpsc::UnboundedSender<Notice>,
1687
+ ) -> (Arc<Inner>, Arc<Mutex<Vec<(i16, bool)>>>) {
1688
+ let (peer_events, _peer_events_rx) = tokio_mpsc::unbounded_channel();
1689
+ let (peer, _offer) = VideoPeer::offer(
1690
+ TransportOptions {
1691
+ ice_servers: Vec::new(),
1692
+ loopback_tcp: false,
1693
+ pace_bps: 20_000_000.0,
1694
+ },
1695
+ peer_events,
1696
+ )
1697
+ .await
1698
+ .expect("a peer connection builds without a network");
1699
+ // The pipeline codes nothing before the transport connects.
1700
+ peer.assume_connected();
1701
+ let recorded = Arc::new(Mutex::new(Vec::new()));
1702
+ let inner = Arc::new(Inner {
1703
+ id: String::from("test-session"),
1704
+ generation: 1,
1705
+ permissions: vec![Permission::Control],
1706
+ source: SelectedSource {
1707
+ node_id: 0,
1708
+ width: 640,
1709
+ height: 480,
1710
+ position: None,
1711
+ source_type: None,
1712
+ origin_x: 0,
1713
+ origin_y: 0,
1714
+ },
1715
+ geometry: serde_json::json!({
1716
+ "source": { "width": 640, "height": 480 },
1717
+ "encoded": { "width": 640, "height": 480 },
1718
+ "origin": { "x": 0, "y": 0 },
1719
+ }),
1720
+ metrics: Arc::new(Mutex::new(Metrics::default())),
1721
+ peer: Arc::new(peer),
1722
+ encoder: Mutex::new(Encoder::new(64, 64, 1000, 30, 1).expect("an encoder")),
1723
+ input: Mutex::new(Some(InputTarget {
1724
+ applier: Applier::Recording(recorded.clone()),
1725
+ held: HeldState::default(),
1726
+ explicit_modifiers: Vec::new(),
1727
+ chord_modifiers: Vec::new(),
1728
+ chord_keys: Vec::new(),
1729
+ wheel_rest: (0.0, 0.0),
1730
+ })),
1731
+ capture: Mutex::new(None),
1732
+ layout: Mutex::new(Layout::from_environment().expect("a keymap")),
1733
+ last_seq: Mutex::new(0),
1734
+ control_open: AtomicBool::new(true),
1735
+ closed: AtomicBool::new(false),
1736
+ pipeline: Arc::new(AtomicBool::new(true)),
1737
+ events,
1738
+ });
1739
+ (inner, recorded)
1740
+ }
1741
+
1742
+ #[tokio::test]
1743
+ async fn a_lease_that_expires_ends_the_session_and_releases_held_input() {
1744
+ let (events, mut received) = tokio_mpsc::unbounded_channel();
1745
+ let (inner, recorded) = test_inner(events).await;
1746
+
1747
+ inner.apply(ControlMessage::Key {
1748
+ name: None,
1749
+ character: Some(String::from("a")),
1750
+ down: true,
1751
+ modifiers: Vec::new(),
1752
+ seq: 1,
1753
+ });
1754
+ assert!(
1755
+ recorded.lock().unwrap().iter().any(|(_, down)| *down),
1756
+ "the press reached the desktop",
1757
+ );
1758
+
1759
+ spawn_lease(&inner, Duration::from_millis(20));
1760
+ tokio::time::sleep(Duration::from_millis(150)).await;
1761
+
1762
+ assert!(
1763
+ inner.revoked_reason().is_some(),
1764
+ "an expired lease must end the session"
1765
+ );
1766
+ assert!(
1767
+ !inner.pipeline.load(Ordering::SeqCst),
1768
+ "an expired lease must stop capture",
1769
+ );
1770
+ assert!(
1771
+ recorded.lock().unwrap().iter().any(|(_, down)| !*down),
1772
+ "an expired lease must release what the session pressed",
1773
+ );
1774
+ assert!(
1775
+ matches!(
1776
+ received.try_recv(),
1777
+ Ok(Notice {
1778
+ event: SessionEvent::Revoked { .. },
1779
+ ..
1780
+ })
1781
+ ),
1782
+ "the consumer is still told why the session ended",
1783
+ );
1784
+
1785
+ let rejected = inner.metrics.lock().unwrap().input_rejected;
1786
+ inner.apply(ControlMessage::Key {
1787
+ name: None,
1788
+ character: Some(String::from("b")),
1789
+ down: true,
1790
+ modifiers: Vec::new(),
1791
+ seq: 2,
1792
+ });
1793
+ assert_eq!(
1794
+ inner.metrics.lock().unwrap().input_rejected,
1795
+ rejected + 1,
1796
+ "a control message after expiry must be refused",
1797
+ );
1798
+ }
1799
+
1800
+ #[tokio::test]
1801
+ async fn a_held_modifier_survives_character_named_and_text_chords() {
1802
+ use crate::input::keycode;
1803
+ let (events, _received) = tokio_mpsc::unbounded_channel();
1804
+ let (inner, recorded) = test_inner(events).await;
1805
+ for (seq, name) in ["Control", "Shift", "Alt", "Meta"].iter().enumerate() {
1806
+ inner.apply(ControlMessage::Key {
1807
+ name: Some((*name).into()),
1808
+ character: None,
1809
+ down: true,
1810
+ modifiers: Vec::new(),
1811
+ seq: seq as u64 + 1,
1812
+ });
1813
+ }
1814
+ inner.apply(ControlMessage::Key {
1815
+ name: None,
1816
+ character: Some(String::from("C")),
1817
+ down: true,
1818
+ modifiers: vec![String::from("Control"), String::from("Shift")],
1819
+ seq: 5,
1820
+ });
1821
+ inner.apply(ControlMessage::Key {
1822
+ name: None,
1823
+ character: Some(String::from("C")),
1824
+ down: false,
1825
+ modifiers: vec![String::from("Control"), String::from("Shift")],
1826
+ seq: 6,
1827
+ });
1828
+ inner.apply(ControlMessage::Key {
1829
+ name: Some(String::from("ArrowLeft")),
1830
+ character: None,
1831
+ down: true,
1832
+ modifiers: vec![String::from("Alt"), String::from("Meta")],
1833
+ seq: 7,
1834
+ });
1835
+ inner.apply(ControlMessage::Key {
1836
+ name: Some(String::from("ArrowLeft")),
1837
+ character: None,
1838
+ down: false,
1839
+ modifiers: vec![String::from("Alt"), String::from("Meta")],
1840
+ seq: 8,
1841
+ });
1842
+ inner.apply(ControlMessage::Text {
1843
+ text: String::from("C"),
1844
+ seq: 9,
1845
+ });
1846
+ let modifiers = [
1847
+ keycode::LEFT_CTRL,
1848
+ keycode::LEFT_SHIFT,
1849
+ keycode::LEFT_ALT,
1850
+ keycode::LEFT_META,
1851
+ ];
1852
+ let strokes = recorded.lock().unwrap().clone();
1853
+ for code in modifiers {
1854
+ assert_eq!(
1855
+ strokes
1856
+ .iter()
1857
+ .filter(|stroke| **stroke == (code, true))
1858
+ .count(),
1859
+ 1
1860
+ );
1861
+ assert!(
1862
+ !strokes.contains(&(code, false)),
1863
+ "the separate key still holds {code}"
1864
+ );
1865
+ }
1866
+ for (seq, name) in ["Control", "Shift", "Alt", "Meta"].iter().enumerate() {
1867
+ inner.apply(ControlMessage::Key {
1868
+ name: Some((*name).into()),
1869
+ character: None,
1870
+ down: false,
1871
+ modifiers: Vec::new(),
1872
+ seq: seq as u64 + 10,
1873
+ });
1874
+ }
1875
+ {
1876
+ let strokes = recorded.lock().unwrap();
1877
+ for code in modifiers {
1878
+ assert_eq!(
1879
+ strokes
1880
+ .iter()
1881
+ .filter(|stroke| **stroke == (code, false))
1882
+ .count(),
1883
+ 1
1884
+ );
1885
+ }
1886
+ }
1887
+ inner.apply(ControlMessage::Key {
1888
+ name: Some(String::from("ArrowLeft")),
1889
+ character: None,
1890
+ down: true,
1891
+ modifiers: vec![String::from("Control")],
1892
+ seq: 14,
1893
+ });
1894
+ inner.apply(ControlMessage::Key {
1895
+ name: Some(String::from("ArrowLeft")),
1896
+ character: None,
1897
+ down: false,
1898
+ modifiers: vec![String::from("Control")],
1899
+ seq: 15,
1900
+ });
1901
+ {
1902
+ let strokes = recorded.lock().unwrap();
1903
+ assert_eq!(
1904
+ strokes
1905
+ .iter()
1906
+ .filter(|stroke| **stroke == (keycode::LEFT_CTRL, true))
1907
+ .count(),
1908
+ 2
1909
+ );
1910
+ assert_eq!(
1911
+ strokes
1912
+ .iter()
1913
+ .filter(|stroke| **stroke == (keycode::LEFT_CTRL, false))
1914
+ .count(),
1915
+ 2
1916
+ );
1917
+ }
1918
+ inner.apply(ControlMessage::Key {
1919
+ name: Some(String::from("Control")),
1920
+ character: None,
1921
+ down: true,
1922
+ modifiers: Vec::new(),
1923
+ seq: 16,
1924
+ });
1925
+ for seq in [17, 18] {
1926
+ inner.apply(ControlMessage::Key {
1927
+ name: None,
1928
+ character: Some(String::from("c")),
1929
+ down: true,
1930
+ modifiers: vec![String::from("Control")],
1931
+ seq,
1932
+ });
1933
+ }
1934
+ inner.apply(ControlMessage::Key {
1935
+ name: Some(String::from("Control")),
1936
+ character: None,
1937
+ down: false,
1938
+ modifiers: Vec::new(),
1939
+ seq: 19,
1940
+ });
1941
+ assert!(!recorded
1942
+ .lock()
1943
+ .unwrap()
1944
+ .iter()
1945
+ .rev()
1946
+ .take(2)
1947
+ .any(|event| *event == (keycode::LEFT_CTRL, false)));
1948
+ inner.apply(ControlMessage::Key {
1949
+ name: None,
1950
+ character: Some(String::from("c")),
1951
+ down: false,
1952
+ modifiers: Vec::new(),
1953
+ seq: 20,
1954
+ });
1955
+ inner.apply(ControlMessage::Key {
1956
+ name: Some(String::from("ArrowLeft")),
1957
+ character: None,
1958
+ down: true,
1959
+ modifiers: vec![String::from("Shift")],
1960
+ seq: 21,
1961
+ });
1962
+ inner.apply(ControlMessage::Key {
1963
+ name: Some(String::from("ArrowLeft")),
1964
+ character: None,
1965
+ down: false,
1966
+ modifiers: Vec::new(),
1967
+ seq: 22,
1968
+ });
1969
+ let strokes = recorded.lock().unwrap();
1970
+ assert_eq!(
1971
+ strokes
1972
+ .iter()
1973
+ .filter(|stroke| **stroke == (keycode::LEFT_CTRL, true))
1974
+ .count(),
1975
+ 3
1976
+ );
1977
+ assert_eq!(
1978
+ strokes
1979
+ .iter()
1980
+ .filter(|stroke| **stroke == (keycode::LEFT_CTRL, false))
1981
+ .count(),
1982
+ 3
1983
+ );
1984
+ assert_eq!(
1985
+ strokes
1986
+ .iter()
1987
+ .filter(|stroke| **stroke == (keycode::LEFT_SHIFT, false))
1988
+ .count(),
1989
+ 2
1990
+ );
1991
+ }
1992
+
1993
+ #[tokio::test]
1994
+ async fn x11_capture_refuses_both_clipboard_directions() {
1995
+ let (events, _received) = tokio_mpsc::unbounded_channel();
1996
+ let (mut inner, _recorded) = test_inner(events).await;
1997
+ let owned = Arc::get_mut(&mut inner).unwrap();
1998
+ owned.source.source_type = Some(String::from("x11-root"));
1999
+ owned.permissions.push(Permission::Clipboard);
2000
+ let session = Session {
2001
+ inner: inner.clone(),
2002
+ };
2003
+ assert!(session
2004
+ .read_clipboard()
2005
+ .await
2006
+ .unwrap_err()
2007
+ .contains("unavailable"));
2008
+ assert!(session
2009
+ .write_clipboard(String::from("secret"))
2010
+ .await
2011
+ .unwrap_err()
2012
+ .contains("unavailable"));
2013
+ inner.clipboard_read(String::from("read"));
2014
+ inner.clipboard_write(String::from("write"), String::from("secret"));
2015
+ assert_eq!(inner.metrics.lock().unwrap().input_rejected, 2);
2016
+ }
2017
+
2018
+ #[tokio::test]
2019
+ async fn a_frame_the_encoder_refuses_ends_the_session_with_a_reason() {
2020
+ let (events, mut received) = tokio_mpsc::unbounded_channel();
2021
+ let (inner, _recorded) = test_inner(events).await;
2022
+ let (frame_tx, frame_rx) = latest_frame();
2023
+ spawn_pipeline(&inner, frame_rx, Arc::new(AtomicBool::new(true)), 30, 1000);
2024
+
2025
+ // The encoder is 64x64; a 32x32 frame is the dimension mismatch that
2026
+ // used to be counted and dropped behind a permanently black picture.
2027
+ frame_tx.put(I420 {
2028
+ width: 32,
2029
+ height: 32,
2030
+ data: vec![128u8; 32 * 32 + 2 * 16 * 16],
2031
+ });
2032
+ tokio::time::sleep(Duration::from_millis(150)).await;
2033
+
2034
+ let mut reason = None;
2035
+ while let Ok(event) = received.try_recv() {
2036
+ if let Notice {
2037
+ event: SessionEvent::Revoked { reason: why },
2038
+ ..
2039
+ } = event
2040
+ {
2041
+ reason = Some(why);
2042
+ }
2043
+ }
2044
+ let reason = reason.expect("a refused frame must surface as a revocation");
2045
+ assert_eq!(
2046
+ reason, "the encoder rejected a frame",
2047
+ "the reason is a stable token the client can map to copy",
2048
+ );
2049
+ assert!(
2050
+ inner.revoked_reason().is_some(),
2051
+ "the session is closed, not left black"
2052
+ );
2053
+ }
2054
+
2055
+ #[tokio::test]
2056
+ async fn a_lost_transport_releases_input_then_ends_the_session_without_sctp_close() {
2057
+ use webrtc::peer_connection::RTCPeerConnectionState as State;
2058
+ let (events, mut received) = tokio_mpsc::unbounded_channel();
2059
+ let (inner, recorded) = test_inner(events).await;
2060
+ inner.apply(ControlMessage::Key {
2061
+ name: Some(String::from("Control")),
2062
+ character: None,
2063
+ down: true,
2064
+ modifiers: Vec::new(),
2065
+ seq: 1,
2066
+ });
2067
+ assert!(recorded.lock().unwrap().iter().any(|(_, down)| *down));
2068
+
2069
+ let (peer_tx, peer_rx) = tokio_mpsc::unbounded_channel();
2070
+ spawn_peer_events(&inner, peer_rx);
2071
+ peer_tx.send(PeerEvent::State(State::Disconnected)).unwrap();
2072
+ tokio::time::timeout(Duration::from_secs(2), received.recv())
2073
+ .await
2074
+ .unwrap();
2075
+ assert!(
2076
+ recorded.lock().unwrap().iter().any(|(_, down)| !*down),
2077
+ "a disconnected transport must release held input without SCTP OnClose"
2078
+ );
2079
+ assert!(
2080
+ inner.revoked_reason().is_none(),
2081
+ "a transient loss may recover"
2082
+ );
2083
+
2084
+ peer_tx.send(PeerEvent::State(State::Failed)).unwrap();
2085
+ tokio::time::timeout(Duration::from_secs(2), async {
2086
+ while let Some(notice) = received.recv().await {
2087
+ if matches!(
2088
+ notice.event,
2089
+ SessionEvent::State {
2090
+ capture: "ended",
2091
+ ..
2092
+ }
2093
+ ) {
2094
+ return;
2095
+ }
2096
+ }
2097
+ panic!("failed transport did not end capture");
2098
+ })
2099
+ .await
2100
+ .unwrap();
2101
+ assert!(inner.revoked_reason().is_some());
2102
+ assert!(!inner.pipeline.load(Ordering::SeqCst));
2103
+ assert!(inner.input.lock().unwrap().is_none());
2104
+ }
2105
+
2106
+ #[tokio::test]
2107
+ async fn the_pipeline_caps_the_rate_codes_the_last_frame_and_refines_a_still_desktop_once() {
2108
+ let (events, _events_rx) = tokio_mpsc::unbounded_channel();
2109
+ let (inner, _recorded) = test_inner(events).await;
2110
+ let (frame_tx, frame_rx) = latest_frame();
2111
+ spawn_pipeline(&inner, frame_rx, Arc::new(AtomicBool::new(true)), 20, 1000);
2112
+
2113
+ // 40 frames over 200 ms is 200 fps; the requested 20 fps caps what leaves.
2114
+ for _ in 0..40 {
2115
+ frame_tx.put(I420 {
2116
+ width: 64,
2117
+ height: 64,
2118
+ data: vec![128u8; 64 * 64 + 2 * 32 * 32],
2119
+ });
2120
+ tokio::time::sleep(Duration::from_millis(5)).await;
2121
+ }
2122
+ let motion = inner.metrics.lock().unwrap().encoded_frames;
2123
+ assert!(
2124
+ motion <= 7,
2125
+ "the requested rate is a cap, got {motion} of 40"
2126
+ );
2127
+ assert!(
2128
+ motion >= 2,
2129
+ "the pipeline must still emit frames, got {motion}"
2130
+ );
2131
+
2132
+ // Still for longer than a refinement takes: the last frame of the burst
2133
+ // is coded (a refinement only follows a coded frame), then refined once.
2134
+ tokio::time::sleep(Duration::from_millis(450)).await;
2135
+ let metrics = inner.metrics.lock().unwrap().clone();
2136
+ assert_eq!(
2137
+ metrics.refined_frames, 1,
2138
+ "a still desktop is refined exactly once"
2139
+ );
2140
+ assert_eq!(metrics.key_frames, 1, "only the first frame is a key frame");
2141
+ tokio::time::sleep(Duration::from_millis(300)).await;
2142
+ assert_eq!(
2143
+ inner.metrics.lock().unwrap().encoded_frames,
2144
+ metrics.encoded_frames,
2145
+ "nothing is sent while still"
2146
+ );
2147
+
2148
+ // A still stream is kept alive, so a receiver never waits long enough
2149
+ // to ask for a key frame; the keepalive is neither a key frame nor
2150
+ // another refinement.
2151
+ tokio::time::sleep(Duration::from_millis(900)).await;
2152
+ let later = inner.metrics.lock().unwrap().clone();
2153
+ assert!(
2154
+ later.encoded_frames > metrics.encoded_frames,
2155
+ "a still stream still sends"
2156
+ );
2157
+ assert_eq!((later.key_frames, later.refined_frames), (1, 1));
2158
+ }
2159
+ }