@desklink/host 0.5.4 → 0.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -2
- package/dist/bridge.js +3 -3
- package/dist/bridge.js.map +1 -1
- package/dist/engineProcess.d.ts.map +1 -1
- package/dist/engineProcess.js +1 -0
- package/dist/engineProcess.js.map +1 -1
- package/dist/protocol.d.ts +39 -0
- package/dist/protocol.d.ts.map +1 -1
- package/dist/protocol.js.map +1 -1
- package/docs/BOUNDARY.md +4 -2
- package/docs/PROTOCOL.md +95 -9
- package/engine/Cargo.lock +1 -1
- package/engine/Cargo.toml +1 -1
- package/engine/src/audio.rs +398 -0
- package/engine/src/capture.rs +0 -1
- package/engine/src/hyprland.rs +319 -0
- package/engine/src/mac.rs +1 -0
- package/engine/src/main.rs +24 -0
- package/engine/src/peer.rs +73 -0
- package/engine/src/protocol.rs +22 -0
- package/engine/src/session.rs +240 -4
- package/engine/src/win.rs +1 -0
- package/engine/vendor/rtc-ice/DESKLINK-PATCH.md +53 -1
- package/engine/vendor/rtc-ice/src/agent/agent_selector.rs +81 -0
- package/engine/vendor/rtc-ice/src/agent/agent_test.rs +206 -0
- package/engine/vendor/rtc-ice/src/agent/mod.rs +91 -0
- package/engine/vendor/rtc-ice/src/dl_trace.rs +49 -0
- package/engine/vendor/rtc-ice/src/lib.rs +3 -0
- package/package.json +3 -3
package/engine/src/session.rs
CHANGED
|
@@ -93,6 +93,10 @@ pub struct Metrics {
|
|
|
93
93
|
/// Control messages forwarded back to the consumer that owns the device,
|
|
94
94
|
/// instead of being applied to a desktop this engine holds.
|
|
95
95
|
pub input_forwarded: u64,
|
|
96
|
+
pub audio_frames: u64,
|
|
97
|
+
pub audio_bytes: u64,
|
|
98
|
+
pub audio_dropped_bytes: u64,
|
|
99
|
+
pub audio_gap_micros: u64,
|
|
96
100
|
}
|
|
97
101
|
|
|
98
102
|
fn tile_hashes(raw: &[u8], width: usize, height: usize) -> Vec<u64> {
|
|
@@ -708,6 +712,8 @@ pub struct OpenRequest {
|
|
|
708
712
|
/// Keep the latest frame and its tile hashes for `session.frame` and
|
|
709
713
|
/// `session.frame.changed`; off, a frame is converted once, for the encoder.
|
|
710
714
|
pub local_frames: bool,
|
|
715
|
+
/// Capture the desktop's own system audio for `session.audio`.
|
|
716
|
+
pub audio: bool,
|
|
711
717
|
}
|
|
712
718
|
|
|
713
719
|
/// Events a session raises for its consumer.
|
|
@@ -736,6 +742,14 @@ pub enum SessionEvent {
|
|
|
736
742
|
seq: u64,
|
|
737
743
|
damage: Vec<[usize; 4]>,
|
|
738
744
|
},
|
|
745
|
+
/// One chunk was captured, even if the full queue dropped it, on the same
|
|
746
|
+
/// clock `session.frame.changed` uses.
|
|
747
|
+
Audio {
|
|
748
|
+
seq: u64,
|
|
749
|
+
timestamp_us: u64,
|
|
750
|
+
bytes: u64,
|
|
751
|
+
gap_micros: u64,
|
|
752
|
+
},
|
|
739
753
|
/// One control-channel message from the client, for a source whose device
|
|
740
754
|
/// the consumer drives. The engine applied nothing locally.
|
|
741
755
|
Input {
|
|
@@ -802,6 +816,11 @@ struct Inner {
|
|
|
802
816
|
point_overlay: Mutex<Option<crate::indicator::Indicator>>,
|
|
803
817
|
#[cfg(target_os = "linux")]
|
|
804
818
|
point_desktop: PointDesktop,
|
|
819
|
+
/// A portal window share's place on a Hyprland desktop; absent elsewhere,
|
|
820
|
+
/// where a shared window's taps are refused. Dropped on close, which
|
|
821
|
+
/// closes its Hyprland event socket.
|
|
822
|
+
#[cfg(target_os = "linux")]
|
|
823
|
+
window: Mutex<Option<crate::hyprland::WindowLocator>>,
|
|
805
824
|
capture: Mutex<Option<FrameSource>>,
|
|
806
825
|
/// Absent for an encoded source: there is no local keyboard to translate a
|
|
807
826
|
/// character through, because the client's keys are forwarded instead.
|
|
@@ -817,6 +836,9 @@ struct Inner {
|
|
|
817
836
|
encoded: Mutex<Option<Encoded>>,
|
|
818
837
|
/// `OpenRequest::local_frames`: whether `latest` is kept at all.
|
|
819
838
|
local_frames: bool,
|
|
839
|
+
/// Dropped on close, stopping the PipeWire stream.
|
|
840
|
+
#[cfg(target_os = "linux")]
|
|
841
|
+
audio: Mutex<Option<crate::audio::AudioCapture>>,
|
|
820
842
|
cursor_info: serde_json::Value,
|
|
821
843
|
}
|
|
822
844
|
|
|
@@ -869,6 +891,9 @@ pub fn capabilities() -> serde_json::Value {
|
|
|
869
891
|
.map(|desktop| serde_json::json!([desktop.screen_size().0, desktop.screen_size().1]))
|
|
870
892
|
.unwrap_or(serde_json::Value::Null);
|
|
871
893
|
let clipboard = wayland_clipboard_available();
|
|
894
|
+
// Asked, not taken: the probe connects to this session's PipeWire, reads the
|
|
895
|
+
// registry and leaves. No portal, no prompt, no capture.
|
|
896
|
+
let audio = crate::audio::probe();
|
|
872
897
|
let (input, grant_state, unavailable) = match crate::input::probe() {
|
|
873
898
|
Ok(()) => (serde_json::json!(true), "granted", serde_json::Value::Null),
|
|
874
899
|
Err(unavailable) => (
|
|
@@ -896,7 +921,9 @@ pub fn capabilities() -> serde_json::Value {
|
|
|
896
921
|
"cursor": "embedded",
|
|
897
922
|
"cursor_modes": ["embedded", "hidden"],
|
|
898
923
|
"cursor_positions": {"x11": true, "wayland": "ext-image-copy-capture-v1"},
|
|
899
|
-
"audio":
|
|
924
|
+
"audio": audio.is_some(),
|
|
925
|
+
"audio_formats": crate::audio::FORMATS,
|
|
926
|
+
"audio_source": audio.map(|source| source.name),
|
|
900
927
|
},
|
|
901
928
|
"encode": encode_capabilities(),
|
|
902
929
|
"input": {
|
|
@@ -949,6 +976,7 @@ pub fn capabilities() -> serde_json::Value {
|
|
|
949
976
|
"formats": ["bgra"],
|
|
950
977
|
"cursor": "embedded",
|
|
951
978
|
"cursor_modes": ["embedded"],
|
|
979
|
+
// ScreenCaptureKit audio is not wired up; macOS is out of scope.
|
|
952
980
|
"audio": false,
|
|
953
981
|
"displays": crate::mac::displays(),
|
|
954
982
|
"grant": if capture_granted { "granted" } else { "missing-screen-recording" },
|
|
@@ -1007,6 +1035,13 @@ impl Session {
|
|
|
1007
1035
|
request: OpenRequest,
|
|
1008
1036
|
events: tokio_mpsc::UnboundedSender<Notice>,
|
|
1009
1037
|
) -> std::result::Result<Self, SessionError> {
|
|
1038
|
+
#[cfg(not(target_os = "linux"))]
|
|
1039
|
+
if request.audio {
|
|
1040
|
+
return Err(SessionError::new(
|
|
1041
|
+
"audio-unavailable",
|
|
1042
|
+
"system audio capture requires Linux PipeWire",
|
|
1043
|
+
));
|
|
1044
|
+
}
|
|
1010
1045
|
if request.cursor == crate::protocol::CursorMode::Hidden {
|
|
1011
1046
|
#[cfg(not(target_os = "linux"))]
|
|
1012
1047
|
return Err(SessionError::new(
|
|
@@ -1165,6 +1200,8 @@ impl Session {
|
|
|
1165
1200
|
None
|
|
1166
1201
|
};
|
|
1167
1202
|
let mut cursor_info = serde_json::json!({"mode":"embedded", "positions":false});
|
|
1203
|
+
#[cfg(target_os = "linux")]
|
|
1204
|
+
let mut share_watch = None;
|
|
1168
1205
|
let Selected {
|
|
1169
1206
|
source,
|
|
1170
1207
|
capture,
|
|
@@ -1195,6 +1232,9 @@ impl Session {
|
|
|
1195
1232
|
_ => {
|
|
1196
1233
|
#[cfg(target_os = "linux")]
|
|
1197
1234
|
{
|
|
1235
|
+
// Hyprland names the window it shares only on its
|
|
1236
|
+
// event socket, as the share starts.
|
|
1237
|
+
share_watch = crate::hyprland::ShareWatch::start();
|
|
1198
1238
|
portal::open(request.restore_token.as_deref(), request.cursor).await
|
|
1199
1239
|
}
|
|
1200
1240
|
#[cfg(not(target_os = "linux"))]
|
|
@@ -1324,6 +1364,10 @@ impl Session {
|
|
|
1324
1364
|
let source_w = source.width.max(1) as usize;
|
|
1325
1365
|
let source_h = source.height.max(1) as usize;
|
|
1326
1366
|
let (width, height) = fit(source_w, source_h, request.max_width, request.max_height);
|
|
1367
|
+
#[cfg(target_os = "linux")]
|
|
1368
|
+
let window = share_watch
|
|
1369
|
+
.filter(|_| source.source_type.as_deref() == Some("Window"))
|
|
1370
|
+
.map(|watch| crate::hyprland::WindowLocator::new(watch, source.width, source.height));
|
|
1327
1371
|
|
|
1328
1372
|
let bitrate_kbps = if request.bitrate_kbps == 0 {
|
|
1329
1373
|
auto_bitrate(width, height)
|
|
@@ -1370,6 +1414,14 @@ impl Session {
|
|
|
1370
1414
|
SessionError::new("input", format!("no keyboard layout: {error:#}"))
|
|
1371
1415
|
})?;
|
|
1372
1416
|
|
|
1417
|
+
// Refuse before allocating the peer; never silently downgrade.
|
|
1418
|
+
#[cfg(target_os = "linux")]
|
|
1419
|
+
let audio = if request.audio {
|
|
1420
|
+
Some(start_audio(&id, &events)?)
|
|
1421
|
+
} else {
|
|
1422
|
+
None
|
|
1423
|
+
};
|
|
1424
|
+
|
|
1373
1425
|
// H.264 is offered beside VP9 only when an encoder for it starts on
|
|
1374
1426
|
// this machine; the answer then decides which one is coded.
|
|
1375
1427
|
let codec = if !crate::h264::fits(width, height) {
|
|
@@ -1431,8 +1483,12 @@ impl Session {
|
|
|
1431
1483
|
} else {
|
|
1432
1484
|
PointDesktop::Wayland
|
|
1433
1485
|
},
|
|
1486
|
+
#[cfg(target_os = "linux")]
|
|
1487
|
+
window: Mutex::new(window),
|
|
1434
1488
|
capture: Mutex::new(Some(capture)),
|
|
1435
1489
|
layout: Mutex::new(Some(layout)),
|
|
1490
|
+
#[cfg(target_os = "linux")]
|
|
1491
|
+
audio: Mutex::new(audio),
|
|
1436
1492
|
last_seq: Mutex::new(0),
|
|
1437
1493
|
control_open: AtomicBool::new(false),
|
|
1438
1494
|
closed: AtomicBool::new(false),
|
|
@@ -1474,6 +1530,12 @@ impl Session {
|
|
|
1474
1530
|
height: usize,
|
|
1475
1531
|
events: tokio_mpsc::UnboundedSender<Notice>,
|
|
1476
1532
|
) -> std::result::Result<Self, SessionError> {
|
|
1533
|
+
if request.audio {
|
|
1534
|
+
return Err(SessionError::new(
|
|
1535
|
+
"operation",
|
|
1536
|
+
"an encoded source carries video the consumer already encoded; there is no local desktop audio to capture",
|
|
1537
|
+
));
|
|
1538
|
+
}
|
|
1477
1539
|
if width == 0 || height == 0 || width > 7680 || height > 4320 {
|
|
1478
1540
|
return Err(SessionError::new(
|
|
1479
1541
|
"source",
|
|
@@ -1521,8 +1583,12 @@ impl Session {
|
|
|
1521
1583
|
point_overlay: Mutex::new(None),
|
|
1522
1584
|
#[cfg(target_os = "linux")]
|
|
1523
1585
|
point_desktop: PointDesktop::Unavailable("layer_shell_unavailable"),
|
|
1586
|
+
#[cfg(target_os = "linux")]
|
|
1587
|
+
window: Mutex::new(None),
|
|
1524
1588
|
capture: Mutex::new(None),
|
|
1525
1589
|
layout: Mutex::new(None),
|
|
1590
|
+
#[cfg(target_os = "linux")]
|
|
1591
|
+
audio: Mutex::new(None),
|
|
1526
1592
|
last_seq: Mutex::new(0),
|
|
1527
1593
|
control_open: AtomicBool::new(false),
|
|
1528
1594
|
closed: AtomicBool::new(false),
|
|
@@ -1850,11 +1916,92 @@ impl Session {
|
|
|
1850
1916
|
}
|
|
1851
1917
|
|
|
1852
1918
|
pub fn metrics(&self) -> Metrics {
|
|
1853
|
-
self
|
|
1919
|
+
let mut metrics = self
|
|
1920
|
+
.inner
|
|
1854
1921
|
.metrics
|
|
1855
1922
|
.lock()
|
|
1856
1923
|
.map(|m| m.clone())
|
|
1857
|
-
.unwrap_or_default()
|
|
1924
|
+
.unwrap_or_default();
|
|
1925
|
+
#[cfg(target_os = "linux")]
|
|
1926
|
+
if let Some(audio) = lock(&self.inner.audio).as_ref() {
|
|
1927
|
+
let (frames, bytes, dropped_bytes, gap_micros) = audio.totals();
|
|
1928
|
+
metrics.audio_frames = frames;
|
|
1929
|
+
metrics.audio_bytes = bytes;
|
|
1930
|
+
metrics.audio_dropped_bytes = dropped_bytes;
|
|
1931
|
+
metrics.audio_gap_micros = gap_micros;
|
|
1932
|
+
}
|
|
1933
|
+
metrics
|
|
1934
|
+
}
|
|
1935
|
+
|
|
1936
|
+
pub fn audio_info(&self) -> Option<serde_json::Value> {
|
|
1937
|
+
#[cfg(target_os = "linux")]
|
|
1938
|
+
{
|
|
1939
|
+
lock(&self.inner.audio).as_ref()?; // start() accepted the fixed format.
|
|
1940
|
+
return Some(serde_json::json!({
|
|
1941
|
+
"source": "pipewire-monitor", "format": crate::audio::FORMAT,
|
|
1942
|
+
"sample_rate": crate::audio::SAMPLE_RATE, "channels": crate::audio::CHANNELS,
|
|
1943
|
+
}));
|
|
1944
|
+
}
|
|
1945
|
+
#[cfg(not(target_os = "linux"))]
|
|
1946
|
+
None
|
|
1947
|
+
}
|
|
1948
|
+
|
|
1949
|
+
/// Consume one queued PCM chunk; samples never travel through JSON.
|
|
1950
|
+
#[cfg(target_os = "linux")]
|
|
1951
|
+
pub fn audio(
|
|
1952
|
+
&self,
|
|
1953
|
+
path: &str,
|
|
1954
|
+
max_bytes: usize,
|
|
1955
|
+
) -> std::result::Result<serde_json::Value, SessionError> {
|
|
1956
|
+
let no_audio = if lock(&self.inner.encoded).is_some() {
|
|
1957
|
+
"this session forwards an encoded stream and has no local audio"
|
|
1958
|
+
} else {
|
|
1959
|
+
"this session was opened without audio"
|
|
1960
|
+
};
|
|
1961
|
+
let held = lock(&self.inner.audio);
|
|
1962
|
+
let capture = held
|
|
1963
|
+
.as_ref()
|
|
1964
|
+
.ok_or_else(|| SessionError::new("operation", no_audio))?;
|
|
1965
|
+
let frame_bytes = crate::audio::BYTES_PER_FRAME as usize;
|
|
1966
|
+
if max_bytes < frame_bytes {
|
|
1967
|
+
return Err(SessionError::new(
|
|
1968
|
+
"malformed",
|
|
1969
|
+
"max_bytes must hold at least one sample frame",
|
|
1970
|
+
));
|
|
1971
|
+
}
|
|
1972
|
+
let chunk = capture
|
|
1973
|
+
.take()
|
|
1974
|
+
.ok_or_else(|| SessionError::new("audio", "no audio chunk has arrived yet"))?;
|
|
1975
|
+
let wanted = (max_bytes / frame_bytes * frame_bytes).min(chunk.bytes.len());
|
|
1976
|
+
let written = if path.is_empty() { 0 } else { wanted };
|
|
1977
|
+
let result = serde_json::json!({
|
|
1978
|
+
"seq": chunk.seq,
|
|
1979
|
+
"timestamp_us": crate::protocol::timestamp_us(chunk.at),
|
|
1980
|
+
"frames": written as u64 / crate::audio::BYTES_PER_FRAME,
|
|
1981
|
+
"bytes": written,
|
|
1982
|
+
"format": crate::audio::FORMAT,
|
|
1983
|
+
"sample_rate": crate::audio::SAMPLE_RATE,
|
|
1984
|
+
"channels": crate::audio::CHANNELS,
|
|
1985
|
+
"gap_ms": chunk.gap_micros / 1000,
|
|
1986
|
+
"written": !path.is_empty(),
|
|
1987
|
+
});
|
|
1988
|
+
if written > 0 {
|
|
1989
|
+
std::fs::write(path, &chunk.bytes[..written])
|
|
1990
|
+
.map_err(|error| SessionError::new("audio", error.to_string()))?;
|
|
1991
|
+
}
|
|
1992
|
+
Ok(result)
|
|
1993
|
+
}
|
|
1994
|
+
|
|
1995
|
+
#[cfg(not(target_os = "linux"))]
|
|
1996
|
+
pub fn audio(
|
|
1997
|
+
&self,
|
|
1998
|
+
_path: &str,
|
|
1999
|
+
_max_bytes: usize,
|
|
2000
|
+
) -> std::result::Result<serde_json::Value, SessionError> {
|
|
2001
|
+
Err(SessionError::new(
|
|
2002
|
+
"operation",
|
|
2003
|
+
"this platform has no system audio capture",
|
|
2004
|
+
))
|
|
1858
2005
|
}
|
|
1859
2006
|
|
|
1860
2007
|
pub async fn accept_answer(&self, sdp: String) -> Result<()> {
|
|
@@ -1987,6 +2134,10 @@ impl Inner {
|
|
|
1987
2134
|
// out of scope, so the consumer can truthfully say the desktop stopped.
|
|
1988
2135
|
let capture = self.capture.lock().ok().and_then(|mut held| held.take());
|
|
1989
2136
|
drop(capture);
|
|
2137
|
+
#[cfg(target_os = "linux")]
|
|
2138
|
+
drop(self.audio.lock().ok().and_then(|mut held| held.take()));
|
|
2139
|
+
#[cfg(target_os = "linux")]
|
|
2140
|
+
drop(lock(&self.window).take());
|
|
1990
2141
|
self.notify(SessionEvent::State {
|
|
1991
2142
|
capture: "ended",
|
|
1992
2143
|
transport: String::from("closed"),
|
|
@@ -2112,10 +2263,33 @@ impl Inner {
|
|
|
2112
2263
|
self.forward(message);
|
|
2113
2264
|
return;
|
|
2114
2265
|
}
|
|
2266
|
+
// A shared window's place on the desktop is unknown (the portal reports
|
|
2267
|
+
// no origin for it), while the pointer device spans the whole desktop:
|
|
2268
|
+
// a tap would land somewhere the person never pointed. Only Hyprland
|
|
2269
|
+
// says where the window is.
|
|
2270
|
+
#[cfg(target_os = "linux")]
|
|
2271
|
+
let located = lock(&self.window).is_some();
|
|
2272
|
+
#[cfg(not(target_os = "linux"))]
|
|
2273
|
+
let located = false;
|
|
2274
|
+
let feedback = if located {
|
|
2275
|
+
feedback.filter(|(kind, ..)| *kind == 'T')
|
|
2276
|
+
} else {
|
|
2277
|
+
feedback
|
|
2278
|
+
};
|
|
2279
|
+
if self.source.source_type.as_deref() == Some("Window")
|
|
2280
|
+
&& !located
|
|
2281
|
+
&& matches!(&message, ControlMessage::Pointer { phase, .. } if !matches!(phase, PointerPhase::Cancel))
|
|
2282
|
+
{
|
|
2283
|
+
self.reject(seq, "input-unavailable", WINDOW_POINTER_REFUSAL);
|
|
2284
|
+
return;
|
|
2285
|
+
}
|
|
2286
|
+
// A located window's pointer moves through the compositor; only its
|
|
2287
|
+
// buttons need the input device, which `pointer` checks first.
|
|
2115
2288
|
if !matches!(
|
|
2116
2289
|
&message,
|
|
2117
2290
|
ControlMessage::ClipboardRead { .. } | ControlMessage::ClipboardWrite { .. }
|
|
2118
|
-
) &&
|
|
2291
|
+
) && !(located && matches!(&message, ControlMessage::Pointer { .. }))
|
|
2292
|
+
&& lock(&self.input).is_none()
|
|
2119
2293
|
{
|
|
2120
2294
|
self.reject(seq, "input-unavailable", "no input backend");
|
|
2121
2295
|
return;
|
|
@@ -2240,6 +2414,28 @@ impl Inner {
|
|
|
2240
2414
|
// created for, and clamps to, the source's own pixels. One conversion
|
|
2241
2415
|
// here keeps every caller honest.
|
|
2242
2416
|
let (x, y) = to_source_pixels(x, y, (width, height), &self.source);
|
|
2417
|
+
#[cfg(target_os = "linux")]
|
|
2418
|
+
if let Some(window) = lock(&self.window).as_ref() {
|
|
2419
|
+
let place = || {
|
|
2420
|
+
window
|
|
2421
|
+
.place(x, y)
|
|
2422
|
+
.map_err(|message| ("input-unavailable", message))
|
|
2423
|
+
};
|
|
2424
|
+
return match phase {
|
|
2425
|
+
PointerPhase::Move => place(),
|
|
2426
|
+
PointerPhase::Down => {
|
|
2427
|
+
// Refused before the pointer moves: nothing could press.
|
|
2428
|
+
self.with_input(|_| Ok(()))?;
|
|
2429
|
+
place()?;
|
|
2430
|
+
self.with_input(|target| target.button(Button::from_number(button), true))
|
|
2431
|
+
}
|
|
2432
|
+
PointerPhase::Up => {
|
|
2433
|
+
self.with_input(|target| target.button(Button::from_number(button), false))?;
|
|
2434
|
+
place()
|
|
2435
|
+
}
|
|
2436
|
+
PointerPhase::Cancel => self.with_input(InputTarget::release_all),
|
|
2437
|
+
};
|
|
2438
|
+
}
|
|
2243
2439
|
self.with_input(|target| match phase {
|
|
2244
2440
|
PointerPhase::Move => target.move_absolute(x, y),
|
|
2245
2441
|
PointerPhase::Down => {
|
|
@@ -2445,6 +2641,27 @@ fn frame_interval(max_fps: u32) -> Duration {
|
|
|
2445
2641
|
Duration::from_secs_f64(1.0 / max_fps.max(1) as f64)
|
|
2446
2642
|
}
|
|
2447
2643
|
|
|
2644
|
+
#[cfg(target_os = "linux")]
|
|
2645
|
+
fn start_audio(
|
|
2646
|
+
id: &str,
|
|
2647
|
+
events: &tokio_mpsc::UnboundedSender<Notice>,
|
|
2648
|
+
) -> std::result::Result<crate::audio::AudioCapture, SessionError> {
|
|
2649
|
+
let events = events.clone();
|
|
2650
|
+
let id = id.to_owned();
|
|
2651
|
+
crate::audio::AudioCapture::start(Box::new(move |notice| {
|
|
2652
|
+
let _ = events.send(Notice {
|
|
2653
|
+
session_id: id.clone(),
|
|
2654
|
+
event: SessionEvent::Audio {
|
|
2655
|
+
seq: notice.seq,
|
|
2656
|
+
timestamp_us: crate::protocol::timestamp_us(notice.at),
|
|
2657
|
+
bytes: notice.bytes,
|
|
2658
|
+
gap_micros: notice.gap_micros,
|
|
2659
|
+
},
|
|
2660
|
+
});
|
|
2661
|
+
}))
|
|
2662
|
+
.map_err(|reason| SessionError::new("audio-unavailable", reason))
|
|
2663
|
+
}
|
|
2664
|
+
|
|
2448
2665
|
/// The newest captured frame and when it was captured, handed from the capture
|
|
2449
2666
|
/// thread to the encoder. A newer frame replaces one the encoder has not taken,
|
|
2450
2667
|
/// so however far behind it falls, what it codes next is the desktop as it is
|
|
@@ -3243,6 +3460,14 @@ fn spawn_peer_events(inner: &Arc<Inner>, mut events: tokio_mpsc::UnboundedReceiv
|
|
|
3243
3460
|
first_frame: inner.metrics.lock().map(|m| m.encoded_frames).unwrap_or(0)
|
|
3244
3461
|
> 0,
|
|
3245
3462
|
});
|
|
3463
|
+
// desklink scout trace: the session's transport emit, one
|
|
3464
|
+
// line after the peer event that caused it.
|
|
3465
|
+
if crate::peer::dl_trace_on() {
|
|
3466
|
+
crate::peer::dl_eng!(
|
|
3467
|
+
"session_transport_emit {}",
|
|
3468
|
+
format!("{state}").to_lowercase()
|
|
3469
|
+
);
|
|
3470
|
+
}
|
|
3246
3471
|
}
|
|
3247
3472
|
PeerEvent::ControlOpen => {
|
|
3248
3473
|
inner.control_open.store(true, Ordering::SeqCst);
|
|
@@ -3301,6 +3526,8 @@ fn available_parallelism() -> usize {
|
|
|
3301
3526
|
.unwrap_or(4)
|
|
3302
3527
|
}
|
|
3303
3528
|
|
|
3529
|
+
pub(crate) const WINDOW_POINTER_REFUSAL: &str = "a shared window cannot take taps: its place on the desktop is unknown; share the whole screen to point and click";
|
|
3530
|
+
|
|
3304
3531
|
/// Encoded-surface pixels to the source's own pixels. The client aims at what it
|
|
3305
3532
|
/// sees; the applier is sized and clamped to the source, so the layout origin
|
|
3306
3533
|
/// must not be added here.
|
|
@@ -3415,8 +3642,12 @@ mod tests {
|
|
|
3415
3642
|
point_overlay: Mutex::new(None),
|
|
3416
3643
|
#[cfg(target_os = "linux")]
|
|
3417
3644
|
point_desktop: PointDesktop::Unavailable("layer_shell_unavailable"),
|
|
3645
|
+
#[cfg(target_os = "linux")]
|
|
3646
|
+
window: Mutex::new(None),
|
|
3418
3647
|
capture: Mutex::new(None),
|
|
3419
3648
|
layout: Mutex::new(None),
|
|
3649
|
+
#[cfg(target_os = "linux")]
|
|
3650
|
+
audio: Mutex::new(None),
|
|
3420
3651
|
last_seq: Mutex::new(0),
|
|
3421
3652
|
control_open: AtomicBool::new(false),
|
|
3422
3653
|
closed: AtomicBool::new(false),
|
|
@@ -3672,7 +3903,11 @@ mod tests {
|
|
|
3672
3903
|
point_overlay: Mutex::new(None),
|
|
3673
3904
|
#[cfg(target_os = "linux")]
|
|
3674
3905
|
point_desktop: PointDesktop::Unavailable("layer_shell_unavailable"),
|
|
3906
|
+
#[cfg(target_os = "linux")]
|
|
3907
|
+
window: Mutex::new(None),
|
|
3675
3908
|
layout: Mutex::new(Some(Layout::from_environment().expect("a keymap"))),
|
|
3909
|
+
#[cfg(target_os = "linux")]
|
|
3910
|
+
audio: Mutex::new(None),
|
|
3676
3911
|
last_seq: Mutex::new(0),
|
|
3677
3912
|
control_open: AtomicBool::new(true),
|
|
3678
3913
|
closed: AtomicBool::new(false),
|
|
@@ -4080,6 +4315,7 @@ mod tests {
|
|
|
4080
4315
|
loopback_tcp: false,
|
|
4081
4316
|
agent_indicator: false,
|
|
4082
4317
|
local_frames: true,
|
|
4318
|
+
audio: false,
|
|
4083
4319
|
}
|
|
4084
4320
|
}
|
|
4085
4321
|
|
package/engine/src/win.rs
CHANGED
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
# desklink patch to vendored `rtc-ice` 0.21.0
|
|
2
2
|
|
|
3
|
+
Two shapes, both for the roam-persistence goal: keep media flowing across an
|
|
4
|
+
ICE restart, and nominate the restarted generation without waiting for a check
|
|
5
|
+
tick.
|
|
6
|
+
|
|
3
7
|
## What
|
|
4
8
|
|
|
9
|
+
### 1. Media keeps flowing on the previous pair across a restart
|
|
10
|
+
|
|
5
11
|
`Agent::apply_restart` (src/agent/mod.rs) no longer interrupts media that is
|
|
6
12
|
already flowing on the current selected ICE pair. When a restart is applied, the
|
|
7
13
|
pair media was using is remembered (`restart_media_pair`), and
|
|
@@ -11,6 +17,19 @@ local credentials are installed, remote credentials are dropped, binding
|
|
|
11
17
|
requests and candidate pairs are cleared, the agent re-checks, and the
|
|
12
18
|
connection state goes to `Checking`.
|
|
13
19
|
|
|
20
|
+
### 2. The nomination does not wait for the next check tick
|
|
21
|
+
|
|
22
|
+
`ControllingSelector::handle_success_response` (src/agent/agent_selector.rs)
|
|
23
|
+
requests an immediate check pass when the response makes a pair `Succeeded` and
|
|
24
|
+
that pair is the one the next pass would nominate. Nomination is decided in
|
|
25
|
+
`contact_candidates`, which the driver only reaches on a `check_interval` tick
|
|
26
|
+
(200 ms) or when an inbound Binding request happens to arrive while a valid pair
|
|
27
|
+
already exists; a pair that becomes valid just after a tick therefore waits a
|
|
28
|
+
full tick for its `USE-CANDIDATE`. The request uses
|
|
29
|
+
`Agent::request_connectivity_check()`, the deferred-check mechanism added for
|
|
30
|
+
issue #88, so `contact()` is not called from the read path and `poll_timeout`
|
|
31
|
+
reports an already-due deadline instead.
|
|
32
|
+
|
|
14
33
|
Every changed spot is marked with a `// desklink patch:` comment.
|
|
15
34
|
|
|
16
35
|
## Why
|
|
@@ -33,8 +52,20 @@ The behavior is pinned by the engine integration test
|
|
|
33
52
|
`a_restart_keeps_media_flowing_on_the_selected_pair_until_the_new_one_nominates`
|
|
34
53
|
in `packages/desktop-host/engine/src/peer.rs`.
|
|
35
54
|
|
|
55
|
+
Shape 2 exists because the tick is the engine's largest single bounded cost
|
|
56
|
+
between a restart offer and `transport=connected`: in the routed roam leg the
|
|
57
|
+
engine's first pair became valid 332 ms after the answer and its `USE-CANDIDATE`
|
|
58
|
+
left 197 ms later — one 200 ms tick. A peer that runs no connectivity checks of
|
|
59
|
+
its own leaves an inbound request unavailable, and the nomination then waits for
|
|
60
|
+
the tick. The engine integration test asserts the reconnect bound; the loopback
|
|
61
|
+
harness cannot grade sub-tick timing on its own (its client answers checks and
|
|
62
|
+
sends triggered checks in the same burst, so an inbound request is available
|
|
63
|
+
almost immediately), which is why the field measurement is the reference.
|
|
64
|
+
|
|
36
65
|
## Scope of the change
|
|
37
66
|
|
|
67
|
+
### Shape 1
|
|
68
|
+
|
|
38
69
|
- `Agent` gains `restart_media_pair: Option<(Candidate, Candidate)>`, initialised
|
|
39
70
|
to `None`.
|
|
40
71
|
- `apply_restart` clones the current `(local, remote)` into it before the
|
|
@@ -45,6 +76,16 @@ in `packages/desktop-host/engine/src/peer.rs`.
|
|
|
45
76
|
- `update_connection_state` clears it on `Failed` / `Closed`, so a restart that
|
|
46
77
|
never nominates does not keep media pointed at an unreachable pair.
|
|
47
78
|
|
|
79
|
+
### Shape 2
|
|
80
|
+
|
|
81
|
+
- `ControllingSelector::handle_success_response` calls
|
|
82
|
+
`request_connectivity_check()` after a pair becomes `Succeeded`, when no pair is
|
|
83
|
+
selected or nominated yet, this pair is the best valid pair, and both of its
|
|
84
|
+
candidates have passed their acceptance wait (`is_nominatable`).
|
|
85
|
+
- Nothing else changes: the same `contact_candidates` pass decides the
|
|
86
|
+
nomination, the acceptance waits still gate it, and a nomination already in
|
|
87
|
+
flight is not re-requested, so the extra pass cannot repeat per response.
|
|
88
|
+
|
|
48
89
|
No public API changed; no signalling or wire change.
|
|
49
90
|
|
|
50
91
|
## How to drop the patch
|
|
@@ -55,5 +96,16 @@ No public API changed; no signalling or wire change.
|
|
|
55
96
|
3. Update `packages/desktop-host/engine/Cargo.lock` (`cargo update -p rtc-ice`).
|
|
56
97
|
|
|
57
98
|
Do this once upstream rtc-ice keeps the previous selected pair across a restart
|
|
58
|
-
on its own
|
|
99
|
+
on its own and nominates a freshly validated pair without waiting for the next
|
|
100
|
+
check tick (an upstream PR is a follow-up; it needs the captain's word first,
|
|
59
101
|
since it would carry his name).
|
|
102
|
+
|
|
103
|
+
## Dev-only measurement instrumentation (not part of the patch)
|
|
104
|
+
|
|
105
|
+
`src/dl_trace.rs` and its call sites — marked `// desklink scout trace:` — are
|
|
106
|
+
a private-lab instrument, not a shipped behaviour change. They are off unless
|
|
107
|
+
`DESKLINK_ICE_TRACE=1` is set, and when on they only write `[ice <us>]` lines to
|
|
108
|
+
stderr, which the bridge forwards untouched. To drop them: delete
|
|
109
|
+
`src/dl_trace.rs`, remove `pub(crate) mod dl_trace;` from `src/lib.rs`, and
|
|
110
|
+
remove the marked blocks from `src/agent/mod.rs` and
|
|
111
|
+
`src/agent/agent_selector.rs` (grep `desklink scout trace`).
|
|
@@ -10,6 +10,9 @@ use stun::textattrs::*;
|
|
|
10
10
|
|
|
11
11
|
use crate::attributes::{control::*, priority::*, use_candidate::*};
|
|
12
12
|
use crate::candidate::{candidate_pair::*, *};
|
|
13
|
+
// desklink scout trace (dl-roam-postnom-attr):
|
|
14
|
+
use crate::dl_trace;
|
|
15
|
+
use crate::dl_trace::dl_ice;
|
|
13
16
|
|
|
14
17
|
trait ControllingSelector {
|
|
15
18
|
fn start(&mut self, now: Instant);
|
|
@@ -133,6 +136,20 @@ impl Agent {
|
|
|
133
136
|
};
|
|
134
137
|
|
|
135
138
|
if let Some((msg, local, remote)) = result {
|
|
139
|
+
// desklink scout trace: a USE-CANDIDATE (nomination) request going
|
|
140
|
+
// out — the first transmission, or a tick-gated retransmission
|
|
141
|
+
// while the previous one is still unacknowledged.
|
|
142
|
+
if dl_trace::enabled() {
|
|
143
|
+
dl_ice!(
|
|
144
|
+
"{} NOMINATE txn={:?} local={} remote={} nominated_pair={:?} outstanding={}",
|
|
145
|
+
self.get_name(),
|
|
146
|
+
msg.transaction_id,
|
|
147
|
+
self.local_candidates[local].addr(),
|
|
148
|
+
self.remote_candidates[remote].addr(),
|
|
149
|
+
self.nominated_pair,
|
|
150
|
+
self.pending_binding_requests.len(),
|
|
151
|
+
);
|
|
152
|
+
}
|
|
136
153
|
self.send_binding_request(now, &msg, local, remote);
|
|
137
154
|
}
|
|
138
155
|
}
|
|
@@ -222,6 +239,27 @@ impl ControllingSelector for Agent {
|
|
|
222
239
|
|
|
223
240
|
let nominated_pair_is_some = self.nominated_pair.is_some();
|
|
224
241
|
|
|
242
|
+
// desklink scout trace: which nomination branch this contact() pass took.
|
|
243
|
+
if dl_trace::enabled() {
|
|
244
|
+
let best = self
|
|
245
|
+
.get_best_valid_candidate_pair()
|
|
246
|
+
.map(|i| {
|
|
247
|
+
let p = &self.candidate_pairs[i];
|
|
248
|
+
format!(
|
|
249
|
+
"{}<->{}",
|
|
250
|
+
self.local_candidates[p.local_index].addr(),
|
|
251
|
+
self.remote_candidates[p.remote_index].addr()
|
|
252
|
+
)
|
|
253
|
+
})
|
|
254
|
+
.unwrap_or_else(|| "-none-valid-".to_string());
|
|
255
|
+
dl_ice!(
|
|
256
|
+
"{} contact_branch selected={:?} nominated={nominated_pair_is_some} best_valid={best} pairs={}",
|
|
257
|
+
self.get_name(),
|
|
258
|
+
self.selected_pair,
|
|
259
|
+
self.candidate_pairs.len(),
|
|
260
|
+
);
|
|
261
|
+
}
|
|
262
|
+
|
|
225
263
|
if self.get_selected_pair().is_some() {
|
|
226
264
|
if self.validate_selected_pair(now) {
|
|
227
265
|
self.check_keepalive(now);
|
|
@@ -336,7 +374,39 @@ impl ControllingSelector for Agent {
|
|
|
336
374
|
rtt.as_millis()
|
|
337
375
|
);
|
|
338
376
|
if pending_request.is_use_candidate && selected_pair_is_none {
|
|
377
|
+
// desklink scout trace: the browser's answer to a nomination
|
|
378
|
+
// request — this is what converts nominated into selected.
|
|
379
|
+
if dl_trace::enabled() {
|
|
380
|
+
dl_ice!(
|
|
381
|
+
"{} NOMINATION_ACK txn={:?} rtt_us={} local={} remote={}",
|
|
382
|
+
self.get_name(),
|
|
383
|
+
m.transaction_id,
|
|
384
|
+
rtt.as_micros(),
|
|
385
|
+
self.local_candidates[local_index].addr(),
|
|
386
|
+
self.remote_candidates[remote_index].addr(),
|
|
387
|
+
);
|
|
388
|
+
}
|
|
339
389
|
self.set_selected_pair(Some(now), Some(pair_index));
|
|
390
|
+
} else {
|
|
391
|
+
// desklink patch: nomination is decided in `contact_candidates`,
|
|
392
|
+
// and that only runs on a `check_interval` tick, so a pair that
|
|
393
|
+
// becomes valid just after a tick waits a full tick for its
|
|
394
|
+
// USE-CANDIDATE. Ask for the check pass now, when this is the pair
|
|
395
|
+
// that tick would nominate. `request_connectivity_check` is the
|
|
396
|
+
// deferred-check request (issue #88): it makes `poll_timeout`
|
|
397
|
+
// report an already-due deadline, so the driver runs `contact()`
|
|
398
|
+
// at once instead of at the next tick. See DESKLINK-PATCH.md.
|
|
399
|
+
if self.get_selected_pair().is_none()
|
|
400
|
+
&& self.nominated_pair.is_none()
|
|
401
|
+
&& self.get_best_valid_candidate_pair() == Some(pair_index)
|
|
402
|
+
{
|
|
403
|
+
let p = &self.candidate_pairs[pair_index];
|
|
404
|
+
if self.is_nominatable(now, p.local_index, true)
|
|
405
|
+
&& self.is_nominatable(now, p.remote_index, false)
|
|
406
|
+
{
|
|
407
|
+
self.request_connectivity_check();
|
|
408
|
+
}
|
|
409
|
+
}
|
|
340
410
|
}
|
|
341
411
|
} else {
|
|
342
412
|
// This shouldn't happen
|
|
@@ -367,6 +437,17 @@ impl ControllingSelector for Agent {
|
|
|
367
437
|
let p = &self.candidate_pairs[pair_index];
|
|
368
438
|
let nominated_pair_is_none = self.nominated_pair.is_none();
|
|
369
439
|
|
|
440
|
+
// desklink scout trace: an inbound binding request on the
|
|
441
|
+
// controlling side — the browser's own connectivity check.
|
|
442
|
+
if dl_trace::enabled() {
|
|
443
|
+
dl_ice!(
|
|
444
|
+
"{} RECV_REQUEST controlling use_candidate={} pair_state={:?} nominated={nominated_pair_is_none}",
|
|
445
|
+
self.get_name(),
|
|
446
|
+
m.contains(ATTR_USE_CANDIDATE),
|
|
447
|
+
p.state,
|
|
448
|
+
);
|
|
449
|
+
}
|
|
450
|
+
|
|
370
451
|
trace!(
|
|
371
452
|
"controllingSelector: after findPair {}, p.state: {}, {}",
|
|
372
453
|
p,
|