@desklink/host 0.5.4 → 0.5.7

package/engine/src/hyprland.rs

@@ -0,0 +1,319 @@
+//! Where a shared window sits on a Hyprland desktop, from Hyprland's own IPC.
+//!
+//! Hyprland's ScreenCast portal reports every stream at (0, 0) and never says
+//! which window was picked, while the pointer device spans the whole desktop.
+//! Hyprland itself announces each window share by title on its event socket
+//! (`screencastv2>>1,window,<title>`) and lists every window's place in
+//! `j/clients`. So the shared window is the one with that title and the
+//! stream's size, followed by its address from then on, and the pointer goes
+//! onto it through Hyprland's own cursor warp at the window's current place:
+//! a window moved after the share started still maps. Anything less certain
+//! (no announcement, no such window, two alike, a window off screen) is a
+//! refusal, never a guess.
+
+use serde_json::Value;
+use std::io::{BufRead, BufReader, Read, Write};
+use std::net::Shutdown;
+use std::os::unix::net::UnixStream;
+use std::path::PathBuf;
+use std::sync::{Arc, Mutex, MutexGuard};
+
+use crate::session::WINDOW_POINTER_REFUSAL as UNKNOWN;
+
+fn instance() -> Option<PathBuf> {
+    let signature = std::env::var_os("HYPRLAND_INSTANCE_SIGNATURE")?;
+    let runtime = std::env::var_os("XDG_RUNTIME_DIR")?;
+    Some(PathBuf::from(runtime).join("hypr").join(signature))
+}
+
+fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
+    mutex
+        .lock()
+        .unwrap_or_else(|poisoned| poisoned.into_inner())
+}
+
+/// Hyprland's window-share announcements, heard from before the portal asks
+/// for consent. Dropping it closes the event socket, which ends its reader.
+pub struct ShareWatch {
+    socket: UnixStream,
+    titles: Arc<Mutex<Vec<String>>>,
+}
+
+impl ShareWatch {
+    /// `None` when this is not a Hyprland session.
+    pub fn start() -> Option<Self> {
+        let socket = UnixStream::connect(instance()?.join(".socket2.sock")).ok()?;
+        let reader = socket.try_clone().ok()?;
+        let titles = Arc::new(Mutex::new(Vec::new()));
+        let heard = titles.clone();
+        std::thread::spawn(move || {
+            for line in BufReader::new(reader).lines().map_while(Result::ok) {
+                if let Some(title) = line.strip_prefix("screencastv2>>1,window,") {
+                    lock(&heard).push(title.to_owned());
+                }
+            }
+        });
+        Some(Self { socket, titles })
+    }
+}
+
+impl Drop for ShareWatch {
+    fn drop(&mut self) {
+        let _ = self.socket.shutdown(Shutdown::Both);
+    }
+}
+
+/// The window a portal window share shows, placed by Hyprland.
+pub struct WindowLocator {
+    watch: ShareWatch,
+    /// The stream's own pixel size, which the window's size times its
+    /// monitor's scale must match.
+    source: (i64, i64),
+    address: Mutex<Option<String>>,
+    /// Whether this Hyprland reads a Lua config, whose `dispatch` takes Lua.
+    lua: Mutex<Option<bool>>,
+}
+
+impl WindowLocator {
+    pub fn new(watch: ShareWatch, width: i32, height: i32) -> Self {
+        Self {
+            watch,
+            source: (i64::from(width.max(1)), i64::from(height.max(1))),
+            address: Mutex::new(None),
+            lua: Mutex::new(None),
+        }
+    }
+
+    /// Put the pointer on `(x, y)` in the shared window's own pixels.
+    pub fn place(&self, x: i64, y: i64) -> Result<(), String> {
+        let (clients, monitors) = (query("j/clients")?, query("j/monitors")?);
+        let clients = clients.as_array().map(Vec::as_slice).unwrap_or_default();
+        // A window nobody sees has no point on the screen.
+        let shown = |window: &Value| {
+            let workspace = &window["workspace"]["id"];
+            window["mapped"].as_bool() != Some(false)
+                && window["hidden"].as_bool() != Some(true)
+                && (window["pinned"].as_bool() == Some(true)
+                    || monitors.as_array().into_iter().flatten().any(|monitor| {
+                        &monitor["activeWorkspace"]["id"] == workspace
+                            || &monitor["specialWorkspace"]["id"] == workspace
+                    }))
+        };
+        let mut address = lock(&self.address);
+        if address.is_none() {
+            *address = Some(self.identify(clients, &monitors)?);
+        }
+        let index = clients
+            .iter()
+            .position(|client| client["address"].as_str() == address.as_deref())
+            .ok_or("a shared window cannot take taps: the window has closed")?;
+        let window = &clients[index];
+        if !shown(window) {
+            return Err(String::from(
+                "a shared window cannot take taps while it is on a hidden workspace",
+            ));
+        }
+        let (left, top, width, height) = bounds(window).ok_or(UNKNOWN)?;
+        let (source_w, source_h) = self.source;
+        let (px, py) = (
+            left + x as f64 * width / source_w as f64,
+            top + y as f64 * height / source_h as f64,
+        );
+        // Hyprland clamps the cursor to the screen edge, so a point off every
+        // monitor would press something else or nothing.
+        let on_screen = monitors.as_array().into_iter().flatten().any(|monitor| {
+            let scale = monitor["scale"].as_f64().filter(|scale| *scale > 0.0);
+            monitor["disabled"].as_bool() != Some(true)
+                && matches!(
+                    (
+                        monitor["x"].as_f64(),
+                        monitor["y"].as_f64(),
+                        monitor["width"].as_f64(),
+                        monitor["height"].as_f64(),
+                        scale
+                    ),
+                    (Some(mx), Some(my), Some(mw), Some(mh), Some(scale))
+                        if (mx..mx + mw / scale).contains(&px)
+                            && (my..my + mh / scale).contains(&py)
+                )
+        });
+        if !on_screen {
+            return Err(String::from(
+                "a shared window cannot take taps where it hangs off screen",
+            ));
+        }
+        // Hyprland draws tiled windows, then floating ones, then fullscreen
+        // ones, and pinned ones above all; j/clients lists them in the
+        // compositor's own stacking order, so a later layer-mate holding the
+        // point is what a press there would reach. A floating window let over
+        // fullscreen draws above it, but j/clients cannot tell that one from
+        // any floating window, so either pairing over the point is a refusal.
+        let layer = |w: &Value| {
+            if w["pinned"].as_bool() == Some(true) {
+                3
+            } else if w["fullscreen"].as_i64().is_some_and(|mode| mode > 0) {
+                2
+            } else {
+                u8::from(w["floating"].as_bool() == Some(true))
+            }
+        };
+        let covered = clients.iter().enumerate().any(|(i, other)| {
+            shown(other)
+                && bounds(other).is_some_and(|(l, t, w, h)| {
+                    (l..l + w).contains(&px) && (t..t + h).contains(&py)
+                })
+                && match (layer(other), layer(window), i > index) {
+                    (1, 2, _) | (2, 1, _) => true,
+                    (above, own, later) => above > own || (above == own && later),
+                }
+        });
+        if covered {
+            return Err(String::from(
+                "a shared window cannot take taps where another window covers it",
+            ));
+        }
+        // Bars, notifications and other layer surfaces draw above windows and
+        // are absent from the window-only video: a top or overlay one holding
+        // the point is what a press there would reach. Their boxes come local
+        // to their monitor, which j/monitors places.
+        let anchored = |name: &str| -> Option<(f64, f64)> {
+            for monitor in monitors.as_array().into_iter().flatten() {
+                if monitor["name"].as_str() == Some(name) {
+                    return Some((monitor["x"].as_f64()?, monitor["y"].as_f64()?));
+                }
+            }
+            None
+        };
+        let overlaid = query("j/layers")?
+            .as_object()
+            .into_iter()
+            .flatten()
+            .map(|(name, monitor)| Some((anchored(name)?, monitor)))
+            .collect::<Option<Vec<((f64, f64), &Value)>>>()
+            .ok_or(String::from(
+                "a shared window cannot take taps: a screen's layers are unknown",
+            ))?
+            .into_iter()
+            .any(|((mx, my), monitor)| {
+                monitor["levels"]
+                    .as_object()
+                    .into_iter()
+                    .flatten()
+                    .any(|(level, surfaces)| {
+                        let level = level.parse::<u8>().unwrap_or(0);
+                        (level >= 2)
+                            && surfaces.as_array().into_iter().flatten().any(|surface| {
+                                let (sx, sy) = (surface["x"].as_f64(), surface["y"].as_f64());
+                                let (sw, sh) = (surface["w"].as_f64(), surface["h"].as_f64());
+                                !matches!(
+                                    surface["namespace"].as_str(),
+                                    Some("desklink-agent" | "desklink-point")
+                                ) && matches!(
+                                    (sx, sy, sw, sh),
+                                    (Some(sx), Some(sy), Some(sw), Some(sh))
+                                        if (mx + sx..mx + sx + sw).contains(&px)
+                                            && (my + sy..my + sy + sh).contains(&py)
+                                )
+                            })
+                    })
+            });
+        if overlaid {
+            return Err(String::from(
+                "a shared window cannot take taps where an overlay covers it",
+            ));
+        }
+        self.warp((px.floor() as i64, py.floor() as i64))
+    }
+
+    /// The address of the one window Hyprland said it is sharing.
+    fn identify(&self, clients: &[Value], monitors: &Value) -> Result<String, String> {
+        let mut titles = lock(&self.watch.titles).clone();
+        titles.dedup();
+        let [title] = titles.as_slice() else {
+            return Err(String::from(UNKNOWN));
+        };
+        let (source_w, source_h) = self.source;
+        let scale = |monitor: &Value| {
+            monitors
+                .as_array()
+                .into_iter()
+                .flatten()
+                .find(|m| &m["id"] == monitor)
+                .and_then(|m| m["scale"].as_f64())
+        };
+        let alike: Vec<&Value> = clients
+            .iter()
+            .filter(|client| {
+                let size = |i: usize| client["size"][i].as_f64();
+                client["title"].as_str() == Some(title.as_str())
+                    && client["mapped"].as_bool() != Some(false)
+                    && client["hidden"].as_bool() != Some(true)
+                    && matches!(
+                        (size(0), size(1), scale(&client["monitor"])),
+                        (Some(w), Some(h), Some(s))
+                            if ((w * s).round() as i64 - source_w).abs() <= 1
+                                && ((h * s).round() as i64 - source_h).abs() <= 1
+                    )
+            })
+            .collect();
+        match alike.as_slice() {
+            [window] => window["address"]
+                .as_str()
+                .map(str::to_owned)
+                .ok_or_else(|| String::from(UNKNOWN)),
+            [] => Err(String::from(UNKNOWN)),
+            _ => Err(format!(
+                "a shared window cannot take taps: {} windows look like it; share the whole screen to point and click",
+                alike.len()
+            )),
+        }
+    }
+
+    fn warp(&self, (x, y): (i64, i64)) -> Result<(), String> {
+        let legacy = format!("dispatch movecursor {x} {y}");
+        let lua = format!("dispatch hl.dsp.cursor.move({{ x = {x}, y = {y} }})");
+        let mut dialect = lock(&self.lua);
+        let tries: &[(bool, &str)] = match *dialect {
+            Some(true) => &[(true, &lua)],
+            Some(false) => &[(false, &legacy)],
+            None => &[(false, &legacy), (true, &lua)],
+        };
+        let mut said = String::new();
+        for (is_lua, command) in tries {
+            said = request(command)?;
+            if said.trim() == "ok" {
+                *dialect = Some(*is_lua);
+                return Ok(());
+            }
+        }
+        Err(format!(
+            "Hyprland did not move the pointer: {}",
+            said.trim()
+        ))
+    }
+}
+
+/// A window's box in the layout: left, top, width, height.
+fn bounds(window: &Value) -> Option<(f64, f64, f64, f64)> {
+    let pair = |key: &str| Some((window[key][0].as_f64()?, window[key][1].as_f64()?));
+    let ((left, top), (width, height)) = pair("at").zip(pair("size"))?;
+    Some((left, top, width, height))
+}
+
+fn request(command: &str) -> Result<String, String> {
+    let ask = || -> std::io::Result<String> {
+        let path = instance()
+            .ok_or_else(|| std::io::Error::other("HYPRLAND_INSTANCE_SIGNATURE is not set"))?;
+        let mut socket = UnixStream::connect(path.join(".socket.sock"))?;
+        socket.write_all(command.as_bytes())?;
+        let mut reply = String::new();
+        socket.read_to_string(&mut reply)?;
+        Ok(reply)
+    };
+    ask().map_err(|error| format!("Hyprland's control socket did not answer: {error}"))
+}
+
+fn query(command: &str) -> Result<Value, String> {
+    serde_json::from_str(&request(command)?)
+        .map_err(|error| format!("Hyprland's {command} was not JSON: {error}"))
+}

package/engine/src/mac.rs

@@ -184,6 +184,7 @@ fn capabilities() -> Value {
             "formats": ["bgra"],
             "cursor": "embedded",
             "cursor_modes": ["embedded"],
+            // ScreenCaptureKit audio is not wired up; macOS is out of scope.
             "audio": false,
             "displays": displays(),
             "grant": if capture_granted { "granted" } else { "missing-screen-recording" },

package/engine/src/main.rs

@@ -6,6 +6,8 @@
 //! process, owns its stdin/stdout, and carries the SDP/ICE the engine produces
 //! over whatever authenticated channel it already has.
 
+#[cfg(target_os = "linux")]
+mod audio;
 #[cfg(target_os = "linux")]
 mod capture;
 #[cfg(target_os = "macos")]
@@ -43,6 +45,8 @@ mod h264_openh264;
 mod h264_vaapi;
 #[cfg(all(target_os = "macos", desklink_vpx))]
 mod h264_vt;
+#[cfg(target_os = "linux")]
+mod hyprland;
 #[cfg(any(
     target_os = "linux",
     all(any(target_os = "macos", target_os = "windows"), desklink_vpx)
@@ -574,6 +578,15 @@ fn render_event(notice: session::Notice) -> Option<String> {
             "session.frame.changed",
             serde_json::json!({ "sessionId": session_id, "seq": seq, "damage": damage, "timestamp_us": timestamp_us }),
         ),
+        session::SessionEvent::Audio {
+            seq,
+            timestamp_us,
+            bytes,
+            gap_micros,
+        } => (
+            "session.audio.changed",
+            serde_json::json!({ "sessionId": session_id, "seq": seq, "timestamp_us": timestamp_us, "bytes": bytes, "gap_ms": gap_micros / 1000 }),
+        ),
         session::SessionEvent::Input { input } => (
             "session.input",
             serde_json::json!({ "sessionId": session_id, "input": input }),
@@ -658,6 +671,7 @@ async fn dispatch(
                 loopback_tcp: params.loopback_tcp,
                 agent_indicator: params.agent_indicator,
                 local_frames: params.local_frames,
+                audio: params.audio,
                 ice_servers: params
                     .ice_servers
                     .into_iter()
@@ -684,6 +698,7 @@ async fn dispatch(
                 "geometry": session.geometry(),
                 "cursor": session.cursor_info(),
                 "agentIndicator": session.agent_indicator(),
+                "audio": session.audio_info(),
             });
             *current = Some(session);
             Ok(result)
@@ -787,6 +802,15 @@ async fn dispatch(
                 .frame(params.since, &params.path, params.region)
                 .map_err(|error| ErrorBody::new(error.code, error.message))
         }
+        "session.audio" => {
+            let session = require_session(current)?;
+            let params: protocol::AudioParams = serde_json::from_value(request.params.clone())
+                .map_err(|error| ErrorBody::new("malformed", error.to_string()))?;
+            check_session(session, &params.session_id, None)?;
+            session
+                .audio(&params.path, params.max_bytes)
+                .map_err(|error| ErrorBody::new(error.code, error.message))
+        }
         "session.feed" => {
             let session = require_session(current)?;
             let params: protocol::FeedParams = serde_json::from_value(request.params.clone())

package/engine/src/peer.rs

@@ -47,6 +47,33 @@ use webrtc::peer_connection::{
 };
 use webrtc::runtime::default_runtime;
 
+// desklink scout trace (dl-roam-postnom-attr): engine-side restart timeline.
+// Enabled with `DESKLINK_ICE_TRACE=1`; stamps microseconds since the Unix epoch
+// so these lines interleave with the vendored ICE agent's `[ice ...]` lines in
+// one order. Droppable: grep `desklink scout trace`.
+pub(crate) fn dl_trace_on() -> bool {
+    std::env::var("DESKLINK_ICE_TRACE")
+        .map(|v| v != "0" && !v.is_empty())
+        .unwrap_or(false)
+}
+
+pub(crate) fn dl_trace_us() -> u128 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .map(|d| d.as_micros())
+        .unwrap_or(0)
+}
+
+macro_rules! dl_eng {
+    ($($arg:tt)*) => {
+        if $crate::peer::dl_trace_on() {
+            eprintln!("[eng {}] {}", $crate::peer::dl_trace_us(), format_args!($($arg)*));
+        }
+    };
+}
+
+pub(crate) use dl_eng;
+
 /// VP9 clock rate, fixed by RFC 9628.
 const VP9_CLOCK_RATE: u32 = 90_000;
 
@@ -420,6 +447,12 @@ impl PeerConnectionEventHandler for Handler {
     }
 
     async fn on_connection_state_change(&self, state: RTCPeerConnectionState) {
+        // desklink scout trace (dl-roam-postnom-attr): the engine's rtc-level
+        // transport state event, on the same microsecond clock as the vendored
+        // agent's `[ice ...]` lines, so the rtc Connected gate is separable.
+        if dl_trace_on() {
+            dl_eng!("peer_state_change {state:?}");
+        }
         let connected = matches!(state, RTCPeerConnectionState::Connected);
         if connected {
             self.wants_keyframe.store(true, Ordering::SeqCst);
@@ -1590,6 +1623,15 @@ pub(crate) mod tests {
             .expect("the restart answer is accepted");
     }
 
+    /// How long the restarted generation may take to nominate and reconnect,
+    /// measured from the answered offer: two `check_interval` ticks. The vendored
+    /// agent ticks every 200 ms (`vendor/rtc-ice/src/agent/agent_config.rs`), and
+    /// the routed roam leg measured the engine's own first-valid-pair ->
+    /// USE-CANDIDATE at 197 ms, one tick; the extra tick is headroom for the
+    /// loopback's round trip and for a loaded machine. See the assertion in
+    /// `a_restart_keeps_media_flowing_on_the_selected_pair_until_the_new_one_nominates`.
+    const RECONNECT_BUDGET: Duration = Duration::from_millis(400);
+
     /// A receiver peer that records when each inbound RTP packet arrives and
     /// reports its own ICE candidates, so two peers can really connect.
     struct MediaReceiver {
@@ -1783,6 +1825,37 @@ pub(crate) mod tests {
             .accept_answer(reanswer.sdp)
             .await
             .expect("the restart answer is accepted");
+        let answered_at = Instant::now();
+
+        // The restarted generation has to be nominated off its own check cycle:
+        // nomination is decided in the agent's `contact_candidates` pass, which the
+        // driver reaches on a `check_interval` tick (200 ms) or when an inbound
+        // Binding request arrives while a valid pair already exists. A pair that
+        // becomes valid just after a tick would therefore wait a full tick for its
+        // USE-CANDIDATE, and one tick is the whole engine-side cost measured on the
+        // routed roam leg (first valid pair -> USE-CANDIDATE: 197 ms, i.e. one tick).
+        //
+        // What this asserts, and what it cannot: the engine's "first valid pair"
+        // instant is not observable from here (the peer connection exposes no pair
+        // state), so the bound is measured from the answer, the earliest instant the
+        // engine could nominate, at two ticks. The loopback's client answers checks
+        // and sends its own triggered checks in the same burst, so an inbound request
+        // is nearly always available and the base engine meets this bound too; the
+        // tick itself is graded by the routed roam leg, whose relay log carries both
+        // instants (197 ms, one tick). This guards the property going forward: a
+        // lost pass, a skipped tick or a stalled nomination blows the bound.
+        let reconnected = async {
+            while !engine.is_connected() {
+                tokio::time::sleep(Duration::from_millis(5)).await;
+            }
+            answered_at.elapsed()
+        };
+        let reconnect = tokio::time::timeout(RECONNECT_BUDGET, reconnected)
+            .await
+            .expect("the restarted generation was never nominated");
+        eprintln!(
+            "restart: transport reconnected {reconnect:?} after the answer (budget {RECONNECT_BUDGET:?})"
+        );
 
         // Keep the picture advancing across and after the restart.
         tokio::time::sleep(Duration::from_millis(800)).await;

package/engine/src/protocol.rs

@@ -144,6 +144,11 @@ pub struct OpenParams {
     /// the engine then spends nothing per frame on either.
     #[serde(default = "default_local_frames")]
     pub local_frames: bool,
+    /// Capture the desktop's own system audio for `session.audio`. Absent means
+    /// video only; there is never a silent downgrade, so a caller that asks and
+    /// cannot be answered is refused.
+    #[serde(default)]
+    pub audio: bool,
 }
 
 fn default_local_frames() -> bool {
@@ -244,6 +249,23 @@ pub struct SessionRef {
     pub generation: Option<u64>,
 }
 
+/// Read the desktop's own audio, the way `FrameParams` reads a frame: bytes go
+/// to the caller's path and JSON carries only metadata.
+#[derive(Debug, Deserialize)]
+pub struct AudioParams {
+    pub session_id: String,
+    #[serde(default)]
+    pub path: String,
+    /// The most bytes to write. Bounded to whole sample frames, so a truncated
+    /// chunk never ends mid-sample.
+    #[serde(default = "default_max_audio_bytes")]
+    pub max_bytes: usize,
+}
+
+fn default_max_audio_bytes() -> usize {
+    1 << 20
+}
+
 #[derive(Debug, Deserialize)]
 pub struct DescriptionParams {
     pub session_id: String,