@desklink/host 0.2.0 → 0.3.1

package/engine/src/capture.rs

@@ -13,7 +13,7 @@ use pw::spa;
 use pw::spa::pod::{serialize::PodSerializer, Pod};
 use std::io::Cursor;
 use std::os::fd::OwnedFd;
-use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::sync::{mpsc, Arc, Mutex};
 use std::thread::JoinHandle;
 use std::time::{Duration, Instant};
@@ -44,13 +44,18 @@ pub struct StreamGeometry {
     pub buffer_type: String,
 }
 
-pub type FrameSink = Box<dyn Fn(I420, u64, Vec<u8>) + Send + 'static>;
+/// Receives each converted frame, its sequence number and a way to produce the
+/// frame's packed BGRX pixels, which only a local frame consumer needs; the
+/// sink calls it at most once, and only then are the pixels copied.
+pub type FrameSink = Box<dyn Fn(I420, u64, Pixels) + Send + 'static>;
+pub type Pixels<'a> = &'a mut dyn FnMut() -> Option<Vec<u8>>;
 
 /// A running capture. Dropping it quits the PipeWire loop and joins its thread.
 pub struct Capture {
     quit: pw::channel::Sender<()>,
     thread: Option<JoinHandle<()>>,
     pub source: SelectedSource,
+    cursor_positions: Arc<AtomicBool>,
     geometry: Arc<Mutex<Option<StreamGeometry>>>,
     frames: Arc<AtomicU64>,
     dropped: Arc<AtomicU64>,
@@ -59,6 +64,9 @@ pub struct Capture {
 }
 
 impl Capture {
+    pub fn cursor_positions(&self) -> bool {
+        self.cursor_positions.load(Ordering::Relaxed)
+    }
     pub fn geometry(&self) -> Option<StreamGeometry> {
         self.geometry.lock().ok().and_then(|g| g.clone())
     }
@@ -189,16 +197,29 @@ fn buffer_pod(buffer: &mut Vec<u8>) -> Result<()> {
 }
 
 /// Start capturing `session`'s node, delivering I420 frames to `sink`.
+#[allow(clippy::too_many_arguments)]
 pub fn start(
     session: PortalSession,
     encoded_width: usize,
     encoded_height: usize,
     max_fps: u32,
+    cursor_sink: Option<crate::cursor::CursorSink>,
     sink: FrameSink,
     indicator_position: Option<Arc<Mutex<Option<(i64, i64)>>>>,
     on_stop: Box<dyn Fn(String) + Send>,
 ) -> Result<Capture> {
-    let PortalSession { fd, source, .. } = session;
+    let PortalSession {
+        fd,
+        source,
+        cursor_mode,
+        ..
+    } = session;
+    let cursor_positions = Arc::new(AtomicBool::new(false));
+    let cursor_sink = if cursor_mode == "metadata" {
+        cursor_sink
+    } else {
+        None
+    };
     let geometry = Arc::new(Mutex::new(None));
     let frames = Arc::new(AtomicU64::new(0));
     let dropped = Arc::new(AtomicU64::new(0));
@@ -206,7 +227,9 @@ pub fn start(
     let (ready_tx, ready_rx) = mpsc::channel::<Result<(), String>>();
 
     let node_id = source.node_id;
+    let source_size = (source.width.max(1) as usize, source.height.max(1) as usize);
     let thread = {
+        let cursor_positions = cursor_positions.clone();
         let geometry = geometry.clone();
         let frames = frames.clone();
         let dropped = dropped.clone();
@@ -217,9 +240,12 @@ pub fn start(
                 let result = run_loop(
                     fd,
                     node_id,
+                    source_size,
                     encoded_width,
                     encoded_height,
                     max_fps,
+                    cursor_sink,
+                    cursor_positions,
                     sink,
                     indicator_position,
                     on_stop,
@@ -246,6 +272,7 @@ pub fn start(
         quit,
         thread: Some(thread),
         source,
+        cursor_positions,
         geometry,
         frames,
         dropped,
@@ -264,9 +291,12 @@ pub fn start(
 fn run_loop(
     fd: OwnedFd,
     node_id: u32,
+    source_size: (usize, usize),
     encoded_width: usize,
     encoded_height: usize,
     max_fps: u32,
+    cursor_sink: Option<crate::cursor::CursorSink>,
+    cursor_positions: Arc<AtomicBool>,
     sink: FrameSink,
     indicator_position: Option<Arc<Mutex<Option<(i64, i64)>>>>,
     on_stop: Box<dyn Fn(String) + Send>,
@@ -298,6 +328,8 @@ fn run_loop(
     let stream = pw::stream::StreamBox::new(&core, "desklink-capture", props)
         .context("pw_stream_new failed")?;
 
+    let wants_cursor_meta = cursor_sink.is_some();
+    let mut cursor = cursor_sink.map(crate::cursor::Reporter::new);
     let mut state = StreamState {
         box_w: encoded_width,
         box_h: encoded_height,
@@ -316,14 +348,26 @@ fn run_loop(
     let mut ready = Some(ready);
     let _listener = stream
         .add_local_listener_with_user_data(&mut state)
-        .state_changed({ let main_loop = main_loop.clone(); move |_, state, old, new| {
-            let reason = match new {
-                pw::stream::StreamState::Error(message) => Some(format!("PipeWire stream stopped: {message}")),
-                pw::stream::StreamState::Unconnected if old == pw::stream::StreamState::Streaming => Some(String::from("PipeWire stream disconnected")),
-                _ => None,
-            };
-            if let Some(reason) = reason { (state.on_stop)(reason); main_loop.quit(); }
-        } })
+        .state_changed({
+            let main_loop = main_loop.clone();
+            move |_, state, old, new| {
+                let reason = match new {
+                    pw::stream::StreamState::Error(message) => {
+                        Some(format!("PipeWire stream stopped: {message}"))
+                    }
+                    pw::stream::StreamState::Unconnected
+                        if old == pw::stream::StreamState::Streaming =>
+                    {
+                        Some(String::from("PipeWire stream disconnected"))
+                    }
+                    _ => None,
+                };
+                if let Some(reason) = reason {
+                    (state.on_stop)(reason);
+                    main_loop.quit();
+                }
+            }
+        })
         .param_changed(|_, state, id, param| {
             if id != pw::spa::param::ParamType::Format.as_raw() {
                 return;
@@ -351,6 +395,31 @@ fn run_loop(
             let Some(mut buffer) = stream.dequeue_buffer() else {
                 return;
             };
+            if let Some(cursor) = cursor.as_mut() {
+                if let Some(meta) = buffer.find_meta::<spa::buffer::meta::MetaCursor>() {
+                    cursor_positions.store(true, Ordering::Relaxed);
+                    // The wrapper's size check bounds the bitmap header;
+                    // never follow an unchecked bitmap offset.
+                    let bitmap = buffer
+                        .find_meta::<CursorWithBitmap>()
+                        .filter(|m| {
+                            m.cursor.bitmap_offset as usize
+                                == std::mem::size_of::<spa::sys::spa_meta_cursor>()
+                        })
+                        .map(|m| &m.bitmap);
+                    if state.logical_w == 0 || state.logical_h == 0 {
+                        return;
+                    }
+                    let mut position = *meta.as_raw();
+                    position.position.x = (i64::from(position.position.x) * source_size.0 as i64)
+                        .div_euclid(state.logical_w as i64)
+                        as i32;
+                    position.position.y = (i64::from(position.position.y) * source_size.1 as i64)
+                        .div_euclid(state.logical_h as i64)
+                        as i32;
+                    cursor.update_spa(&position, bitmap, source_size.0, source_size.1);
+                }
+            }
             let Some(format) = state.format else { return };
             let datas = buffer.datas_mut();
             if datas.is_empty() {
@@ -383,54 +452,58 @@ fn run_loop(
             // second time and hand the encoder a frame it refuses whenever the
             // stream's size differs from the one the portal reported.
             let seq = state.frames.load(Ordering::Relaxed);
-            let raw = crate::convert::to_bgrx(
-                chunk_bytes,
-                w,
-                h,
-                stride,
-                format,
-                state.box_w,
-                state.box_h,
-            );
-            if let Some(mut raw) = raw {
-                let frame = if let Some(mask) = state.mask.as_mut() {
-                    mask.apply(&mut raw, state.box_w, state.box_h, w, h);
-                    to_i420(
-                        &raw,
-                        state.box_w,
-                        state.box_h,
-                        state.box_w * 4,
-                        PixelFormat::Bgrx,
-                        state.box_w,
-                        state.box_h,
-                    )
-                } else {
-                    to_i420(chunk_bytes, w, h, stride, format, state.box_w, state.box_h)
+            let (box_w, box_h) = (state.box_w, state.box_h);
+            // The capture indicator is masked out of BGRX pixels, so a masked
+            // frame is converted from that copy; otherwise straight from the
+            // stream, and BGRX is only made if the sink asks for it.
+            let mut masked = None;
+            let frame = if let Some(mask) = state.mask.as_mut() {
+                let Some(mut raw) =
+                    crate::convert::to_bgrx(chunk_bytes, w, h, stride, format, box_w, box_h)
+                else {
+                    state.dropped.fetch_add(1, Ordering::Relaxed);
+                    return;
                 };
-                if let Some(i420) = frame {
-                    if state.geometry.lock().map(|g| g.is_none()).unwrap_or(false) {
-                        if let Ok(mut g) = state.geometry.lock() {
-                            *g = Some(StreamGeometry {
-                                source_width: w,
-                                source_height: h,
-                                origin_x: 0,
-                                origin_y: 0,
-                                format: format!("{format:?}").to_lowercase(),
-                                buffer_type: state.buffer_type.clone(),
-                            });
-                        }
-                    }
-                    state.frames.fetch_add(1, Ordering::Relaxed);
-                    (state.sink)(i420, seq, raw);
-                    // The first frame the loop actually delivers is the only
-                    // proof capture started; sending this when `run_loop`
-                    // returns would be after the main loop quits, too late for
-                    // `start` to wait on.
-                    if let Some(sender) = ready.take() {
-                        let _ = sender.send(Ok(()));
+                mask.apply(&mut raw, box_w, box_h, w, h);
+                let frame = to_i420(
+                    &raw,
+                    box_w,
+                    box_h,
+                    box_w * 4,
+                    PixelFormat::Bgrx,
+                    box_w,
+                    box_h,
+                );
+                masked = Some(raw);
+                frame
+            } else {
+                to_i420(chunk_bytes, w, h, stride, format, box_w, box_h)
+            };
+            if let Some(i420) = frame {
+                if state.geometry.lock().map(|g| g.is_none()).unwrap_or(false) {
+                    if let Ok(mut g) = state.geometry.lock() {
+                        *g = Some(StreamGeometry {
+                            source_width: w,
+                            source_height: h,
+                            origin_x: 0,
+                            origin_y: 0,
+                            format: format!("{format:?}").to_lowercase(),
+                            buffer_type: state.buffer_type.clone(),
+                        });
                     }
-                } else {
-                    state.dropped.fetch_add(1, Ordering::Relaxed);
+                }
+                state.frames.fetch_add(1, Ordering::Relaxed);
+                (state.sink)(i420, seq, &mut || {
+                    masked.take().or_else(|| {
+                        crate::convert::to_bgrx(chunk_bytes, w, h, stride, format, box_w, box_h)
+                    })
+                });
+                // The first frame the loop actually delivers is the only
+                // proof capture started; sending this when `run_loop`
+                // returns would be after the main loop quits, too late for
+                // `start` to wait on.
+                if let Some(sender) = ready.take() {
+                    let _ = sender.send(Ok(()));
                 }
             } else {
                 state.dropped.fetch_add(1, Ordering::Relaxed);
@@ -443,10 +516,15 @@ fn run_loop(
     format_pod(&mut format_buf, max_fps)?;
     let mut buffer_buf = Vec::new();
     buffer_pod(&mut buffer_buf)?;
-    let mut params = [
+    let mut cursor_buf = Vec::new();
+    cursor_pod(&mut cursor_buf)?;
+    let mut params = vec![
         Pod::from_bytes(&format_buf).context("invalid EnumFormat pod")?,
         Pod::from_bytes(&buffer_buf).context("invalid Buffers pod")?,
     ];
+    if wants_cursor_meta {
+        params.push(Pod::from_bytes(&cursor_buf).context("invalid Cursor Meta pod")?);
+    }
 
     stream
         .connect(
@@ -638,6 +716,7 @@ mod tests {
                 origin_x: 0,
                 origin_y: 0,
             },
+            cursor_positions: Arc::new(AtomicBool::new(false)),
             geometry: Arc::new(Mutex::new(None)),
             frames: Arc::new(AtomicU64::new(0)),
             dropped: Arc::new(AtomicU64::new(0)),
@@ -658,3 +737,33 @@ mod tests {
             .expect("capture stopped and joined");
     }
 }
+
+fn cursor_pod(buffer: &mut Vec<u8>) -> Result<()> {
+    use spa::pod::{object, property, Value};
+    let obj = object!(
+        spa::utils::SpaTypes::ObjectParamMeta,
+        spa::param::ParamType::Meta,
+        property!(
+            RawKey(spa::sys::SPA_PARAM_META_type),
+            Id,
+            RawKey(spa::sys::SPA_META_Cursor)
+        ),
+        property!(
+            RawKey(spa::sys::SPA_PARAM_META_size),
+            Int,
+            (std::mem::size_of::<CursorWithBitmap>() + 256 * 256 * 4) as i32
+        )
+    );
+    PodSerializer::serialize(Cursor::new(buffer), &Value::Object(obj))
+        .map(|_| ())
+        .map_err(|e| anyhow::anyhow!("failed to build Cursor Meta pod: {e}"))
+}
+
+#[repr(C)]
+struct CursorWithBitmap {
+    cursor: spa::sys::spa_meta_cursor,
+    bitmap: spa::sys::spa_meta_bitmap,
+}
+impl spa::buffer::meta::Metadata for CursorWithBitmap {
+    const META_TYPE: u32 = spa::sys::SPA_META_Cursor;
+}

package/engine/src/capture_mac.rs

@@ -4,7 +4,10 @@ use anyhow::{bail, Result};
 use std::ffi::{c_char, c_void, CStr};
 use std::sync::Mutex;
 
-pub type FrameSink = Box<dyn Fn(I420, u64, Vec<u8>) + Send + 'static>;
+/// See the Linux `capture::FrameSink`: the closure makes the BGRX copy, only
+/// when the sink asks for it.
+pub type FrameSink = Box<dyn Fn(I420, u64, Pixels) + Send + 'static>;
+pub type Pixels<'a> = &'a mut dyn FnMut() -> Option<Vec<u8>>;
 
 struct State {
     sink: FrameSink,
@@ -41,8 +44,13 @@ unsafe extern "C" {
 extern "C" fn stream_status(context: *mut c_void, reason: *const c_char, running: bool) {
     let state = unsafe { &*(context as *const Mutex<State>) };
     if let Ok(state) = state.lock() {
-        let reason = if reason.is_null() { String::from("stream stopped") }
-            else { unsafe { CStr::from_ptr(reason) }.to_string_lossy().into_owned() };
+        let reason = if reason.is_null() {
+            String::from("stream stopped")
+        } else {
+            unsafe { CStr::from_ptr(reason) }
+                .to_string_lossy()
+                .into_owned()
+        };
         (state.status)(running, reason);
     }
 }
@@ -67,15 +75,6 @@ extern "C" fn receive_frame(
     let state = unsafe { &*(context as *const Mutex<State>) };
     let Ok(mut state) = state.lock() else { return };
     let source = unsafe { std::slice::from_raw_parts(pixels, length) };
-    let raw = to_bgrx(
-        source,
-        source_width,
-        source_height,
-        stride,
-        PixelFormat::Bgra,
-        state.width,
-        state.height,
-    );
     let frame = to_i420(
         source,
         source_width,
@@ -85,8 +84,19 @@ extern "C" fn receive_frame(
         state.width,
         state.height,
     );
-    if let (Some(frame), Some(raw)) = (frame, raw) {
-        (state.sink)(frame, state.sequence, raw);
+    if let Some(frame) = frame {
+        let (width, height) = (state.width, state.height);
+        (state.sink)(frame, state.sequence, &mut || {
+            to_bgrx(
+                source,
+                source_width,
+                source_height,
+                stride,
+                PixelFormat::Bgra,
+                width,
+                height,
+            )
+        });
         state.sequence += 1;
     }
 }

package/engine/src/capture_win.rs

@@ -0,0 +1,23 @@
+//! Windows desktop capture is deferred; encoded sources never call this adapter.
+use crate::convert::I420;
+use crate::portal::PortalSession;
+use anyhow::{bail, Result};
+
+pub type FrameSink = Box<dyn Fn(I420, u64, Pixels) + Send + 'static>;
+pub type Pixels<'a> = &'a mut dyn FnMut() -> Option<Vec<u8>>;
+pub struct Capture;
+
+pub fn unavailable_reason() -> &'static str {
+    "Windows display capture is not implemented in this build."
+}
+
+pub fn start(
+    _session: PortalSession,
+    _width: usize,
+    _height: usize,
+    _fps: u32,
+    _sink: FrameSink,
+    _status: Box<dyn Fn(bool, String) + Send>,
+) -> Result<Capture> {
+    bail!("{}", unavailable_reason())
+}

package/engine/src/clipboard_win.rs

@@ -0,0 +1,8 @@
+//! Windows clipboard seam. This build never reads or writes the clipboard.
+pub const MAX_CLIPBOARD_BYTES: usize = 256 * 1024;
+pub fn read_or_explain() -> Result<(String, bool), String> {
+    Err("Windows clipboard transfer is not implemented in this build.".into())
+}
+pub fn write(_text: &str) -> anyhow::Result<()> {
+    anyhow::bail!("Windows clipboard transfer is not implemented in this build.")
+}