@desklink/host 0.5.4 → 0.5.6

package/engine/src/audio.rs

@@ -0,0 +1,398 @@
+//! System audio probing and opt-in capture of the default PipeWire sink monitor.
+//!
+//! Linux captures through a PipeWire monitor of a sink, so the machine fact is
+//! "a reachable sink exposes monitor ports". `capabilities` may answer that: it
+//! must not prompt and must not start a capture, so this connects as any client
+//! does, reads the registry and leaves — no portal, no D-Bus, no consent dialog,
+//! no stream. It answers for the machine and not for the build: the same binary
+//! says `false` where no PipeWire, or no sink, is reachable, and a host running
+//! PulseAudio without PipeWire is `false` too, because this build has no
+//! PulseAudio client and no capture path that could use one.
+
+use anyhow::Context as _;
+use pipewire as pw;
+use std::cell::RefCell;
+use std::collections::{HashSet, VecDeque};
+use std::rc::Rc;
+use std::result::Result;
+use std::sync::{Arc, Mutex, MutexGuard, mpsc};
+use std::thread;
+use std::time::{Duration, Instant};
+
+/// The sample format this seam carries. It describes this build, so it is
+/// reported whether or not this machine has a sink; the capture asks PipeWire
+/// for exactly it.
+pub const FORMAT: &str = "s16le";
+pub const SAMPLE_RATE: u32 = 48_000;
+pub const CHANNELS: u32 = 2;
+
+/// The sample format a session would carry, best first.
+pub const FORMATS: &[&str] = &[FORMAT];
+
+/// What a session would capture from, and in what format.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct AudioSource {
+    /// Names the mechanism, so a consumer can tell a sink monitor from a future
+    /// consent-carrying portal audio node.
+    pub name: &'static str,
+    pub formats: &'static [&'static str],
+}
+
+/// The desktop's own mix, as the sink sends it to its monitor ports.
+const MONITOR: AudioSource = AudioSource {
+    name: "pipewire-monitor",
+    formats: FORMATS,
+};
+
+/// How long the registry may take to answer; `capabilities` must not hang
+/// behind a server that is up but silent.
+const TIMEOUT: Duration = Duration::from_millis(1500);
+
+#[derive(Default)]
+struct Scan {
+    /// Node ids whose `media.class` is `Audio/Sink`, and the node id each
+    /// `port.monitor` output port belongs to. A sink with a monitor is the node
+    /// a capture stream would link to; a sink without one cannot be captured
+    /// this way.
+    sinks: HashSet<u32>,
+    monitors: HashSet<u32>,
+    answered: bool,
+    failed: bool,
+}
+
+/// `Some(source)` exactly when opening a session with audio here would capture
+/// the desktop's system audio without prompting; `None` otherwise.
+pub fn probe() -> Option<AudioSource> {
+    pw::init();
+    let main_loop = pw::main_loop::MainLoopRc::new(None).ok()?;
+    let context = pw::context::ContextRc::new(&main_loop, None).ok()?;
+    let core = context.connect_rc(None).ok()?;
+    let registry = core.get_registry().ok()?;
+
+    // The round trip is asked for before its listener exists; a reply cannot be
+    // dispatched until the loop below iterates, so nothing is missed.
+    let pending = core.sync(0).ok();
+    let scan = Rc::new(RefCell::new(Scan::default()));
+
+    let _core = core
+        .add_listener_local()
+        .done({
+            let scan = Rc::clone(&scan);
+            move |_, seq| {
+                if Some(seq) == pending {
+                    scan.borrow_mut().answered = true;
+                }
+            }
+        })
+        .error({
+            let scan = Rc::clone(&scan);
+            move |_, _, _, _| {
+                let mut scan = scan.borrow_mut();
+                scan.failed = true;
+                scan.answered = true;
+            }
+        })
+        .register();
+
+    let _registry = registry
+        .add_listener_local()
+        .global({
+            let scan = Rc::clone(&scan);
+            move |global| {
+                let Some(props) = global.props else { return };
+                let mut scan = scan.borrow_mut();
+                match global.type_ {
+                    pw::types::ObjectType::Node
+                        if props.get(*pw::keys::MEDIA_CLASS) == Some("Audio/Sink") =>
+                    {
+                        scan.sinks.insert(global.id);
+                    }
+                    pw::types::ObjectType::Port
+                        if props.get(*pw::keys::PORT_MONITOR) == Some("true")
+                            && props.get(*pw::keys::PORT_DIRECTION) == Some("out") =>
+                    {
+                        if let Some(node) = props.get(*pw::keys::NODE_ID).and_then(|id| id.parse().ok()) {
+                            scan.monitors.insert(node);
+                        }
+                    }
+                    _ => {}
+                }
+            }
+        })
+        .register();
+
+    let deadline = Instant::now() + TIMEOUT;
+    while Instant::now() < deadline {
+        if main_loop
+            .loop_()
+            .iterate(pw::loop_::Timeout::Finite(Duration::from_millis(50)))
+            < 0
+        {
+            break;
+        }
+        let scan = scan.borrow();
+        if scan.answered {
+            let capturable = !scan.failed && scan.sinks.intersection(&scan.monitors).next().is_some();
+            return capturable.then_some(MONITOR);
+        }
+    }
+    None
+}
+
+/// s16le interleaved: the bytes of one sample frame across every channel.
+pub const BYTES_PER_FRAME: u64 = 2 * CHANNELS as u64;
+
+/// 100 ms per chunk: prompt enough to show progress, calm enough that one
+/// chunk is worth its own file write and its own change event.
+const CHUNK_FRAMES: usize = SAMPLE_RATE as usize / 10;
+const CHUNK_BYTES: usize = CHUNK_FRAMES * BYTES_PER_FRAME as usize;
+
+/// Chunks that may wait for the consumer; past this the newest chunk is
+/// dropped and counted, so what is read stays in order.
+const QUEUE_CHUNKS: usize = 8;
+
+/// One captured chunk, s16le interleaved, stamped on the engine clock.
+pub struct Chunk {
+    pub seq: u64,
+    pub at: Instant,
+    pub bytes: Vec<u8>,
+    /// How far this chunk's start fell behind the previous chunk's end: an
+    /// under-run reads as a gap, never as silence.
+    pub gap_micros: u64,
+}
+
+/// A chunk announcement that borrows nothing, so the callback runs outside
+/// the lock.
+#[derive(Clone, Copy)]
+pub struct ChunkNotice {
+    pub seq: u64,
+    pub at: Instant,
+    pub bytes: u64,
+    pub gap_micros: u64,
+}
+
+#[derive(Default)]
+struct Queue {
+    chunks: VecDeque<Chunk>,
+    frames: u64,
+    bytes: u64,
+    dropped_bytes: u64,
+    gap_micros: u64,
+}
+
+/// What the capture thread and its session share.
+struct Shared {
+    queue: Mutex<Queue>,
+    ready: Mutex<Option<mpsc::Sender<Result<(), String>>>>,
+    on_chunk: Box<dyn Fn(ChunkNotice) + Send + Sync>,
+}
+
+impl Shared {
+    fn report(&self, result: Result<(), String>) {
+        if let Some(ready) = lock(&self.ready).take() {
+            let _ = ready.send(result);
+        }
+    }
+}
+
+/// A running capture. Dropping it quits the PipeWire loop and joins the
+/// thread; the session holds it for the session's lifetime.
+pub struct AudioCapture {
+    shared: Arc<Shared>,
+    quit: pw::channel::Sender<()>,
+    thread: Option<thread::JoinHandle<()>>,
+}
+
+impl AudioCapture {
+    /// Capture the default sink's monitor over this session's own PipeWire
+    /// connection. Returns once the stream is up on the negotiated format; a
+    /// machine with no capturable sink fails here with a readable reason.
+    pub fn start(on_chunk: Box<dyn Fn(ChunkNotice) + Send + Sync>) -> Result<Self, String> {
+        let (ready, ready_rx) = mpsc::channel();
+        let shared = Arc::new(Shared {
+            queue: Mutex::new(Queue::default()),
+            ready: Mutex::new(Some(ready)),
+            on_chunk,
+        });
+        let (quit, stop) = pw::channel::channel();
+        let thread = {
+            let shared = Arc::clone(&shared);
+            thread::Builder::new()
+                .name("desklink-audio".into())
+                .spawn(move || {
+                    if let Err(error) = run(stop, Arc::clone(&shared)) {
+                        // After the stream is up this is meaningless; before
+                        // that it is the only path the reason has out.
+                        shared.report(Err(format!("{error:#}")));
+                    }
+                })
+                .map_err(|error| format!("failed to spawn audio capture: {error}"))?
+        };
+        let reason = match ready_rx.recv_timeout(Duration::from_secs(5)) {
+            Ok(Ok(())) => return Ok(Self { shared, quit, thread: Some(thread) }),
+            Ok(Err(reason)) => reason,
+            Err(_) => String::from(
+                "the audio stream did not start within 5s: no sink to capture, or no session manager to link it",
+            ),
+        };
+        let _ = quit.send(());
+        let _ = thread.join();
+        Err(reason)
+    }
+
+    /// The next chunk in order, or `None` once everything captured is read.
+    pub fn take(&self) -> Option<Chunk> {
+        lock(&self.shared.queue).chunks.pop_front()
+    }
+
+    /// Captured frames, captured bytes, dropped bytes, under-run microseconds.
+    pub fn totals(&self) -> (u64, u64, u64, u64) {
+        let queue = lock(&self.shared.queue);
+        (queue.frames, queue.bytes, queue.dropped_bytes, queue.gap_micros)
+    }
+}
+
+impl Drop for AudioCapture {
+    fn drop(&mut self) {
+        let _ = self.quit.send(());
+        if let Some(thread) = self.thread.take() {
+            let _ = thread.join();
+        }
+    }
+}
+
+/// The capture thread's own state: the chunk being filled, where it started,
+/// when the last one ended, and the next sequence number.
+#[derive(Default)]
+struct Stream {
+    pending: Vec<u8>,
+    started: Option<Instant>,
+    end: Option<Instant>,
+    seq: u64,
+}
+
+fn chunk_span() -> Duration {
+    Duration::from_secs_f64(CHUNK_FRAMES as f64 / SAMPLE_RATE as f64)
+}
+
+/// Hand one chunk over, or drop and count it when the consumer lags; the
+/// sequence always advances, so a drop reads as a gap, not contiguous audio.
+fn enqueue(shared: &Shared, state: &mut Stream, bytes: Vec<u8>, started: Instant) {
+    state.seq += 1;
+    let len = bytes.len() as u64;
+    let gap = state.end.map_or(Duration::ZERO, |end| started.saturating_duration_since(end));
+    state.end = Some(started + chunk_span());
+    let notice = ChunkNotice { seq: state.seq, at: started, bytes: len, gap_micros: gap.as_micros() as u64 };
+    let chunk = Chunk { seq: notice.seq, at: started, bytes, gap_micros: notice.gap_micros };
+    {
+        let mut queue = lock(&shared.queue);
+        queue.frames += len / BYTES_PER_FRAME;
+        queue.bytes += len;
+        queue.gap_micros += notice.gap_micros;
+        if queue.chunks.len() < QUEUE_CHUNKS {
+            queue.chunks.push_back(chunk);
+        } else {
+            queue.dropped_bytes += len;
+        }
+    }
+    (shared.on_chunk)(notice);
+}
+
+fn run(stop: pw::channel::Receiver<()>, shared: Arc<Shared>) -> anyhow::Result<()> {
+    pw::init();
+    let main_loop = pw::main_loop::MainLoopRc::new(None).context("pw_main_loop_new failed")?;
+    let _stop = stop.attach(main_loop.loop_(), {
+        let main_loop = main_loop.clone();
+        move |_| main_loop.quit()
+    });
+    let context = pw::context::ContextBox::new(main_loop.loop_(), None).context("pw_context_new failed")?;
+    // The video path arrives holding a portal fd; audio has none and connects
+    // to this session's own PipeWire, which is what makes a private runtime
+    // directory the isolation boundary.
+    let core = context.connect(None).context("pw_context_connect failed")?;
+
+    let props = pw::properties::properties! {
+        *pw::keys::MEDIA_TYPE => "Audio",
+        *pw::keys::MEDIA_CATEGORY => "Capture",
+        *pw::keys::MEDIA_ROLE => "Music",
+        // The sink's own output, not a source's input: the desktop's mix.
+        *pw::keys::STREAM_CAPTURE_SINK => "true",
+    };
+    let stream =
+        pw::stream::StreamBox::new(&core, "desklink-audio", props).context("pw_stream_new failed")?;
+
+    let listener = stream
+        .add_local_listener_with_user_data(Stream::default())
+        .state_changed({
+            let shared = Arc::clone(&shared);
+            move |_, _, _old, new| match new {
+                pw::stream::StreamState::Streaming => shared.report(Ok(())),
+                pw::stream::StreamState::Error(message) => shared.report(Err(format!("PipeWire audio stream stopped: {message}"))),
+                _ => {}
+            }
+        })
+        .process({
+            let shared = Arc::clone(&shared);
+            move |stream, state| {
+                let Some(mut buffer) = stream.dequeue_buffer() else { return };
+                let datas = buffer.datas_mut();
+                let Some(data) = datas.first_mut() else { return };
+                let c = data.chunk();
+                let (offset, size) = (c.offset() as usize, c.size() as usize);
+                let Some(bytes) = data.data() else { return };
+                let Some(bytes) = bytes.get(offset..offset.saturating_add(size)) else { return };
+                if bytes.is_empty() { return; }
+                // A chunk starts when its first byte arrives, so a stall reads
+                // as a gap; a split inside one buffer keeps the sample-derived span.
+                let mut started = Instant::now();
+                if !state.pending.is_empty() {
+                    started = state.started.unwrap_or(started);
+                }
+                state.pending.extend_from_slice(bytes);
+                while state.pending.len() >= CHUNK_BYTES {
+                    let rest = state.pending.split_off(CHUNK_BYTES);
+                    let filled = std::mem::replace(&mut state.pending, rest);
+                    enqueue(&shared, state, filled, started);
+                    started += chunk_span();
+                }
+                state.started = (!state.pending.is_empty()).then_some(started);
+            }
+        })
+        .register()
+        .context("pw_stream_add_listener failed")?;
+
+    // Ask for the one format this seam carries; `audioconvert` adapts the sink.
+    let mut info = pw::spa::param::audio::AudioInfoRaw::new();
+    info.set_format(pw::spa::param::audio::AudioFormat::S16LE);
+    info.set_rate(SAMPLE_RATE);
+    info.set_channels(CHANNELS);
+    let object = pw::spa::pod::Object {
+        type_: pw::spa::utils::SpaTypes::ObjectParamFormat.as_raw(),
+        id: pw::spa::param::ParamType::EnumFormat.as_raw(),
+        properties: info.into(),
+    };
+    let pod = pw::spa::pod::serialize::PodSerializer::serialize(
+        std::io::Cursor::new(Vec::new()),
+        &pw::spa::pod::Value::Object(object),
+    )
+    .map_err(|error| anyhow::anyhow!("format pod serialization failed: {error}"))?;
+    let pod = pod.0.into_inner();
+    let mut params =
+        [pw::spa::pod::Pod::from_bytes(&pod).context("the format pod is not a valid pod")?];
+    stream
+        .connect(
+            pw::spa::utils::Direction::Input,
+            None,
+            pw::stream::StreamFlags::AUTOCONNECT | pw::stream::StreamFlags::MAP_BUFFERS,
+            &mut params,
+        )
+        .context("pw_stream_connect failed")?;
+
+    main_loop.run();
+    drop(listener);
+    Ok(())
+}
+
+fn lock<T>(m: &Mutex<T>) -> MutexGuard<'_, T> {
+    m.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
+}

package/engine/src/mac.rs

@@ -184,6 +184,7 @@ fn capabilities() -> Value {
             "formats": ["bgra"],
             "cursor": "embedded",
             "cursor_modes": ["embedded"],
+            // ScreenCaptureKit audio is not wired up; macOS is out of scope.
             "audio": false,
             "displays": displays(),
             "grant": if capture_granted { "granted" } else { "missing-screen-recording" },

package/engine/src/main.rs

@@ -6,6 +6,8 @@
 //! process, owns its stdin/stdout, and carries the SDP/ICE the engine produces
 //! over whatever authenticated channel it already has.
 
+#[cfg(target_os = "linux")]
+mod audio;
 #[cfg(target_os = "linux")]
 mod capture;
 #[cfg(target_os = "macos")]
@@ -574,6 +576,15 @@ fn render_event(notice: session::Notice) -> Option<String> {
             "session.frame.changed",
             serde_json::json!({ "sessionId": session_id, "seq": seq, "damage": damage, "timestamp_us": timestamp_us }),
         ),
+        session::SessionEvent::Audio {
+            seq,
+            timestamp_us,
+            bytes,
+            gap_micros,
+        } => (
+            "session.audio.changed",
+            serde_json::json!({ "sessionId": session_id, "seq": seq, "timestamp_us": timestamp_us, "bytes": bytes, "gap_ms": gap_micros / 1000 }),
+        ),
         session::SessionEvent::Input { input } => (
             "session.input",
             serde_json::json!({ "sessionId": session_id, "input": input }),
@@ -658,6 +669,7 @@ async fn dispatch(
                 loopback_tcp: params.loopback_tcp,
                 agent_indicator: params.agent_indicator,
                 local_frames: params.local_frames,
+                audio: params.audio,
                 ice_servers: params
                     .ice_servers
                     .into_iter()
@@ -684,6 +696,7 @@ async fn dispatch(
                 "geometry": session.geometry(),
                 "cursor": session.cursor_info(),
                 "agentIndicator": session.agent_indicator(),
+                "audio": session.audio_info(),
             });
             *current = Some(session);
             Ok(result)
@@ -787,6 +800,15 @@ async fn dispatch(
                 .frame(params.since, &params.path, params.region)
                 .map_err(|error| ErrorBody::new(error.code, error.message))
         }
+        "session.audio" => {
+            let session = require_session(current)?;
+            let params: protocol::AudioParams = serde_json::from_value(request.params.clone())
+                .map_err(|error| ErrorBody::new("malformed", error.to_string()))?;
+            check_session(session, &params.session_id, None)?;
+            session
+                .audio(&params.path, params.max_bytes)
+                .map_err(|error| ErrorBody::new(error.code, error.message))
+        }
         "session.feed" => {
             let session = require_session(current)?;
             let params: protocol::FeedParams = serde_json::from_value(request.params.clone())

package/engine/src/peer.rs

@@ -47,6 +47,33 @@ use webrtc::peer_connection::{
 };
 use webrtc::runtime::default_runtime;
 
+// desklink scout trace (dl-roam-postnom-attr): engine-side restart timeline.
+// Enabled with `DESKLINK_ICE_TRACE=1`; stamps microseconds since the Unix epoch
+// so these lines interleave with the vendored ICE agent's `[ice ...]` lines in
+// one order. Droppable: grep `desklink scout trace`.
+pub(crate) fn dl_trace_on() -> bool {
+    std::env::var("DESKLINK_ICE_TRACE")
+        .map(|v| v != "0" && !v.is_empty())
+        .unwrap_or(false)
+}
+
+pub(crate) fn dl_trace_us() -> u128 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .map(|d| d.as_micros())
+        .unwrap_or(0)
+}
+
+macro_rules! dl_eng {
+    ($($arg:tt)*) => {
+        if $crate::peer::dl_trace_on() {
+            eprintln!("[eng {}] {}", $crate::peer::dl_trace_us(), format_args!($($arg)*));
+        }
+    };
+}
+
+pub(crate) use dl_eng;
+
 /// VP9 clock rate, fixed by RFC 9628.
 const VP9_CLOCK_RATE: u32 = 90_000;
 
@@ -420,6 +447,12 @@ impl PeerConnectionEventHandler for Handler {
     }
 
     async fn on_connection_state_change(&self, state: RTCPeerConnectionState) {
+        // desklink scout trace (dl-roam-postnom-attr): the engine's rtc-level
+        // transport state event, on the same microsecond clock as the vendored
+        // agent's `[ice ...]` lines, so the rtc Connected gate is separable.
+        if dl_trace_on() {
+            dl_eng!("peer_state_change {state:?}");
+        }
         let connected = matches!(state, RTCPeerConnectionState::Connected);
         if connected {
             self.wants_keyframe.store(true, Ordering::SeqCst);
@@ -1590,6 +1623,15 @@ pub(crate) mod tests {
             .expect("the restart answer is accepted");
     }
 
+    /// How long the restarted generation may take to nominate and reconnect,
+    /// measured from the answered offer: two `check_interval` ticks. The vendored
+    /// agent ticks every 200 ms (`vendor/rtc-ice/src/agent/agent_config.rs`), and
+    /// the routed roam leg measured the engine's own first-valid-pair ->
+    /// USE-CANDIDATE at 197 ms, one tick; the extra tick is headroom for the
+    /// loopback's round trip and for a loaded machine. See the assertion in
+    /// `a_restart_keeps_media_flowing_on_the_selected_pair_until_the_new_one_nominates`.
+    const RECONNECT_BUDGET: Duration = Duration::from_millis(400);
+
     /// A receiver peer that records when each inbound RTP packet arrives and
     /// reports its own ICE candidates, so two peers can really connect.
     struct MediaReceiver {
@@ -1783,6 +1825,37 @@ pub(crate) mod tests {
             .accept_answer(reanswer.sdp)
             .await
             .expect("the restart answer is accepted");
+        let answered_at = Instant::now();
+
+        // The restarted generation has to be nominated off its own check cycle:
+        // nomination is decided in the agent's `contact_candidates` pass, which the
+        // driver reaches on a `check_interval` tick (200 ms) or when an inbound
+        // Binding request arrives while a valid pair already exists. A pair that
+        // becomes valid just after a tick would therefore wait a full tick for its
+        // USE-CANDIDATE, and one tick is the whole engine-side cost measured on the
+        // routed roam leg (first valid pair -> USE-CANDIDATE: 197 ms, i.e. one tick).
+        //
+        // What this asserts, and what it cannot: the engine's "first valid pair"
+        // instant is not observable from here (the peer connection exposes no pair
+        // state), so the bound is measured from the answer, the earliest instant the
+        // engine could nominate, at two ticks. The loopback's client answers checks
+        // and sends its own triggered checks in the same burst, so an inbound request
+        // is nearly always available and the base engine meets this bound too; the
+        // tick itself is graded by the routed roam leg, whose relay log carries both
+        // instants (197 ms, one tick). This guards the property going forward: a
+        // lost pass, a skipped tick or a stalled nomination blows the bound.
+        let reconnected = async {
+            while !engine.is_connected() {
+                tokio::time::sleep(Duration::from_millis(5)).await;
+            }
+            answered_at.elapsed()
+        };
+        let reconnect = tokio::time::timeout(RECONNECT_BUDGET, reconnected)
+            .await
+            .expect("the restarted generation was never nominated");
+        eprintln!(
+            "restart: transport reconnected {reconnect:?} after the answer (budget {RECONNECT_BUDGET:?})"
+        );
 
         // Keep the picture advancing across and after the restart.
         tokio::time::sleep(Duration::from_millis(800)).await;

package/engine/src/protocol.rs

@@ -144,6 +144,11 @@ pub struct OpenParams {
     /// the engine then spends nothing per frame on either.
     #[serde(default = "default_local_frames")]
     pub local_frames: bool,
+    /// Capture the desktop's own system audio for `session.audio`. Absent means
+    /// video only; there is never a silent downgrade, so a caller that asks and
+    /// cannot be answered is refused.
+    #[serde(default)]
+    pub audio: bool,
 }
 
 fn default_local_frames() -> bool {
@@ -244,6 +249,23 @@ pub struct SessionRef {
     pub generation: Option<u64>,
 }
 
+/// Read the desktop's own audio, the way `FrameParams` reads a frame: bytes go
+/// to the caller's path and JSON carries only metadata.
+#[derive(Debug, Deserialize)]
+pub struct AudioParams {
+    pub session_id: String,
+    #[serde(default)]
+    pub path: String,
+    /// The most bytes to write. Bounded to whole sample frames, so a truncated
+    /// chunk never ends mid-sample.
+    #[serde(default = "default_max_audio_bytes")]
+    pub max_bytes: usize,
+}
+
+fn default_max_audio_bytes() -> usize {
+    1 << 20
+}
+
 #[derive(Debug, Deserialize)]
 pub struct DescriptionParams {
     pub session_id: String,