@desklink/host 0.1.0 → 0.2.0

package/engine/src/x11.rs

@@ -83,6 +83,15 @@ impl X11Desktop {
     /// compositor this is every window drawn onto the framebuffer, which is what
     /// the viewer expects to see.
     pub fn capture(&mut self, max_width: usize, max_height: usize) -> Result<I420> {
+        self.capture_with_pixels(max_width, max_height)
+            .map(|(frame, _)| frame)
+    }
+
+    pub fn capture_with_pixels(
+        &mut self,
+        max_width: usize,
+        max_height: usize,
+    ) -> Result<(I420, Vec<u8>)> {
         let image = self
             .connection
             .get_image(
@@ -110,16 +119,27 @@ impl X11Desktop {
         // The server's depth is the only field that can tell us how the pixel is
         // packed; the engine assumes the byte order every TrueColor server uses.
         let _ = self.depth;
-        to_i420(
+        let frame = to_i420(
+            &image.data,
+            self.width,
+            self.height,
+            stride,
+            crate::convert::PixelFormat::Bgrx,
+            width,
+            height,
+        )
+        .context("the captured X11 pixels are not a format this engine can read")?;
+        let raw = crate::convert::to_bgrx(
             &image.data,
             self.width,
             self.height,
             stride,
-            crate::capture::PixelFormat::Bgrx,
+            crate::convert::PixelFormat::Bgrx,
             width,
             height,
         )
-        .context("the captured X11 pixels are not a format this engine can read")
+        .context("the captured X11 pixels could not be copied")?;
+        Ok((frame, raw))
     }
 
     fn flush(&self) -> Result<()> {

package/engine/src/x11_mac.rs

@@ -0,0 +1,40 @@
+use crate::convert::I420;
+use anyhow::{bail, Result};
+
+pub mod button {
+    pub const LEFT: u8 = 1;
+    pub const MIDDLE: u8 = 2;
+    pub const RIGHT: u8 = 3;
+}
+
+pub struct X11Desktop;
+
+impl X11Desktop {
+    pub fn connect(_: Option<&str>) -> Result<Self> {
+        bail!("the X11 backend is only available on Linux")
+    }
+
+    pub fn screen_size(&self) -> (i32, i32) {
+        (0, 0)
+    }
+
+    pub fn capture_with_pixels(&mut self, _: usize, _: usize) -> Result<(I420, Vec<u8>)> {
+        bail!("the X11 backend is only available on Linux")
+    }
+
+    pub fn move_pointer(&mut self, _: i64, _: i64) -> Result<()> {
+        bail!("the X11 backend is only available on Linux")
+    }
+
+    pub fn button(&mut self, _: u8, _: bool) -> Result<()> {
+        bail!("the X11 backend is only available on Linux")
+    }
+
+    pub fn scroll(&mut self, _: i64, _: i64) -> Result<()> {
+        bail!("the X11 backend is only available on Linux")
+    }
+
+    pub fn key(&mut self, _: i16, _: bool) -> Result<()> {
+        bail!("the X11 backend is only available on Linux")
+    }
+}

package/examples/reference-client.html

@@ -67,12 +67,15 @@ const overlay = document.getElementById('overlay');
 const status = document.getElementById('status');
 const reconnect = document.getElementById('reconnect');
 const keyboard = document.getElementById('remote-keyboard');
+const keyboardButton = document.getElementById('keyboard');
 const copy = document.getElementById('copy');
 const paste = document.getElementById('paste');
 
 let socket = null;
 let peer = null;
 let channel = null;
+let currentSessionId = null;
+let pendingRemoteCandidates = [];
 let surface = { width: 0, height: 0 };
 let nextSeq = 0;
 let pending = new Map();
@@ -265,10 +268,16 @@ async function start() {
   try {
     const capabilities = await request('capabilities');
     const clipboardAvailable = capabilities.clipboard?.read === true && capabilities.clipboard?.write === true;
-    const session = await request('session.open', {
-      permissions: clipboardAvailable ? ['view', 'control', 'clipboard'] : ['view', 'control'],
-      max_fps: 30,
-    });
+    const controlAvailable = capabilities.input?.pointer === true && capabilities.input?.keyboard === true;
+    const permissions = ['view'];
+    if (controlAvailable) permissions.push('control');
+    if (clipboardAvailable) permissions.push('clipboard');
+    const session = await request('session.open', { permissions, max_fps: 30 });
+    currentSessionId = session.sessionId;
+    keyboard.disabled = !controlAvailable;
+    keyboardButton.disabled = !controlAvailable;
+    keyboardButton.title = controlAvailable ? '' : capabilities.input?.unavailable_reason?.remedy ?? 'This session is view-only';
+    video.style.pointerEvents = controlAvailable ? 'auto' : 'none';
     copy.hidden = !clipboardAvailable;
     paste.hidden = !clipboardAvailable;
     surface = session.geometry.encoded;
@@ -332,6 +341,9 @@ async function start() {
     // The engine is the offerer, so this side waits for the offer and answers.
     const offer = await waitForOffer();
     await pc.setRemoteDescription({ type: 'offer', sdp: offer });
+    for (const pending of pendingRemoteCandidates.splice(0)) {
+      if (pending.sessionId === session.sessionId) await pc.addIceCandidate(pending.candidate);
+    }
     const answer = await pc.createAnswer();
     await pc.setLocalDescription(answer);
     await request('session.description', {
@@ -371,6 +383,8 @@ function teardown(message) {
   peer?.close();
   peer = null;
   channel = null;
+  currentSessionId = null;
+  pendingRemoteCandidates = [];
   video.srcObject = null;
   copy.hidden = true;
   paste.hidden = true;
@@ -392,6 +406,21 @@ function connectSocket() {
       else waiting.resolve(message.result);
       return;
     }
+    if (message.event === 'session.candidate') {
+      const params = message.params;
+      if (currentSessionId !== null && params.sessionId !== currentSessionId) return;
+      const candidate = {
+        candidate: params.candidate,
+        sdpMid: params.sdpMid,
+        sdpMLineIndex: params.sdpMLineIndex,
+      };
+      if (peer?.remoteDescription) {
+        void peer.addIceCandidate(candidate).catch(() => undefined);
+      } else {
+        pendingRemoteCandidates.push({ sessionId: params.sessionId, candidate });
+      }
+      return;
+    }
     if (message.event === 'session.description') {
       const sdp = message.params.description.sdp;
       if (offerWaiter !== null) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@desklink/host",
-    "version": "0.1.0",
+    "version": "0.2.0",
     "license": "Apache-2.0",
     "description": "Per-user, on-demand desktop capture, encode and input engine with a versioned local control protocol",
     "type": "module",
@@ -33,6 +33,6 @@
         "ws": "^8.18.0"
     },
     "optionalDependencies": {
-        "@desklink/host-linux-x64-gnu": "0.1.0"
+        "@desklink/host-linux-x64-gnu": "0.2.0"
     }
 }