@desklink/host 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +3 -0
- package/README.md +50 -16
- package/bin/desklink-host.mjs +13 -2
- package/dist/bridge.d.ts.map +1 -1
- package/dist/bridge.js +5 -1
- package/dist/bridge.js.map +1 -1
- package/dist/engineProcess.d.ts.map +1 -1
- package/dist/engineProcess.js +17 -5
- package/dist/engineProcess.js.map +1 -1
- package/dist/protocol.d.ts +42 -5
- package/dist/protocol.d.ts.map +1 -1
- package/dist/protocol.js +1 -1
- package/dist/protocol.js.map +1 -1
- package/dist/resolveEngine.d.ts +1 -0
- package/dist/resolveEngine.d.ts.map +1 -1
- package/dist/resolveEngine.js +16 -1
- package/dist/resolveEngine.js.map +1 -1
- package/docs/BOUNDARY.md +4 -3
- package/docs/PROTOCOL.md +95 -27
- package/engine/Cargo.lock +195 -1
- package/engine/Cargo.toml +25 -5
- package/engine/build.rs +98 -23
- package/engine/native/agent_overlay_mac.m +133 -0
- package/engine/native/agent_overlay_wayland.c +192 -0
- package/engine/native/agent_overlay_x11.c +103 -0
- package/engine/native/mac_stream.mm +188 -0
- package/engine/src/capture.rs +169 -19
- package/engine/src/capture_mac.rs +145 -0
- package/engine/src/clipboard_mac.rs +182 -0
- package/engine/src/convert.rs +57 -1
- package/engine/src/indicator.rs +162 -0
- package/engine/src/input.rs +12 -0
- package/engine/src/input_mac.rs +312 -0
- package/engine/src/keymap.rs +25 -0
- package/engine/src/keymap_mac.rs +209 -0
- package/engine/src/mac.rs +722 -0
- package/engine/src/main.rs +116 -2
- package/engine/src/portal.rs +4 -0
- package/engine/src/portal_mac.rs +68 -0
- package/engine/src/protocol.rs +48 -1
- package/engine/src/session.rs +556 -62
- package/engine/src/x11.rs +23 -3
- package/engine/src/x11_mac.rs +40 -0
- package/examples/reference-client.html +33 -4
- package/package.json +2 -2
package/engine/src/capture.rs
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
//! Everything PipeWire lives on this module's own thread because its objects are
|
|
6
6
|
//! not `Send`; frames and state changes leave through channels.
|
|
7
7
|
|
|
8
|
-
use crate::convert::{to_i420, I420};
|
|
8
|
+
use crate::convert::{to_i420, PixelFormat, I420};
|
|
9
9
|
use crate::portal::{PortalSession, SelectedSource};
|
|
10
10
|
use anyhow::{Context, Result};
|
|
11
11
|
use pipewire as pw;
|
|
@@ -16,18 +16,7 @@ use std::os::fd::OwnedFd;
|
|
|
16
16
|
use std::sync::atomic::{AtomicU64, Ordering};
|
|
17
17
|
use std::sync::{mpsc, Arc, Mutex};
|
|
18
18
|
use std::thread::JoinHandle;
|
|
19
|
-
|
|
20
|
-
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
|
|
21
|
-
#[serde(rename_all = "snake_case")]
|
|
22
|
-
pub enum PixelFormat {
|
|
23
|
-
Bgrx,
|
|
24
|
-
Rgbx,
|
|
25
|
-
Bgra,
|
|
26
|
-
Rgba,
|
|
27
|
-
Bgr,
|
|
28
|
-
Rgb,
|
|
29
|
-
Unsupported,
|
|
30
|
-
}
|
|
19
|
+
use std::time::{Duration, Instant};
|
|
31
20
|
|
|
32
21
|
impl PixelFormat {
|
|
33
22
|
fn from_spa(raw: u32) -> Self {
|
|
@@ -55,7 +44,7 @@ pub struct StreamGeometry {
|
|
|
55
44
|
pub buffer_type: String,
|
|
56
45
|
}
|
|
57
46
|
|
|
58
|
-
pub type FrameSink = Box<dyn Fn(I420, u64) + Send + 'static>;
|
|
47
|
+
pub type FrameSink = Box<dyn Fn(I420, u64, Vec<u8>) + Send + 'static>;
|
|
59
48
|
|
|
60
49
|
/// A running capture. Dropping it quits the PipeWire loop and joins its thread.
|
|
61
50
|
pub struct Capture {
|
|
@@ -206,6 +195,8 @@ pub fn start(
|
|
|
206
195
|
encoded_height: usize,
|
|
207
196
|
max_fps: u32,
|
|
208
197
|
sink: FrameSink,
|
|
198
|
+
indicator_position: Option<Arc<Mutex<Option<(i64, i64)>>>>,
|
|
199
|
+
on_stop: Box<dyn Fn(String) + Send>,
|
|
209
200
|
) -> Result<Capture> {
|
|
210
201
|
let PortalSession { fd, source, .. } = session;
|
|
211
202
|
let geometry = Arc::new(Mutex::new(None));
|
|
@@ -230,6 +221,8 @@ pub fn start(
|
|
|
230
221
|
encoded_height,
|
|
231
222
|
max_fps,
|
|
232
223
|
sink,
|
|
224
|
+
indicator_position,
|
|
225
|
+
on_stop,
|
|
233
226
|
geometry,
|
|
234
227
|
frames,
|
|
235
228
|
dropped,
|
|
@@ -275,6 +268,8 @@ fn run_loop(
|
|
|
275
268
|
encoded_height: usize,
|
|
276
269
|
max_fps: u32,
|
|
277
270
|
sink: FrameSink,
|
|
271
|
+
indicator_position: Option<Arc<Mutex<Option<(i64, i64)>>>>,
|
|
272
|
+
on_stop: Box<dyn Fn(String) + Send>,
|
|
278
273
|
geometry: Arc<Mutex<Option<StreamGeometry>>>,
|
|
279
274
|
frames: Arc<AtomicU64>,
|
|
280
275
|
dropped: Arc<AtomicU64>,
|
|
@@ -309,6 +304,8 @@ fn run_loop(
|
|
|
309
304
|
format: None,
|
|
310
305
|
buffer_type: String::from("unknown"),
|
|
311
306
|
sink,
|
|
307
|
+
on_stop,
|
|
308
|
+
mask: indicator_position.map(IndicatorMask::new),
|
|
312
309
|
geometry,
|
|
313
310
|
frames,
|
|
314
311
|
dropped,
|
|
@@ -319,6 +316,14 @@ fn run_loop(
|
|
|
319
316
|
let mut ready = Some(ready);
|
|
320
317
|
let _listener = stream
|
|
321
318
|
.add_local_listener_with_user_data(&mut state)
|
|
319
|
+
.state_changed({ let main_loop = main_loop.clone(); move |_, state, old, new| {
|
|
320
|
+
let reason = match new {
|
|
321
|
+
pw::stream::StreamState::Error(message) => Some(format!("PipeWire stream stopped: {message}")),
|
|
322
|
+
pw::stream::StreamState::Unconnected if old == pw::stream::StreamState::Streaming => Some(String::from("PipeWire stream disconnected")),
|
|
323
|
+
_ => None,
|
|
324
|
+
};
|
|
325
|
+
if let Some(reason) = reason { (state.on_stop)(reason); main_loop.quit(); }
|
|
326
|
+
} })
|
|
322
327
|
.param_changed(|_, state, id, param| {
|
|
323
328
|
if id != pw::spa::param::ParamType::Format.as_raw() {
|
|
324
329
|
return;
|
|
@@ -378,8 +383,31 @@ fn run_loop(
|
|
|
378
383
|
// second time and hand the encoder a frame it refuses whenever the
|
|
379
384
|
// stream's size differs from the one the portal reported.
|
|
380
385
|
let seq = state.frames.load(Ordering::Relaxed);
|
|
381
|
-
|
|
382
|
-
|
|
386
|
+
let raw = crate::convert::to_bgrx(
|
|
387
|
+
chunk_bytes,
|
|
388
|
+
w,
|
|
389
|
+
h,
|
|
390
|
+
stride,
|
|
391
|
+
format,
|
|
392
|
+
state.box_w,
|
|
393
|
+
state.box_h,
|
|
394
|
+
);
|
|
395
|
+
if let Some(mut raw) = raw {
|
|
396
|
+
let frame = if let Some(mask) = state.mask.as_mut() {
|
|
397
|
+
mask.apply(&mut raw, state.box_w, state.box_h, w, h);
|
|
398
|
+
to_i420(
|
|
399
|
+
&raw,
|
|
400
|
+
state.box_w,
|
|
401
|
+
state.box_h,
|
|
402
|
+
state.box_w * 4,
|
|
403
|
+
PixelFormat::Bgrx,
|
|
404
|
+
state.box_w,
|
|
405
|
+
state.box_h,
|
|
406
|
+
)
|
|
407
|
+
} else {
|
|
408
|
+
to_i420(chunk_bytes, w, h, stride, format, state.box_w, state.box_h)
|
|
409
|
+
};
|
|
410
|
+
if let Some(i420) = frame {
|
|
383
411
|
if state.geometry.lock().map(|g| g.is_none()).unwrap_or(false) {
|
|
384
412
|
if let Ok(mut g) = state.geometry.lock() {
|
|
385
413
|
*g = Some(StreamGeometry {
|
|
@@ -393,7 +421,7 @@ fn run_loop(
|
|
|
393
421
|
}
|
|
394
422
|
}
|
|
395
423
|
state.frames.fetch_add(1, Ordering::Relaxed);
|
|
396
|
-
(state.sink)(i420, seq);
|
|
424
|
+
(state.sink)(i420, seq, raw);
|
|
397
425
|
// The first frame the loop actually delivers is the only
|
|
398
426
|
// proof capture started; sending this when `run_loop`
|
|
399
427
|
// returns would be after the main loop quits, too late for
|
|
@@ -401,10 +429,11 @@ fn run_loop(
|
|
|
401
429
|
if let Some(sender) = ready.take() {
|
|
402
430
|
let _ = sender.send(Ok(()));
|
|
403
431
|
}
|
|
404
|
-
}
|
|
405
|
-
None => {
|
|
432
|
+
} else {
|
|
406
433
|
state.dropped.fetch_add(1, Ordering::Relaxed);
|
|
407
434
|
}
|
|
435
|
+
} else {
|
|
436
|
+
state.dropped.fetch_add(1, Ordering::Relaxed);
|
|
408
437
|
}
|
|
409
438
|
})
|
|
410
439
|
.register()
|
|
@@ -432,12 +461,116 @@ fn run_loop(
|
|
|
432
461
|
Ok(())
|
|
433
462
|
}
|
|
434
463
|
|
|
464
|
+
// The portal composites layer-shell into its pixels. Keep the last clean
|
|
465
|
+
// backing for the helper's bounded circle and edge strips before diff/encode;
|
|
466
|
+
// the center stays live so a click marker at the caret remains observable.
|
|
467
|
+
struct IndicatorMask {
|
|
468
|
+
position: Arc<Mutex<Option<(i64, i64)>>>,
|
|
469
|
+
clean: Option<Vec<u8>>,
|
|
470
|
+
last: Option<(i64, i64)>,
|
|
471
|
+
from: Option<(i64, i64)>,
|
|
472
|
+
moved_at: Option<Instant>,
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
impl IndicatorMask {
|
|
476
|
+
fn new(position: Arc<Mutex<Option<(i64, i64)>>>) -> Self {
|
|
477
|
+
Self {
|
|
478
|
+
position,
|
|
479
|
+
clean: None,
|
|
480
|
+
last: None,
|
|
481
|
+
from: None,
|
|
482
|
+
moved_at: None,
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
fn apply(
|
|
487
|
+
&mut self,
|
|
488
|
+
raw: &mut [u8],
|
|
489
|
+
width: usize,
|
|
490
|
+
height: usize,
|
|
491
|
+
source_w: usize,
|
|
492
|
+
source_h: usize,
|
|
493
|
+
) {
|
|
494
|
+
let position = self.position.lock().ok().and_then(|value| *value);
|
|
495
|
+
if position != self.last {
|
|
496
|
+
self.from = self.last;
|
|
497
|
+
self.last = position;
|
|
498
|
+
self.moved_at = Some(Instant::now());
|
|
499
|
+
}
|
|
500
|
+
if let Some(clean) = self.clean.as_ref().filter(|clean| clean.len() == raw.len()) {
|
|
501
|
+
let x = position.map_or(-1000, |point| point.0 * width as i64 / source_w as i64);
|
|
502
|
+
let y = position.map_or(-1000, |point| point.1 * height as i64 / source_h as i64);
|
|
503
|
+
let from = self
|
|
504
|
+
.from
|
|
505
|
+
.filter(|_| {
|
|
506
|
+
self.moved_at
|
|
507
|
+
.is_some_and(|at| at.elapsed() < Duration::from_millis(800))
|
|
508
|
+
})
|
|
509
|
+
.map(|point| {
|
|
510
|
+
(
|
|
511
|
+
point.0 * width as i64 / source_w as i64,
|
|
512
|
+
point.1 * height as i64 / source_h as i64,
|
|
513
|
+
)
|
|
514
|
+
});
|
|
515
|
+
let radius = (150 * width / source_w).max(55) as i64;
|
|
516
|
+
let edge = (20 * width / source_w)
|
|
517
|
+
.max(8)
|
|
518
|
+
.min(width / 2)
|
|
519
|
+
.min(height / 2);
|
|
520
|
+
for row in 0..height {
|
|
521
|
+
let start = row * width * 4;
|
|
522
|
+
if row < edge || row + edge >= height {
|
|
523
|
+
raw[start..start + width * 4].copy_from_slice(&clean[start..start + width * 4]);
|
|
524
|
+
continue;
|
|
525
|
+
}
|
|
526
|
+
raw[start..start + edge * 4].copy_from_slice(&clean[start..start + edge * 4]);
|
|
527
|
+
let end = start + width * 4;
|
|
528
|
+
raw[end - edge * 4..end].copy_from_slice(&clean[end - edge * 4..end]);
|
|
529
|
+
let start_y = from.map_or(y, |point| y.min(point.1));
|
|
530
|
+
let end_y = from.map_or(y, |point| y.max(point.1));
|
|
531
|
+
if (row as i64) < start_y - radius || (row as i64) > end_y + radius {
|
|
532
|
+
continue;
|
|
533
|
+
}
|
|
534
|
+
let start_x = from.map_or(x, |point| x.min(point.0));
|
|
535
|
+
let end_x = from.map_or(x, |point| x.max(point.0));
|
|
536
|
+
for col in (start_x - radius).max(0) as usize
|
|
537
|
+
..=(end_x + radius).min(width as i64 - 1).max(0) as usize
|
|
538
|
+
{
|
|
539
|
+
let dx = col as i64 - x;
|
|
540
|
+
let dy = row as i64 - y;
|
|
541
|
+
let distance = dx * dx + dy * dy;
|
|
542
|
+
let along = from.is_some_and(|(fx, fy)| {
|
|
543
|
+
let vx = x - fx;
|
|
544
|
+
let vy = y - fy;
|
|
545
|
+
let projection = ((col as i64 - fx) * vx + (row as i64 - fy) * vy)
|
|
546
|
+
.clamp(0, vx * vx + vy * vy);
|
|
547
|
+
let length = vx * vx + vy * vy;
|
|
548
|
+
let (cx, cy) = if length == 0 {
|
|
549
|
+
(x, y)
|
|
550
|
+
} else {
|
|
551
|
+
(fx + projection * vx / length, fy + projection * vy / length)
|
|
552
|
+
};
|
|
553
|
+
(col as i64 - cx).pow(2) + (row as i64 - cy).pow(2) <= radius * radius
|
|
554
|
+
});
|
|
555
|
+
if (distance <= radius * radius || along) && distance > 16 {
|
|
556
|
+
let at = start + col * 4;
|
|
557
|
+
raw[at..at + 4].copy_from_slice(&clean[at..at + 4]);
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
self.clean = Some(raw.to_vec());
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
|
|
435
566
|
struct StreamState {
|
|
436
567
|
box_w: usize,
|
|
437
568
|
box_h: usize,
|
|
438
569
|
format: Option<PixelFormat>,
|
|
439
570
|
buffer_type: String,
|
|
440
571
|
sink: FrameSink,
|
|
572
|
+
on_stop: Box<dyn Fn(String) + Send>,
|
|
573
|
+
mask: Option<IndicatorMask>,
|
|
441
574
|
geometry: Arc<Mutex<Option<StreamGeometry>>>,
|
|
442
575
|
frames: Arc<AtomicU64>,
|
|
443
576
|
dropped: Arc<AtomicU64>,
|
|
@@ -453,6 +586,23 @@ mod tests {
|
|
|
453
586
|
use super::*;
|
|
454
587
|
use std::time::Duration;
|
|
455
588
|
|
|
589
|
+
#[test]
|
|
590
|
+
fn layer_mask_keeps_click_center_and_removes_ring_and_edge() {
|
|
591
|
+
let position = Arc::new(Mutex::new(Some((50, 50))));
|
|
592
|
+
let mut mask = IndicatorMask::new(position.clone());
|
|
593
|
+
mask.apply(&mut vec![1; 100 * 100 * 4], 100, 100, 100, 100);
|
|
594
|
+
let mut frame = vec![2; 100 * 100 * 4];
|
|
595
|
+
mask.apply(&mut frame, 100, 100, 100, 100);
|
|
596
|
+
assert_eq!(frame[(50 * 100 + 50) * 4], 2);
|
|
597
|
+
assert_eq!(frame[(50 * 100 + 60) * 4], 1);
|
|
598
|
+
assert_eq!(frame[0], 1);
|
|
599
|
+
*position.lock().unwrap() = Some((75, 50));
|
|
600
|
+
let mut moved = vec![3; 100 * 100 * 4];
|
|
601
|
+
mask.apply(&mut moved, 100, 100, 100, 100);
|
|
602
|
+
assert_eq!(moved[(50 * 100 + 60) * 4], 1, "eased path is masked");
|
|
603
|
+
assert_eq!(moved[(50 * 100 + 75) * 4], 3, "click center remains live");
|
|
604
|
+
}
|
|
605
|
+
|
|
456
606
|
#[test]
|
|
457
607
|
fn dropping_capture_stops_and_joins_its_live_pipewire_loop() {
|
|
458
608
|
// Exercise the real event-loop/channel lifetime without a portal,
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
use crate::convert::{to_bgrx, to_i420, PixelFormat, I420};
|
|
2
|
+
use crate::portal::PortalSession;
|
|
3
|
+
use anyhow::{bail, Result};
|
|
4
|
+
use std::ffi::{c_char, c_void, CStr};
|
|
5
|
+
use std::sync::Mutex;
|
|
6
|
+
|
|
7
|
+
pub type FrameSink = Box<dyn Fn(I420, u64, Vec<u8>) + Send + 'static>;
|
|
8
|
+
|
|
9
|
+
struct State {
|
|
10
|
+
sink: FrameSink,
|
|
11
|
+
sequence: u64,
|
|
12
|
+
width: usize,
|
|
13
|
+
height: usize,
|
|
14
|
+
status: Box<dyn Fn(bool, String) + Send>,
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
pub struct Capture {
|
|
18
|
+
handle: *mut c_void,
|
|
19
|
+
state: *mut Mutex<State>,
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
unsafe impl Send for Capture {}
|
|
23
|
+
|
|
24
|
+
#[link(name = "dlmacstream", kind = "static")]
|
|
25
|
+
unsafe extern "C" {
|
|
26
|
+
fn dl_mac_stream_start(
|
|
27
|
+
display_id: u32,
|
|
28
|
+
width: usize,
|
|
29
|
+
height: usize,
|
|
30
|
+
fps: u32,
|
|
31
|
+
indicator_pid: u32,
|
|
32
|
+
context: *mut c_void,
|
|
33
|
+
callback: extern "C" fn(*mut c_void, *const u8, usize, usize, usize, usize),
|
|
34
|
+
status: extern "C" fn(*mut c_void, *const c_char, bool),
|
|
35
|
+
error: *mut c_char,
|
|
36
|
+
capacity: usize,
|
|
37
|
+
) -> *mut c_void;
|
|
38
|
+
fn dl_mac_stream_stop(handle: *mut c_void) -> bool;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
extern "C" fn stream_status(context: *mut c_void, reason: *const c_char, running: bool) {
|
|
42
|
+
let state = unsafe { &*(context as *const Mutex<State>) };
|
|
43
|
+
if let Ok(state) = state.lock() {
|
|
44
|
+
let reason = if reason.is_null() { String::from("stream stopped") }
|
|
45
|
+
else { unsafe { CStr::from_ptr(reason) }.to_string_lossy().into_owned() };
|
|
46
|
+
(state.status)(running, reason);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
extern "C" fn receive_frame(
|
|
51
|
+
context: *mut c_void,
|
|
52
|
+
pixels: *const u8,
|
|
53
|
+
length: usize,
|
|
54
|
+
source_width: usize,
|
|
55
|
+
source_height: usize,
|
|
56
|
+
stride: usize,
|
|
57
|
+
) {
|
|
58
|
+
if pixels.is_null()
|
|
59
|
+
|| source_width == 0
|
|
60
|
+
|| source_height == 0
|
|
61
|
+
|| stride
|
|
62
|
+
.checked_mul(source_height)
|
|
63
|
+
.is_none_or(|bytes| bytes > length)
|
|
64
|
+
{
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
let state = unsafe { &*(context as *const Mutex<State>) };
|
|
68
|
+
let Ok(mut state) = state.lock() else { return };
|
|
69
|
+
let source = unsafe { std::slice::from_raw_parts(pixels, length) };
|
|
70
|
+
let raw = to_bgrx(
|
|
71
|
+
source,
|
|
72
|
+
source_width,
|
|
73
|
+
source_height,
|
|
74
|
+
stride,
|
|
75
|
+
PixelFormat::Bgra,
|
|
76
|
+
state.width,
|
|
77
|
+
state.height,
|
|
78
|
+
);
|
|
79
|
+
let frame = to_i420(
|
|
80
|
+
source,
|
|
81
|
+
source_width,
|
|
82
|
+
source_height,
|
|
83
|
+
stride,
|
|
84
|
+
PixelFormat::Bgra,
|
|
85
|
+
state.width,
|
|
86
|
+
state.height,
|
|
87
|
+
);
|
|
88
|
+
if let (Some(frame), Some(raw)) = (frame, raw) {
|
|
89
|
+
(state.sink)(frame, state.sequence, raw);
|
|
90
|
+
state.sequence += 1;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
impl Drop for Capture {
|
|
95
|
+
fn drop(&mut self) {
|
|
96
|
+
unsafe {
|
|
97
|
+
if dl_mac_stream_stop(self.handle) {
|
|
98
|
+
drop(Box::from_raw(self.state));
|
|
99
|
+
} // A timed-out stop may still deliver a callback; leak the context instead of a UAF.
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
pub fn start(
|
|
105
|
+
session: PortalSession,
|
|
106
|
+
width: usize,
|
|
107
|
+
height: usize,
|
|
108
|
+
max_fps: u32,
|
|
109
|
+
sink: FrameSink,
|
|
110
|
+
indicator_pid: Option<u32>,
|
|
111
|
+
status: Box<dyn Fn(bool, String) + Send>,
|
|
112
|
+
) -> Result<Capture> {
|
|
113
|
+
let state = Box::into_raw(Box::new(Mutex::new(State {
|
|
114
|
+
sink,
|
|
115
|
+
sequence: 0,
|
|
116
|
+
width,
|
|
117
|
+
height,
|
|
118
|
+
status,
|
|
119
|
+
})));
|
|
120
|
+
let mut error = [0i8; 512];
|
|
121
|
+
let handle = unsafe {
|
|
122
|
+
dl_mac_stream_start(
|
|
123
|
+
session.source.node_id,
|
|
124
|
+
width,
|
|
125
|
+
height,
|
|
126
|
+
max_fps,
|
|
127
|
+
indicator_pid.unwrap_or(0),
|
|
128
|
+
state.cast(),
|
|
129
|
+
receive_frame,
|
|
130
|
+
stream_status,
|
|
131
|
+
error.as_mut_ptr(),
|
|
132
|
+
error.len(),
|
|
133
|
+
)
|
|
134
|
+
};
|
|
135
|
+
if handle.is_null() {
|
|
136
|
+
unsafe {
|
|
137
|
+
drop(Box::from_raw(state));
|
|
138
|
+
}
|
|
139
|
+
bail!(
|
|
140
|
+
"ScreenCaptureKit stream: {}",
|
|
141
|
+
unsafe { CStr::from_ptr(error.as_ptr()) }.to_string_lossy()
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
Ok(Capture { handle, state })
|
|
145
|
+
}
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
use anyhow::{bail, Result};
|
|
2
|
+
use std::ffi::{CStr, CString};
|
|
3
|
+
use std::os::raw::{c_char, c_void};
|
|
4
|
+
|
|
5
|
+
pub const MAX_CLIPBOARD_BYTES: usize = 262_144;
|
|
6
|
+
|
|
7
|
+
#[link(name = "AppKit", kind = "framework")]
|
|
8
|
+
unsafe extern "C" {
|
|
9
|
+
fn NSApplicationLoad() -> bool;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
#[link(name = "objc")]
|
|
13
|
+
unsafe extern "C" {
|
|
14
|
+
fn objc_getClass(name: *const c_char) -> *mut c_void;
|
|
15
|
+
fn sel_registerName(name: *const c_char) -> *mut c_void;
|
|
16
|
+
fn objc_msgSend();
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
#[link(name = "CoreFoundation", kind = "framework")]
|
|
20
|
+
unsafe extern "C" {
|
|
21
|
+
fn CFRelease(value: *const c_void);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
type Obj = *mut c_void;
|
|
25
|
+
type Sel = *mut c_void;
|
|
26
|
+
|
|
27
|
+
struct Pool(Obj);
|
|
28
|
+
impl Pool {
|
|
29
|
+
fn new() -> Result<Self> {
|
|
30
|
+
let name = CString::new("NSAutoreleasePool").unwrap();
|
|
31
|
+
let class = unsafe { objc_getClass(name.as_ptr()) };
|
|
32
|
+
anyhow::ensure!(!class.is_null(), "AppKit autorelease pool is unavailable");
|
|
33
|
+
let allocated = send0(class, "alloc");
|
|
34
|
+
let pool = send0(allocated, "init");
|
|
35
|
+
anyhow::ensure!(!pool.is_null(), "could not create AppKit autorelease pool");
|
|
36
|
+
Ok(Self(pool))
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
impl Drop for Pool {
|
|
40
|
+
fn drop(&mut self) {
|
|
41
|
+
send0_void(self.0, "drain");
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
fn selector(name: &str) -> Sel {
|
|
46
|
+
let name = CString::new(name).expect("selector has no nul byte");
|
|
47
|
+
unsafe { sel_registerName(name.as_ptr()) }
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
fn send0(receiver: Obj, name: &str) -> Obj {
|
|
51
|
+
let function: unsafe extern "C" fn(Obj, Sel) -> Obj =
|
|
52
|
+
unsafe { std::mem::transmute(objc_msgSend as *const ()) };
|
|
53
|
+
unsafe { function(receiver, selector(name)) }
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
fn send0_void(receiver: Obj, name: &str) {
|
|
57
|
+
let function: unsafe extern "C" fn(Obj, Sel) =
|
|
58
|
+
unsafe { std::mem::transmute(objc_msgSend as *const ()) };
|
|
59
|
+
unsafe { function(receiver, selector(name)) }
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
fn send0_integer(receiver: Obj, name: &str) -> isize {
|
|
63
|
+
let function: unsafe extern "C" fn(Obj, Sel) -> isize =
|
|
64
|
+
unsafe { std::mem::transmute(objc_msgSend as *const ()) };
|
|
65
|
+
unsafe { function(receiver, selector(name)) }
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
fn send1_object(receiver: Obj, name: &str, argument: Obj) -> Obj {
|
|
69
|
+
let function: unsafe extern "C" fn(Obj, Sel, Obj) -> Obj =
|
|
70
|
+
unsafe { std::mem::transmute(objc_msgSend as *const ()) };
|
|
71
|
+
unsafe { function(receiver, selector(name), argument) }
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
fn send2_object(receiver: Obj, name: &str, first: Obj, second: Obj) -> bool {
|
|
75
|
+
let function: unsafe extern "C" fn(Obj, Sel, Obj, Obj) -> bool =
|
|
76
|
+
unsafe { std::mem::transmute(objc_msgSend as *const ()) };
|
|
77
|
+
unsafe { function(receiver, selector(name), first, second) }
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
fn pasteboard() -> Result<Obj> {
|
|
81
|
+
let _ = unsafe { NSApplicationLoad() };
|
|
82
|
+
let class_name = CString::new("NSPasteboard").unwrap();
|
|
83
|
+
let class = unsafe { objc_getClass(class_name.as_ptr()) };
|
|
84
|
+
anyhow::ensure!(!class.is_null(), "AppKit NSPasteboard is unavailable");
|
|
85
|
+
let board = send0(class, "generalPasteboard");
|
|
86
|
+
anyhow::ensure!(
|
|
87
|
+
!board.is_null(),
|
|
88
|
+
"macOS did not provide the general pasteboard"
|
|
89
|
+
);
|
|
90
|
+
Ok(board)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
fn string_type() -> Result<Obj> {
|
|
94
|
+
let class_name = CString::new("NSString").unwrap();
|
|
95
|
+
let class = unsafe { objc_getClass(class_name.as_ptr()) };
|
|
96
|
+
let allocated = send0(class, "alloc");
|
|
97
|
+
let utf8 = CString::new("public.utf8-plain-text").unwrap();
|
|
98
|
+
let function: unsafe extern "C" fn(Obj, Sel, *const c_char) -> Obj =
|
|
99
|
+
unsafe { std::mem::transmute(objc_msgSend as *const ()) };
|
|
100
|
+
let value = unsafe { function(allocated, selector("initWithUTF8String:"), utf8.as_ptr()) };
|
|
101
|
+
anyhow::ensure!(!value.is_null(), "could not create pasteboard type");
|
|
102
|
+
Ok(value)
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
fn nsstring(text: &str) -> Result<Obj> {
|
|
106
|
+
let class_name = CString::new("NSString").unwrap();
|
|
107
|
+
let class = unsafe { objc_getClass(class_name.as_ptr()) };
|
|
108
|
+
let allocated = send0(class, "alloc");
|
|
109
|
+
let chars: Vec<u16> = text.encode_utf16().collect();
|
|
110
|
+
let function: unsafe extern "C" fn(Obj, Sel, *const u16, usize) -> Obj =
|
|
111
|
+
unsafe { std::mem::transmute(objc_msgSend as *const ()) };
|
|
112
|
+
let value = unsafe {
|
|
113
|
+
function(
|
|
114
|
+
allocated,
|
|
115
|
+
selector("initWithCharacters:length:"),
|
|
116
|
+
chars.as_ptr(),
|
|
117
|
+
chars.len(),
|
|
118
|
+
)
|
|
119
|
+
};
|
|
120
|
+
anyhow::ensure!(!value.is_null(), "could not create clipboard text");
|
|
121
|
+
Ok(value)
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
pub fn write(text: &str) -> Result<()> {
|
|
125
|
+
let _pool = Pool::new()?;
|
|
126
|
+
let board = pasteboard()?;
|
|
127
|
+
let kind = string_type()?;
|
|
128
|
+
anyhow::ensure!(
|
|
129
|
+
text.len() <= MAX_CLIPBOARD_BYTES,
|
|
130
|
+
"refusing to place {} bytes on the clipboard",
|
|
131
|
+
text.len()
|
|
132
|
+
);
|
|
133
|
+
let value = nsstring(text)?;
|
|
134
|
+
let _ = send0_integer(board, "clearContents");
|
|
135
|
+
let written = send2_object(board, "setString:forType:", value, kind);
|
|
136
|
+
unsafe {
|
|
137
|
+
CFRelease(value);
|
|
138
|
+
CFRelease(kind);
|
|
139
|
+
}
|
|
140
|
+
if !written {
|
|
141
|
+
bail!("NSPasteboard refused the text clipboard value");
|
|
142
|
+
}
|
|
143
|
+
Ok(())
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
pub fn read_or_explain() -> Result<(String, bool), String> {
|
|
147
|
+
let _pool = Pool::new().map_err(|error| format!("{error:#}"))?;
|
|
148
|
+
let board = pasteboard().map_err(|error| format!("{error:#}"))?;
|
|
149
|
+
let kind = string_type().map_err(|error| format!("{error:#}"))?;
|
|
150
|
+
let value = send1_object(board, "stringForType:", kind);
|
|
151
|
+
unsafe {
|
|
152
|
+
CFRelease(kind);
|
|
153
|
+
}
|
|
154
|
+
if value.is_null() {
|
|
155
|
+
return Ok((String::new(), false));
|
|
156
|
+
}
|
|
157
|
+
let length_fn: unsafe extern "C" fn(Obj, Sel) -> usize =
|
|
158
|
+
unsafe { std::mem::transmute(objc_msgSend as *const ()) };
|
|
159
|
+
let length = unsafe { length_fn(value, selector("length")) };
|
|
160
|
+
let mut truncated = length > MAX_CLIPBOARD_BYTES;
|
|
161
|
+
let text_value = if truncated {
|
|
162
|
+
let substring_fn: unsafe extern "C" fn(Obj, Sel, usize) -> Obj =
|
|
163
|
+
unsafe { std::mem::transmute(objc_msgSend as *const ()) };
|
|
164
|
+
unsafe { substring_fn(value, selector("substringToIndex:"), MAX_CLIPBOARD_BYTES) }
|
|
165
|
+
} else {
|
|
166
|
+
value
|
|
167
|
+
};
|
|
168
|
+
let utf8 = send0(text_value, "UTF8String").cast::<c_char>();
|
|
169
|
+
let text = if utf8.is_null() {
|
|
170
|
+
String::new()
|
|
171
|
+
} else {
|
|
172
|
+
unsafe { CStr::from_ptr(utf8) }
|
|
173
|
+
.to_string_lossy()
|
|
174
|
+
.into_owned()
|
|
175
|
+
};
|
|
176
|
+
let mut end = text.len().min(MAX_CLIPBOARD_BYTES);
|
|
177
|
+
truncated |= end < text.len();
|
|
178
|
+
while !text.is_char_boundary(end) {
|
|
179
|
+
end -= 1;
|
|
180
|
+
}
|
|
181
|
+
Ok((text[..end].to_owned(), truncated))
|
|
182
|
+
}
|
package/engine/src/convert.rs
CHANGED
|
@@ -5,7 +5,17 @@
|
|
|
5
5
|
//! downscale is the correct filter when the encoded surface is smaller than the
|
|
6
6
|
//! source — it is the only one that does not alias the desktop's text.
|
|
7
7
|
|
|
8
|
-
|
|
8
|
+
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
|
|
9
|
+
#[serde(rename_all = "snake_case")]
|
|
10
|
+
pub enum PixelFormat {
|
|
11
|
+
Bgrx,
|
|
12
|
+
Rgbx,
|
|
13
|
+
Bgra,
|
|
14
|
+
Rgba,
|
|
15
|
+
Bgr,
|
|
16
|
+
Rgb,
|
|
17
|
+
Unsupported,
|
|
18
|
+
}
|
|
9
19
|
|
|
10
20
|
/// A tightly packed I420 frame; the plane strides are the frame width/2 so the
|
|
11
21
|
/// buffer is exactly what `vpx_image_t` wants when we hand libvpx three planes.
|
|
@@ -84,6 +94,39 @@ fn layout(format: PixelFormat) -> Option<(usize, [usize; 3])> {
|
|
|
84
94
|
}
|
|
85
95
|
}
|
|
86
96
|
|
|
97
|
+
/// A tightly packed pre-encode BGRX frame for still observations. At native
|
|
98
|
+
/// resolution this copies each source colour exactly; a smaller encode box
|
|
99
|
+
/// necessarily samples down to the geometry input uses.
|
|
100
|
+
pub fn to_bgrx(
|
|
101
|
+
src: &[u8],
|
|
102
|
+
src_w: usize,
|
|
103
|
+
src_h: usize,
|
|
104
|
+
stride: usize,
|
|
105
|
+
format: PixelFormat,
|
|
106
|
+
width: usize,
|
|
107
|
+
height: usize,
|
|
108
|
+
) -> Option<Vec<u8>> {
|
|
109
|
+
let (bytes, rgb) = layout(format)?;
|
|
110
|
+
let mut out = Vec::with_capacity(width * height * 4);
|
|
111
|
+
if matches!(format, PixelFormat::Bgrx | PixelFormat::Bgra) && width == src_w && height == src_h
|
|
112
|
+
{
|
|
113
|
+
for row in 0..height {
|
|
114
|
+
out.extend_from_slice(src.get(row * stride..row * stride + width * 4)?);
|
|
115
|
+
}
|
|
116
|
+
return Some(out);
|
|
117
|
+
}
|
|
118
|
+
for y in 0..height {
|
|
119
|
+
for x in 0..width {
|
|
120
|
+
let sx = x * src_w / width;
|
|
121
|
+
let sy = y * src_h / height;
|
|
122
|
+
let at = sy * stride + sx * bytes;
|
|
123
|
+
let pixel = src.get(at..at + bytes)?;
|
|
124
|
+
out.extend_from_slice(&[pixel[rgb[2]], pixel[rgb[1]], pixel[rgb[0]], 255]);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
Some(out)
|
|
128
|
+
}
|
|
129
|
+
|
|
87
130
|
/// How many threads convert one frame. A 4K desktop is 8 million pixels, which
|
|
88
131
|
/// one core converts in tens of milliseconds, the whole budget of a frame; a few
|
|
89
132
|
/// bands bring that under the encoder's own cost without taking the machine.
|
|
@@ -292,6 +335,19 @@ mod tests {
|
|
|
292
335
|
assert_eq!((out.width, out.height), (1702, 1064));
|
|
293
336
|
}
|
|
294
337
|
|
|
338
|
+
#[test]
|
|
339
|
+
fn still_pixels_keep_exact_source_colours_before_encoding() {
|
|
340
|
+
let src = [1, 2, 3, 0, 4, 5, 6, 0];
|
|
341
|
+
assert_eq!(
|
|
342
|
+
to_bgrx(&src, 2, 1, 8, PixelFormat::Bgrx, 2, 1).unwrap(),
|
|
343
|
+
src
|
|
344
|
+
);
|
|
345
|
+
assert_eq!(
|
|
346
|
+
to_bgrx(&src, 2, 1, 8, PixelFormat::Rgbx, 2, 1).unwrap(),
|
|
347
|
+
[3, 2, 1, 255, 6, 5, 4, 255]
|
|
348
|
+
);
|
|
349
|
+
}
|
|
350
|
+
|
|
295
351
|
#[test]
|
|
296
352
|
fn unsupported_format_is_reported_not_guessed() {
|
|
297
353
|
assert!(to_i420(&vec![0u8; 64], 4, 4, 16, PixelFormat::Unsupported, 2, 2).is_none());
|