@desklink/host 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +3 -0
- package/README.md +92 -23
- package/bin/desklink-host.mjs +14 -5
- package/dist/bridge.d.ts +3 -4
- package/dist/bridge.d.ts.map +1 -1
- package/dist/bridge.js +18 -7
- package/dist/bridge.js.map +1 -1
- package/dist/engineProcess.d.ts +6 -1
- package/dist/engineProcess.d.ts.map +1 -1
- package/dist/engineProcess.js +34 -5
- package/dist/engineProcess.js.map +1 -1
- package/dist/protocol.d.ts +35 -2
- package/dist/protocol.d.ts.map +1 -1
- package/dist/protocol.js.map +1 -1
- package/dist/resolveEngine.d.ts +6 -0
- package/dist/resolveEngine.d.ts.map +1 -1
- package/dist/resolveEngine.js +45 -6
- package/dist/resolveEngine.js.map +1 -1
- package/docs/BOUNDARY.md +8 -8
- package/docs/PROTOCOL.md +185 -45
- package/engine/Cargo.lock +2 -2
- package/engine/Cargo.toml +7 -12
- package/engine/build.rs +62 -5
- package/engine/native/agent_indicator.h +57 -0
- package/engine/native/agent_overlay_mac.m +51 -4
- package/engine/native/agent_overlay_wayland.c +189 -35
- package/engine/native/agent_overlay_x11.c +186 -55
- package/engine/src/capture.rs +96 -5
- package/engine/src/capture_win.rs +23 -0
- package/engine/src/clipboard.rs +174 -0
- package/engine/src/clipboard_win.rs +8 -0
- package/engine/src/cursor.rs +118 -0
- package/engine/src/indicator.rs +271 -5
- package/engine/src/input_win.rs +91 -0
- package/engine/src/keymap_win.rs +39 -0
- package/engine/src/mac.rs +28 -6
- package/engine/src/main.rs +167 -25
- package/engine/src/peer.rs +10 -4
- package/engine/src/portal.rs +42 -4
- package/engine/src/portal_win.rs +37 -0
- package/engine/src/protocol.rs +56 -10
- package/engine/src/session.rs +749 -203
- package/engine/src/win.rs +110 -0
- package/engine/src/x11.rs +44 -29
- package/engine/src/x11_mac.rs +4 -0
- package/examples/reference-client.html +3 -5
- package/package.json +4 -4
package/docs/PROTOCOL.md
CHANGED
|
@@ -5,10 +5,14 @@ the user's session starts it and talks to it over an inherited private channel
|
|
|
5
5
|
the engine never opens a public listener, never runs as root, and never installs
|
|
6
6
|
a service.
|
|
7
7
|
|
|
8
|
-
This document describes the
|
|
9
|
-
engine uses the same v3 stdio/WebRTC session path, with native display
|
|
10
|
-
input and clipboard adapters.
|
|
11
|
-
|
|
8
|
+
This document describes the Linux, macOS, and Windows engine protocol. The
|
|
9
|
+
macOS engine uses the same v3 stdio/WebRTC session path, with native display
|
|
10
|
+
capture, input and clipboard adapters. A Windows build with static libvpx
|
|
11
|
+
(`DESKLINK_VPX_STATIC_DIR`) also serves consumer-fed H.264 encoded sources over
|
|
12
|
+
the shared session path. Native desktop capture, input and clipboard are not
|
|
13
|
+
implemented on Windows; optional agent feedback is unavailable there. Without static libvpx, the Windows compile seam
|
|
14
|
+
reports encoding unavailable and refuses `serve`. The protocol contains no
|
|
15
|
+
application concepts: no accounts, no chat, no machine ids, no pane ids.
|
|
12
16
|
|
|
13
17
|
## Starting the engine
|
|
14
18
|
|
|
@@ -77,7 +81,7 @@ arrives, every other request is refused the same way; `capabilities` and
|
|
|
77
81
|
## macOS
|
|
78
82
|
|
|
79
83
|
For engine installation and the macOS resolution opt-out, see the
|
|
80
|
-
[host installation guide](../README.md#macos).
|
|
84
|
+
[host installation guide](../README.md#macos-preview).
|
|
81
85
|
|
|
82
86
|
macOS speaks protocol 3 and shares the session lifecycle below. It captures the
|
|
83
87
|
selected display with ScreenCaptureKit, encodes VP9 (and H.264 through
|
|
@@ -91,16 +95,16 @@ dimensions, origin and scale), the current keyboard layout, console state, and
|
|
|
91
95
|
Screen Recording/Accessibility grant states. A `display` source accepts an
|
|
92
96
|
optional `display_id` from `capture.displays`; an absent ID selects the main
|
|
93
97
|
display. Screen Recording consent is requested by `session.open` when capture
|
|
94
|
-
is needed.
|
|
95
|
-
|
|
96
|
-
launching app (for example Terminal, iTerm, or the Node.js host), not to a
|
|
98
|
+
is needed. Input may request Accessibility consent; if it remains unavailable,
|
|
99
|
+
the session still opens and input actions are refused with `input-unavailable`.
|
|
100
|
+
TCC permissions attach to the responsible launching app (for example Terminal, iTerm, or the Node.js host), not to a
|
|
97
101
|
separate DesklinkHost.app bundle. Enable that app in **System Settings → Privacy
|
|
98
102
|
& Security → Screen & System Audio Recording** and **Accessibility**, then
|
|
99
103
|
reconnect. No TCC state is pre-seeded or bypassed.
|
|
100
104
|
|
|
101
|
-
macOS clipboard read/write is plain text and
|
|
102
|
-
|
|
103
|
-
|
|
105
|
+
macOS clipboard read/write is plain text and available to paired consumers.
|
|
106
|
+
`restore_token` is not supported by this native backend; a new session may
|
|
107
|
+
require fresh consent. The macOS engine requires a VP9-enabled build
|
|
104
108
|
(`DESKLINK_VPX_STATIC_DIR`) for `serve`.
|
|
105
109
|
|
|
106
110
|
## Capabilities
|
|
@@ -121,6 +125,8 @@ session may require fresh consent. The macOS engine requires a VP9-enabled build
|
|
|
121
125
|
"backends": ["portal-screencast+pipewire", "x11-root"],
|
|
122
126
|
"formats": ["bgrx", "bgra", "rgbx", "rgba"],
|
|
123
127
|
"cursor": "embedded",
|
|
128
|
+
"cursor_modes": ["embedded", "hidden"],
|
|
129
|
+
"cursor_positions": {"x11": true, "portal": "negotiated"},
|
|
124
130
|
"audio": false
|
|
125
131
|
},
|
|
126
132
|
"encode": {"codecs": ["vp9", "h264"], "hardware": false,
|
|
@@ -144,7 +150,11 @@ appears in `codecs` when an H.264 encoder starts on this machine right now, and
|
|
|
144
150
|
`encode.h264.encoder` names it: `videotoolbox` on macOS; on Linux `nvenc` or
|
|
145
151
|
`vaapi` when the GPU has an H.264 encoder, else `openh264`, Cisco's prebuilt
|
|
146
152
|
library. `encoder` is `null` when there is none. The answer to the offer picks
|
|
147
|
-
between them (see *Codecs* below).
|
|
153
|
+
between them (see *Codecs* below). Windows with static libvpx reports VP9 in
|
|
154
|
+
`encode.codecs`, but has no native desktop capture; without static libvpx its
|
|
155
|
+
`encode.codecs` is empty. On Windows, `encoded.codecs` reports H.264 only when
|
|
156
|
+
static libvpx is linked, for consumer-fed sources; it does not mean the engine
|
|
157
|
+
can encode H.264 itself.
|
|
148
158
|
|
|
149
159
|
openh264 is never part of this package: the engine loads only Cisco's own
|
|
150
160
|
binary, checked against its published SHA-256, from
|
|
@@ -175,12 +185,17 @@ desktop is visible here rather than silent. `capture.backends` lists what this
|
|
|
175
185
|
build has, not what this machine can necessarily use.
|
|
176
186
|
|
|
177
187
|
`input.unavailable_reason` and `input.grant` describe the portal/uinput path:
|
|
178
|
-
without kernel input access a portal session
|
|
179
|
-
|
|
180
|
-
not require that grant. The
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
188
|
+
without kernel input access a portal session opens and captures, but input
|
|
189
|
+
messages are refused with `input-unavailable`. X display capture uses XTest
|
|
190
|
+
instead and does not require that grant. The engine probes XTest on that named
|
|
191
|
+
display; if it is unavailable, capture and clipboard still open, and input
|
|
192
|
+
messages are refused with `input-unavailable`. X11 clipboard uses CLIPBOARD on the
|
|
193
|
+
session's named display; the WebSocket bridge reports it independently of the
|
|
194
|
+
engine's ambient Wayland clipboard probe. The engine's Linux `clipboard.read`
|
|
195
|
+
reports the Wayland session environment; `clipboard.write` additionally requires
|
|
196
|
+
`wl-copy`. A portal bridge preserves those flags even when Xwayland is reachable.
|
|
197
|
+
On Linux the engine never asks for
|
|
198
|
+
privileges on its own; macOS may request Screen Recording or Accessibility consent at `session.open`.
|
|
184
199
|
|
|
185
200
|
`capabilities` is safe to call before any consent has been given and must not
|
|
186
201
|
trigger a capture request.
|
|
@@ -192,7 +207,6 @@ trigger a capture request.
|
|
|
192
207
|
"source": {"kind":"portal"}, // or {"kind":"x11","display":":99"}
|
|
193
208
|
// macOS: {"kind":"display","display_id":123}
|
|
194
209
|
// or {"kind":"encoded","codec":"h264","width":486,"height":1080}
|
|
195
|
-
"permissions": ["view","control","clipboard"],
|
|
196
210
|
"max_width": 3840, "max_height": 2160, // encode box (default); never upscales
|
|
197
211
|
"bitrate_kbps": 0, // 0 (default): sized to the encoded surface
|
|
198
212
|
"max_fps": 60,
|
|
@@ -212,7 +226,7 @@ Result:
|
|
|
212
226
|
"source":{"kind":"monitor","width":2560,"height":1440,"origin":{"x":0,"y":0}},
|
|
213
227
|
"geometry":{"source":{"width":2560,"height":1440},
|
|
214
228
|
"encoded":{"width":2560,"height":1440},
|
|
215
|
-
"origin":{"x":0,"y":0}}}
|
|
229
|
+
"origin":{"x":0,"y":0}},"agentIndicator":false}
|
|
216
230
|
```
|
|
217
231
|
|
|
218
232
|
`source` selects the desktop. `portal` (or absent) asks the compositor for a
|
|
@@ -223,11 +237,12 @@ main display when omitted); `encoded` carries video the consumer encoded itself.
|
|
|
223
237
|
The choice is normally the *consumer's* decision, because a client is not in a
|
|
224
238
|
position to know which backend a machine can offer.
|
|
225
239
|
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
`
|
|
240
|
+
There are no per-session permissions: whoever holds the channel is paired and
|
|
241
|
+
gets all available view, input and clipboard access. The `permissions` field
|
|
242
|
+
from 0.3 clients is accepted and ignored, including unknown scope names.
|
|
243
|
+
Missing input or clipboard backends fail per action, not at open. The
|
|
244
|
+
`permission` refusal code is reserved for 0.3 peers and is no longer emitted by
|
|
245
|
+
this engine.
|
|
231
246
|
|
|
232
247
|
`max_fps` defaults to 60 and is bounded to 1–60. It caps the encoded frame
|
|
233
248
|
rate and X11 capture loop; portal capture offers it as the preferred PipeWire
|
|
@@ -235,14 +250,29 @@ frame rate, and macOS sets its ScreenCaptureKit stream minimum frame interval.
|
|
|
235
250
|
A source may still supply fewer frames.
|
|
236
251
|
|
|
237
252
|
`agent_indicator` is opt-in local feedback for an agent controller; desklink-axi
|
|
238
|
-
sets it for
|
|
239
|
-
adds an eased
|
|
240
|
-
|
|
253
|
+
sets it for sessions it drives (except Windows); human phone control omits it. On
|
|
254
|
+
Linux it adds an eased arrow inside a halo that brightens on clicks and keys,
|
|
255
|
+
animating in and out, with light and dark keylines so it reads on any content.
|
|
256
|
+
On Wayland it uses a click-through layer-shell surface with an edge glow, drawn
|
|
257
|
+
at the highest announced output integer scale, capped at 4; its bounded area is
|
|
241
258
|
masked before capture/diff (pixels behind the effect can remain briefly stale).
|
|
242
|
-
On bare X11 it
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
259
|
+
On bare X11 it is an override-redirect, click-through window that follows the
|
|
260
|
+
pointer, so it stays
|
|
261
|
+
visible over windows that set their own cursor and never replaces the cursor.
|
|
262
|
+
Its size follows `Xft.dpi` from 96–384 DPI, defaulting to 1x otherwise;
|
|
263
|
+
the shaped cue reveals outward from the tip and retracts inward, while a
|
|
264
|
+
compositing manager with alpha enables fading instead. The helper announces each
|
|
265
|
+
state before drawing it, and X11 capture restores just
|
|
266
|
+
the pixels the cue covers from the previous frame, so the halo's interior stays
|
|
267
|
+
live; changes under the arrow and ring show once the cue moves away. Click and
|
|
268
|
+
key feedback only recolour the cue, so a still cue never changes what is
|
|
269
|
+
restored. There is no edge glow on that path.
|
|
270
|
+
macOS adds an eased arrow and halo, click ripple, and typing pulse in a
|
|
271
|
+
nonactivating, click-through window excluded by the ScreenCaptureKit content
|
|
272
|
+
filter. Closing the session animates out and removes the indicator.
|
|
273
|
+
Indicator startup is best-effort on every platform; a missing helper or overlay
|
|
274
|
+
backend does not refuse the session. The open result reports `agentIndicator:
|
|
275
|
+
true` only when the helper started.
|
|
246
276
|
|
|
247
277
|
`local_frames` keeps the latest pre-encode frame and its tile hashes for
|
|
248
278
|
`session.frame` and `session.frame.changed` (see "Latest frame over the local
|
|
@@ -457,11 +487,113 @@ named beside it for the per-frame mean:
|
|
|
457
487
|
frame within measurement noise; a stage that will not reconcile is a stage
|
|
458
488
|
whose accounting is wrong.
|
|
459
489
|
|
|
490
|
+
## Cursor-free capture
|
|
491
|
+
|
|
492
|
+
`session.open` accepts `cursor: "hidden"` (default `"embedded"`). Existing
|
|
493
|
+
callers keep their capture behavior. Linux hidden capture requires portal
|
|
494
|
+
**Metadata** so cursor positions remain available while cursor pixels are
|
|
495
|
+
omitted. A portal without Metadata returns the typed
|
|
496
|
+
`cursor_positions_unavailable` refusal. Successful open returns
|
|
497
|
+
`cursor: {"mode":"metadata"|"hidden"|"embedded"|"unavailable",
|
|
498
|
+
"positions":true|false}`; `mode` names the portal mode accepted by the portal,
|
|
499
|
+
not just the requested option.
|
|
500
|
+
|
|
501
|
+
Linux capabilities advertise `capture.cursor_modes: ["embedded","hidden"]`
|
|
502
|
+
and `capture.cursor_positions: {"x11":true,"portal":"negotiated"}`. Portal
|
|
503
|
+
support is resolved at open: hidden capture requires Metadata and an observed
|
|
504
|
+
SPA cursor metadata buffer. A portal that offers only Hidden refuses open with
|
|
505
|
+
`cursor_positions_unavailable`; accepted hidden sessions report `positions:true`.
|
|
506
|
+
Metadata-mode buffers that contain no new cursor data do not fabricate a move
|
|
507
|
+
or a visibility change.
|
|
508
|
+
|
|
509
|
+
Hidden sessions with positions emit local protocol events independently of
|
|
510
|
+
pixel damage, including on a still X11 desktop:
|
|
511
|
+
|
|
512
|
+
```json
|
|
513
|
+
{"event":"session.cursor","params":{"sessionId":"…","x":400,"y":300,"visible":true,"timestamp_us":123456}}
|
|
514
|
+
```
|
|
515
|
+
|
|
516
|
+
`x,y` name the pointer hotspot in the selected source's coordinates
|
|
517
|
+
(`geometry.source`), before video scaling, without the desktop origin added.
|
|
518
|
+
Portal buffer coordinates are mapped to that geometry. `visible` is false
|
|
519
|
+
outside the source or when SPA supplies an invisible cursor bitmap.
|
|
520
|
+
`timestamp_us` is a monotonic engine-process clock in microseconds, sampled
|
|
521
|
+
when the capture thread observes the pointer. The same clock stamps
|
|
522
|
+
`session.frame.changed` and `session.frame` metadata, so local recording
|
|
523
|
+
consumers can compare cursor samples to frame handoff times. These are not
|
|
524
|
+
wall-clock or RTP timestamps. SPA cursor updates accompany PipeWire buffers;
|
|
525
|
+
X11 pointer queries run at the session frame cap even with no pixel damage.
|
|
526
|
+
Repeated identical positions/visibility are suppressed.
|
|
527
|
+
|
|
528
|
+
X11 root GetImage/MIT-SHM never contains the server cursor and this engine
|
|
529
|
+
never composites it, including for existing embedded-default callers. Its
|
|
530
|
+
open reply therefore reports `mode:"hidden"`; positions are enabled only by
|
|
531
|
+
an explicit `cursor:"hidden"`. macOS and Windows refuse hidden capture with
|
|
532
|
+
`cursor-unavailable` in this build. Consumer-fed encoded sources refuse it too:
|
|
533
|
+
the engine cannot remove pixels the consumer already encoded. Unknown cursor
|
|
534
|
+
options are `malformed`.
|
|
535
|
+
|
|
536
|
+
`@desklink/host` exposes `openSession({cursor:'hidden'})`
|
|
537
|
+
and typed `session.cursor` events through `onEvent`/`drainEvents`. The client
|
|
538
|
+
capture request also accepts `cursor`, and AXI accepts `start --cursor hidden`.
|
|
539
|
+
|
|
540
|
+
## Point on the host desktop
|
|
541
|
+
|
|
542
|
+
A local consumer can guide the person without taking control of their pointer
|
|
543
|
+
or keyboard. `session.point` requires an open session and is independent
|
|
544
|
+
of `agent_indicator`. It draws one blue ring centered at `x,y`, with an optional
|
|
545
|
+
label. A new point replaces the previous one and restarts its timer.
|
|
546
|
+
|
|
547
|
+
```jsonc
|
|
548
|
+
{"id":30,"method":"session.point","params":{"session_id":"…","x":400,"y":300,"label":"Here","timeout_ms":3000}}
|
|
549
|
+
{"id":30,"result":{"shown":true}}
|
|
550
|
+
{"id":31,"method":"session.point","params":{"session_id":"…","clear":true}}
|
|
551
|
+
{"id":31,"result":{"shown":false}}
|
|
552
|
+
```
|
|
553
|
+
|
|
554
|
+
Coordinates are integers in the **selected desktop's original pixels**, top-left
|
|
555
|
+
origin, before video scaling (`geometry.source`, not `geometry.encoded`). On X11
|
|
556
|
+
this is the root screen; on macOS it is the selected display, with native pixels
|
|
557
|
+
converted to display points. Both coordinates must be within its bounds. The
|
|
558
|
+
ring may clip at an edge; its label stays inside the display. `timeout_ms`
|
|
559
|
+
defaults to 3000 and must be an integer from 1 to 120000. Labels are at most 96
|
|
560
|
+
printable ASCII bytes, including spaces; other labels are refused rather than
|
|
561
|
+
rendered incorrectly by the X server's core font. A clear request has no
|
|
562
|
+
coordinates, label or timeout, and is idempotent. Invalid requests do not alter
|
|
563
|
+
an existing marker. Replies acknowledge submission to the local overlay helper,
|
|
564
|
+
not that a person has seen it. Timeout hides it; clear, session replacement,
|
|
565
|
+
lease expiry, close and consumer disconnection remove and reap the helper.
|
|
566
|
+
|
|
567
|
+
X11 uses an override-redirect, shaped window with an empty input shape. It never
|
|
568
|
+
warps the pointer, changes the cursor, requests focus or injects events. macOS
|
|
569
|
+
reuses the nonactivating, mouse-transparent indicator window in a separate
|
|
570
|
+
helper. **Point cues intentionally appear in local frames and outgoing video**:
|
|
571
|
+
the person and observer can see the same cue. They are separate from the
|
|
572
|
+
capture-excluded input activity indicator. Consumers reading frame differences
|
|
573
|
+
should allow for the point's appearance/disappearance.
|
|
574
|
+
|
|
575
|
+
Wayland portal sessions use a wlr-layer-shell overlay with no keyboard
|
|
576
|
+
interactivity and an empty input region. The selected monitor is matched by its
|
|
577
|
+
portal origin against compositor logical output geometry (xdg-output); with one
|
|
578
|
+
output, no origin is needed. Points map from the selected source to that output.
|
|
579
|
+
An ambiguous or missing selected output returns `indicator-unavailable`.
|
|
580
|
+
Compositors without layer-shell return `point_unsupported` with reason
|
|
581
|
+
`layer_shell_unavailable` (including GNOME Mutter). Consumer-fed encoded sources return `operation`
|
|
582
|
+
because they have no host desktop overlay target. Other refusals are `session`,
|
|
583
|
+
`coordinates`, `malformed`, and `indicator-unavailable` (including unavailable
|
|
584
|
+
X Shape support). Unknown fields are refused. No control-channel or remote
|
|
585
|
+
bridge forwarding is added; the consumer chooses when to request a local cue.
|
|
586
|
+
|
|
587
|
+
`@desklink/host` exposes `EngineClient.point(sessionId, {x, y, label?, timeoutMs?})`
|
|
588
|
+
and `EngineClient.point(sessionId, {clear: true})`. `@desklink/axi` exposes
|
|
589
|
+
`desklink-axi point 400,300 --label "Here" --timeout 3000` and
|
|
590
|
+
`desklink-axi point --clear`.
|
|
591
|
+
|
|
460
592
|
## Input and clipboard: the session's control channel
|
|
461
593
|
|
|
462
594
|
Input does **not** travel the local protocol. It rides the WebRTC data channel
|
|
463
595
|
the engine creates for the session, which is inside the DTLS/SRTP session that
|
|
464
|
-
was opened
|
|
596
|
+
was opened by a paired consumer: it inherits that session's identity and dies
|
|
465
597
|
with it, so a client that kept a socket open cannot keep driving a session that
|
|
466
598
|
was revoked, and the consumer's request path never carries a pointer move.
|
|
467
599
|
|
|
@@ -498,12 +630,12 @@ The engine answers on the same channel, and sends two things unprompted:
|
|
|
498
630
|
{"kind":"revoked","reason":"…","code":"closed"} // closed, or a session.revoked code
|
|
499
631
|
```
|
|
500
632
|
|
|
501
|
-
Refusal codes: `
|
|
502
|
-
`text-
|
|
633
|
+
Refusal codes: `session`, `input-replay`, `coordinates`, `text-too-large`,
|
|
634
|
+
`text-unsupported`, `input-unavailable`, `clipboard-unsupported`, `operation`.
|
|
503
635
|
|
|
504
636
|
`seq` is monotonic per session. A repeat or a lower value is refused with
|
|
505
637
|
`input-replay` and **no side effect**, so a replayed message cannot move the
|
|
506
|
-
pointer twice. Validation
|
|
638
|
+
pointer twice. Validation and ordering checks all run before any
|
|
507
639
|
physical effect.
|
|
508
640
|
|
|
509
641
|
The engine holds its own pressed-key/button state. `close`, `release_all`,
|
|
@@ -534,8 +666,7 @@ WebRTC, congestion and teardown logic that is already proven on this path.
|
|
|
534
666
|
it is the only party that knows them (`1..7680` by `1..4320`). Nothing is scaled:
|
|
535
667
|
`geometry.source` and `geometry.encoded` are both that size, the client aims at
|
|
536
668
|
those pixels, and the consumer maps them to its device. `max_width`/
|
|
537
|
-
`max_height`/`bitrate_kbps` are ignored for this source
|
|
538
|
-
as it does anywhere else — with one difference: `control` needs no local input
|
|
669
|
+
`max_height`/`bitrate_kbps` are ignored for this source. Input needs no local
|
|
539
670
|
backend, because nothing is applied locally.
|
|
540
671
|
|
|
541
672
|
`session.open` returns before any offer exists. The offer must name the profile
|
|
@@ -606,7 +737,7 @@ pixels, exactly as a client sends them.
|
|
|
606
737
|
```
|
|
607
738
|
|
|
608
739
|
The event carries the control message as it arrived. The engine still validates
|
|
609
|
-
|
|
740
|
+
ordering and replay before forwarding, so the client keeps the same
|
|
610
741
|
`ack`/`rejected` behaviour and a replayed gesture still cannot be delivered
|
|
611
742
|
twice. `ack` means "accepted for this session", not "applied to a desktop" —
|
|
612
743
|
`session.metrics.input_forwarded` counts what was handed on, and `input_applied`
|
|
@@ -625,7 +756,7 @@ because packetization is the only stage the engine itself runs.
|
|
|
625
756
|
|
|
626
757
|
## Latest frame over the local protocol
|
|
627
758
|
|
|
628
|
-
`session.frame`
|
|
759
|
+
`session.frame` writes the latest pre-encode frame as
|
|
629
760
|
packed BGRX bytes to the caller's path, with optional `[x,y,w,h]` `region`.
|
|
630
761
|
The directory must already exist and be private to the caller; pixels never
|
|
631
762
|
travel through JSON. `seq` advances only when capture changes, `still_ms` is
|
|
@@ -662,9 +793,17 @@ instead; both reach the same clipboard:
|
|
|
662
793
|
{"id":10,"result":{"written":true}}
|
|
663
794
|
```
|
|
664
795
|
|
|
665
|
-
Refused with `error.code = "clipboard"` when the
|
|
666
|
-
|
|
667
|
-
|
|
796
|
+
Refused with `error.code = "clipboard"` when the desktop has no text or a
|
|
797
|
+
clipboard operation fails. Encoded sources have no desktop clipboard; their
|
|
798
|
+
control channel answers with an empty clipboard reply and an explanatory error.
|
|
799
|
+
X11 uses a fresh connection to the session's selected display for each operation.
|
|
800
|
+
Writes verify CLIPBOARD ownership before succeeding and serve TARGETS,
|
|
801
|
+
UTF8_STRING and TEXT; reads request UTF8_STRING. Writes are limited to 256 KiB
|
|
802
|
+
and the server's single request limit; the text lives until another client takes ownership, the engine
|
|
803
|
+
exits, or the display dies. Reads wait at most 2.5 seconds for the owner.
|
|
804
|
+
INCR transfers are refused explicitly as unsupported, not reported as success;
|
|
805
|
+
STRING, PRIMARY and persistence after engine exit are not implemented.
|
|
806
|
+
A read is bounded by `capabilities.clipboard.maxBytes`; when the desktop's text is longer, `truncated`
|
|
668
807
|
is true and the text is cut on a character boundary, so the returned text is
|
|
669
808
|
still valid.
|
|
670
809
|
|
|
@@ -740,8 +879,9 @@ anything to show.
|
|
|
740
879
|
|
|
741
880
|
## What the engine deliberately does not do
|
|
742
881
|
|
|
743
|
-
- It does not decide *who* the user is.
|
|
744
|
-
|
|
882
|
+
- It does not decide *who* the user is. Whoever holds stdio (or an authenticated
|
|
883
|
+
bridge socket) is paired; pairing, identity and revocation are the consumer's.
|
|
884
|
+
Revocation must end both live sessions and future access. There is no second
|
|
745
885
|
account, token or pairing ceremony. For consumer integration, see the
|
|
746
886
|
[BYOKit guidance](../../../README.md#why-desklink-exists).
|
|
747
887
|
- It does not open ports, register a service, install udev rules, join groups or
|
package/engine/Cargo.lock
CHANGED
|
@@ -536,7 +536,7 @@ checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
|
|
|
536
536
|
|
|
537
537
|
[[package]]
|
|
538
538
|
name = "desklink-host"
|
|
539
|
-
version = "0.
|
|
539
|
+
version = "0.4.0"
|
|
540
540
|
dependencies = [
|
|
541
541
|
"anyhow",
|
|
542
542
|
"ashpd",
|
|
@@ -546,7 +546,6 @@ dependencies = [
|
|
|
546
546
|
"bytes",
|
|
547
547
|
"bzip2-rs",
|
|
548
548
|
"cc",
|
|
549
|
-
"futures-util",
|
|
550
549
|
"libc",
|
|
551
550
|
"log",
|
|
552
551
|
"objc2-core-foundation",
|
|
@@ -565,6 +564,7 @@ dependencies = [
|
|
|
565
564
|
"wayland-client",
|
|
566
565
|
"wayland-protocols",
|
|
567
566
|
"webrtc",
|
|
567
|
+
"windows-sys 0.61.2",
|
|
568
568
|
"wl-clipboard-rs",
|
|
569
569
|
"x11rb",
|
|
570
570
|
"xkbcommon",
|
package/engine/Cargo.toml
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "desklink-host"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.4.0"
|
|
4
4
|
edition = "2021"
|
|
5
5
|
rust-version = "1.97.1"
|
|
6
6
|
license = "Apache-2.0"
|
|
@@ -16,10 +16,9 @@ serde = { version = "1", features = ["derive"] }
|
|
|
16
16
|
serde_json = "1"
|
|
17
17
|
base64 = "0.22"
|
|
18
18
|
|
|
19
|
-
[target.'cfg(target_os = "linux")'.dependencies]
|
|
19
|
+
[target.'cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))'.dependencies]
|
|
20
20
|
anyhow = "1"
|
|
21
21
|
tokio = { version = "1", features = ["rt-multi-thread", "macros", "io-util", "io-std", "sync", "time", "signal"] }
|
|
22
|
-
futures-util = "0.3"
|
|
23
22
|
libc = "0.2"
|
|
24
23
|
bytes = "1"
|
|
25
24
|
webrtc = { version = "0.21", default-features = false, features = ["runtime-tokio", "crypto-ring"] }
|
|
@@ -27,6 +26,8 @@ rtc = { version = "0.21", default-features = false, features = ["crypto-ring"] }
|
|
|
27
26
|
log = "0.4"
|
|
28
27
|
async-trait = "0.1"
|
|
29
28
|
rand = "0.8"
|
|
29
|
+
|
|
30
|
+
[target.'cfg(target_os = "linux")'.dependencies]
|
|
30
31
|
ashpd = { version = "0.11", default-features = false, features = ["tokio"] }
|
|
31
32
|
pipewire = { version = "0.10", features = ["v0_3_44"] }
|
|
32
33
|
xkbcommon = "0.8"
|
|
@@ -43,16 +44,10 @@ bzip2-rs = "0.1.2"
|
|
|
43
44
|
wayland-client = "0.31"
|
|
44
45
|
wayland-protocols = { version = "0.32", features = ["client"] }
|
|
45
46
|
|
|
47
|
+
[target.'cfg(target_os = "windows")'.dependencies]
|
|
48
|
+
windows-sys = "0.61.2"
|
|
49
|
+
|
|
46
50
|
[target.'cfg(target_os = "macos")'.dependencies]
|
|
47
|
-
anyhow = "1"
|
|
48
|
-
async-trait = "0.1"
|
|
49
|
-
bytes = "1"
|
|
50
|
-
libc = "0.2"
|
|
51
|
-
log = "0.4"
|
|
52
|
-
rand = "0.8"
|
|
53
|
-
rtc = { version = "0.21", default-features = false, features = ["crypto-ring"] }
|
|
54
|
-
tokio = { version = "1", features = ["rt-multi-thread", "macros", "io-util", "io-std", "sync", "time", "signal"] }
|
|
55
|
-
webrtc = { version = "0.21", default-features = false, features = ["runtime-tokio", "crypto-ring"] }
|
|
56
51
|
objc2-screen-capture-kit = { version = "0.3.2", features = ["SCStream", "SCShareableContent", "SCError", "SCScreenshotManager", "block2", "dispatch2", "objc2-core-media", "objc2-core-graphics"] }
|
|
57
52
|
objc2-foundation = { version = "0.3.2", features = ["NSArray", "NSError", "NSString"] }
|
|
58
53
|
objc2-core-media = "0.3.2"
|
package/engine/build.rs
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
//! Native dependencies of the engine.
|
|
2
2
|
//!
|
|
3
|
-
//! The Linux build
|
|
4
|
-
//!
|
|
5
|
-
//! nothing is installed:
|
|
3
|
+
//! The Linux build compiles its native adapters from local sources and links
|
|
4
|
+
//! system libraries for desktop overlays — no prebuilt binary is fetched at
|
|
5
|
+
//! build time, and nothing is installed:
|
|
6
6
|
//!
|
|
7
7
|
//! * libvpx (BSD-3-Clause) — VP9 encode, through `native/vpx_shim.c` so the
|
|
8
8
|
//! versioned encoder config struct is laid out by a C compiler.
|
|
@@ -38,6 +38,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|
|
38
38
|
println!("cargo:rerun-if-changed=native/mac_stream.mm");
|
|
39
39
|
println!("cargo:rerun-if-changed=native/agent_overlay_mac.m");
|
|
40
40
|
println!("cargo:rerun-if-changed=native/agent_overlay_x11.c");
|
|
41
|
+
println!("cargo:rerun-if-changed=native/agent_indicator.h");
|
|
41
42
|
println!("cargo:rerun-if-changed=native/agent_overlay_wayland.c");
|
|
42
43
|
println!("cargo:rerun-if-changed=Info.plist");
|
|
43
44
|
println!("cargo:rerun-if-changed=vendor/inputtino/src/uinput/include/inputtino/keyboard.hpp");
|
|
@@ -48,6 +49,44 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|
|
48
49
|
println!("cargo:rerun-if-env-changed=DESKLINK_MACOS_CLI");
|
|
49
50
|
|
|
50
51
|
let target_os = std::env::var("CARGO_CFG_TARGET_OS").unwrap_or_default();
|
|
52
|
+
if target_os == "windows" {
|
|
53
|
+
if let Some(prefix) = std::env::var_os("DESKLINK_VPX_STATIC_DIR").map(PathBuf::from) {
|
|
54
|
+
if std::env::var("CARGO_CFG_TARGET_ENV")? != "msvc"
|
|
55
|
+
|| std::env::var("CARGO_CFG_TARGET_ARCH")? != "x86_64"
|
|
56
|
+
{
|
|
57
|
+
return Err("Windows libvpx requires x86_64-pc-windows-msvc".into());
|
|
58
|
+
}
|
|
59
|
+
if !std::env::var("CARGO_CFG_TARGET_FEATURE")?
|
|
60
|
+
.split(',')
|
|
61
|
+
.any(|f| f == "crt-static")
|
|
62
|
+
{
|
|
63
|
+
return Err(
|
|
64
|
+
"Windows static libvpx requires RUSTFLAGS=-Ctarget-feature=+crt-static (/MT)"
|
|
65
|
+
.into(),
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
if !prefix.join("include/vpx/vp8cx.h").is_file()
|
|
69
|
+
|| !prefix.join("lib/vpx.lib").is_file()
|
|
70
|
+
{
|
|
71
|
+
return Err(
|
|
72
|
+
"DESKLINK_VPX_STATIC_DIR must contain include/vpx/vp8cx.h and lib/vpx.lib"
|
|
73
|
+
.into(),
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
cc::Build::new()
|
|
77
|
+
.file("native/vpx_shim.c")
|
|
78
|
+
.include(prefix.join("include"))
|
|
79
|
+
.static_crt(true)
|
|
80
|
+
.compile("dlvpx");
|
|
81
|
+
println!(
|
|
82
|
+
"cargo:rustc-link-search=native={}",
|
|
83
|
+
prefix.join("lib").display()
|
|
84
|
+
);
|
|
85
|
+
println!("cargo:rustc-link-lib=static=vpx");
|
|
86
|
+
println!("cargo:rustc-cfg=desklink_vpx");
|
|
87
|
+
}
|
|
88
|
+
return Ok(());
|
|
89
|
+
}
|
|
51
90
|
if let Some(prefix) = std::env::var_os("DESKLINK_VPX_STATIC_DIR").map(PathBuf::from) {
|
|
52
91
|
if !prefix.join("include/vpx/vp8cx.h").exists() || !prefix.join("lib/libvpx.a").exists() {
|
|
53
92
|
return Err(format!("DESKLINK_VPX_STATIC_DIR must be the libvpx install prefix containing include/vpx/vp8cx.h and lib/libvpx.a, not its lib directory: {}", prefix.display()).into());
|
|
@@ -90,11 +129,29 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|
|
90
129
|
.include("vendor/libva")
|
|
91
130
|
.compile("dlvaapi");
|
|
92
131
|
println!("cargo:rustc-link-lib=dl");
|
|
93
|
-
cc::Build::new()
|
|
132
|
+
let mut overlay = cc::Build::new();
|
|
133
|
+
let includes = std::process::Command::new("pkg-config")
|
|
134
|
+
.args(["--cflags", "freetype2", "fontconfig"])
|
|
135
|
+
.output()?;
|
|
136
|
+
if !includes.status.success() {
|
|
137
|
+
return Err("FreeType and Fontconfig development headers are required".into());
|
|
138
|
+
}
|
|
139
|
+
for flag in String::from_utf8(includes.stdout)?.split_whitespace() {
|
|
140
|
+
overlay.flag(flag);
|
|
141
|
+
}
|
|
142
|
+
overlay
|
|
94
143
|
.file("native/agent_overlay_x11.c")
|
|
95
144
|
.file("native/agent_overlay_wayland.c")
|
|
96
145
|
.compile("dlagentoverlay");
|
|
97
|
-
for lib in [
|
|
146
|
+
for lib in [
|
|
147
|
+
"X11",
|
|
148
|
+
"Xext",
|
|
149
|
+
"Xcursor",
|
|
150
|
+
"wayland-client",
|
|
151
|
+
"fontconfig",
|
|
152
|
+
"freetype",
|
|
153
|
+
"m",
|
|
154
|
+
] {
|
|
98
155
|
println!("cargo:rustc-link-lib={lib}");
|
|
99
156
|
}
|
|
100
157
|
cc::Build::new()
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
// Shared Linux agent cue, in logical pixels. The arrow follows the Mac outline;
|
|
2
|
+
// light and dark keylines keep it visible on both bright and dark desktops.
|
|
3
|
+
#ifndef DESKLINK_AGENT_INDICATOR_H
|
|
4
|
+
#define DESKLINK_AGENT_INDICATOR_H
|
|
5
|
+
#include <math.h>
|
|
6
|
+
#include <stdint.h>
|
|
7
|
+
|
|
8
|
+
static double indicator_segment(double x, double y, double ax, double ay, double bx, double by) {
|
|
9
|
+
double dx=bx-ax, dy=by-ay;
|
|
10
|
+
double t=fmax(0,fmin(1,((x-ax)*dx+(y-ay)*dy)/(dx*dx+dy*dy)));
|
|
11
|
+
return hypot(x-ax-t*dx,y-ay-t*dy);
|
|
12
|
+
}
|
|
13
|
+
static uint32_t indicator_over(uint32_t dst, double coverage, unsigned rgb) {
|
|
14
|
+
unsigned a=(unsigned)(255*fmax(0,fmin(1,coverage))), inverse=255-a;
|
|
15
|
+
unsigned da=dst>>24, r=(dst>>16)&255, g=(dst>>8)&255, b=dst&255;
|
|
16
|
+
return ((a+da*inverse/255)<<24) |
|
|
17
|
+
((((rgb>>16)&255)*a/255+r*inverse/255)<<16) |
|
|
18
|
+
((((rgb>>8)&255)*a/255+g*inverse/255)<<8) |
|
|
19
|
+
((rgb&255)*a/255+b*inverse/255);
|
|
20
|
+
}
|
|
21
|
+
// Recolour without changing coverage, so feedback on a still cue never
|
|
22
|
+
// changes which pixels capture has to restore.
|
|
23
|
+
static uint32_t indicator_tint(uint32_t c, double amount, unsigned rgb) {
|
|
24
|
+
double keep=1-fmax(0,fmin(1,amount)), a=(c>>24)/255.0;
|
|
25
|
+
return (c&0xff000000) |
|
|
26
|
+
((unsigned)(((c>>16)&255)*keep+((rgb>>16)&255)*a*(1-keep))<<16) |
|
|
27
|
+
((unsigned)(((c>>8)&255)*keep+((rgb>>8)&255)*a*(1-keep))<<8) |
|
|
28
|
+
(unsigned)((c&255)*keep+(rgb&255)*a*(1-keep));
|
|
29
|
+
}
|
|
30
|
+
static uint32_t indicator_pixel(double x, double y, double opacity, double click, double typed) {
|
|
31
|
+
double radius=hypot(x,y);
|
|
32
|
+
uint32_t color=0;
|
|
33
|
+
// 62 logical px halo: a blue band, wide white keyline and dark rim, so the
|
|
34
|
+
// ring keeps >=2px of contrast after a 2:1 downscale. Inside the band only
|
|
35
|
+
// the arrow is opaque, so capture can keep the click area live.
|
|
36
|
+
color=indicator_over(color,fmax(0,fmin(1,2.5-fabs(radius-28.5)))*opacity,0x123047);
|
|
37
|
+
color=indicator_over(color,fmax(0,fmin(1,2.5-fabs(radius-24)))*opacity,0xffffff);
|
|
38
|
+
color=indicator_over(color,fmax(0,fmin(1,1.5-fabs(radius-20)))*0.95*opacity,0x2999ff);
|
|
39
|
+
static const double arrow[][2]={{0,0},{2,24},{8,18},{15,31},{21,28},{14,15},{24,14}};
|
|
40
|
+
double distance=1000; int inside=0;
|
|
41
|
+
for(int i=0,j=6;i<7;j=i++) {
|
|
42
|
+
double ax=arrow[j][0],ay=arrow[j][1],bx=arrow[i][0],by=arrow[i][1];
|
|
43
|
+
distance=fmin(distance,indicator_segment(x,y,ax,ay,bx,by));
|
|
44
|
+
if((ay>y)!=(by>y) && x<(bx-ax)*(y-ay)/(by-ay)+ax)inside=!inside;
|
|
45
|
+
}
|
|
46
|
+
double signed_distance=inside?-distance:distance;
|
|
47
|
+
color=indicator_over(color,fmax(0,fmin(1,3-signed_distance))*opacity,0x123047);
|
|
48
|
+
color=indicator_over(color,fmax(0,fmin(1,2-signed_distance))*opacity,0xffffff);
|
|
49
|
+
color=indicator_over(color,fmax(0,fmin(1,-signed_distance))*opacity,0x2999ff);
|
|
50
|
+
// A click sweeps a bright wave out through the cue; a key flashes it.
|
|
51
|
+
if(click>=0 && click<1)
|
|
52
|
+
color=indicator_tint(color,fmax(0,1-fabs(radius-(4+32*click))/5)*0.9*(1-click),0x9fdcff);
|
|
53
|
+
if(typed>=0 && typed<1)
|
|
54
|
+
color=indicator_tint(color,0.6*(1-typed),0xc8ecff);
|
|
55
|
+
return color;
|
|
56
|
+
}
|
|
57
|
+
#endif
|