@desklink/react-native 0.3.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -15
- package/android/src/main/java/expo/modules/desklink/DesklinkModule.kt +15 -0
- package/android/src/main/java/expo/modules/desklink/DesktopSession.kt +27 -0
- package/android/src/main/java/expo/modules/desklink/DesktopView.kt +75 -16
- package/package.json +1 -1
- package/src/DesktopView.ios.spec.tsx +154 -6
- package/src/DesktopView.ios.tsx +49 -20
- package/src/DesktopView.tsx +16 -2
- package/src/DesktopView.web.tsx +12 -0
- package/src/index.ts +1 -1
- package/src/native.ios.ts +5 -0
- package/src/native.ts +2 -1
- package/src/native.web.ts +5 -0
- package/src/protocol.ts +14 -1
- package/src/useDesktopSession.ts +19 -6
- package/src/videoStats.ts +16 -0
package/README.md
CHANGED
|
@@ -58,12 +58,12 @@ changes, and trackpad presses reach the desktop as touches.
|
|
|
58
58
|
## Use
|
|
59
59
|
|
|
60
60
|
```tsx
|
|
61
|
-
import { DesktopView, useDesktopSession
|
|
61
|
+
import { DesktopView, useDesktopSession } from '@desklink/react-native';
|
|
62
62
|
|
|
63
63
|
const desktop = useDesktopSession({
|
|
64
64
|
authorize: async () => ({
|
|
65
65
|
signaling, // your authenticated channel to the host
|
|
66
|
-
session: {
|
|
66
|
+
session: {},
|
|
67
67
|
}),
|
|
68
68
|
});
|
|
69
69
|
|
|
@@ -76,8 +76,17 @@ desktop.setInputEnabled(true);
|
|
|
76
76
|
desktop.showKeyboard();
|
|
77
77
|
desktop.setOrientation('landscape'); // Android only; iOS apps configure supported orientations themselves
|
|
78
78
|
await desktop.pasteLocalToRemote(await Clipboard.getStringAsync());
|
|
79
|
+
// Optional receiver evidence: codec, decoder and video counters; no ICE addresses.
|
|
80
|
+
const stats = await desktop.getStats();
|
|
79
81
|
```
|
|
80
82
|
|
|
83
|
+
Pairing is the only trust boundary. `authorize` is the app's pairing check: it
|
|
84
|
+
returns a channel to a paired computer or throws, and it runs again on every
|
|
85
|
+
reconnect, so a revoked pairing cannot reopen. A paired session has view,
|
|
86
|
+
input and clipboard; there are no in-app grants, and `session.permissions` is
|
|
87
|
+
ignored. Only the operating system's own consent (screen recording,
|
|
88
|
+
accessibility, a screen-sharing prompt) still applies on the computer.
|
|
89
|
+
|
|
81
90
|
A screen that only decides *whether* to offer a desktop does not need the
|
|
82
91
|
session: import the flag from its own entry, which is a constant on web and a
|
|
83
92
|
check for the platform module elsewhere, so the hook and the view stay out of
|
|
@@ -98,7 +107,7 @@ engine only produces frames on change.
|
|
|
98
107
|
Keep the encoded picture inside a 1920×1080 box:
|
|
99
108
|
|
|
100
109
|
```tsx
|
|
101
|
-
session: {
|
|
110
|
+
session: { maxWidth: 1920, maxHeight: 1080 },
|
|
102
111
|
```
|
|
103
112
|
|
|
104
113
|
The engine never upscales, so a smaller desktop is sent at its own size and a
|
|
@@ -120,15 +129,23 @@ heat and battery.
|
|
|
120
129
|
so the iOS receiver asks for H.264 by putting it first in the offer it
|
|
121
130
|
applies; a host with an H.264 encoder then sends H.264, and one without sends
|
|
122
131
|
VP9. Neither platform links a second WebRTC stack.
|
|
123
|
-
- **A sharp, zoomable picture.**
|
|
124
|
-
|
|
125
|
-
|
|
132
|
+
- **A sharp, zoomable picture.** In portrait the default is fit-width;
|
|
133
|
+
in landscape it covers the screen below the status bar (as far as the 2.5
|
|
134
|
+
zoom limit allows). The picture stays undistorted. One finger pans cropped
|
|
135
|
+
content; pinch to zoom before panning when the whole desktop already fits.
|
|
136
|
+
Pass the screen's safe-area `insets` (for example from
|
|
137
|
+
`react-native-safe-area-context`) so the default starts below the status bar
|
|
138
|
+
and beside a camera cutout. Panning brings each desktop edge inside the safe
|
|
139
|
+
area, while landscape coverage extends beneath the cutout and home indicator
|
|
140
|
+
to limit black bars. A pinch zooms up to 2.5 view
|
|
141
|
+
pixels per desktop pixel, or out to the whole desktop. On Android the decoded frame is copied once into the view's
|
|
126
142
|
own texture and drawn with a multi-tap filter when it is shown smaller than
|
|
127
143
|
its size, so a fitted 4K desktop does not alias and a pinch redraws at once.
|
|
128
144
|
iOS uses `RTCView`'s native renderer. That renderer draws a frame only when
|
|
129
145
|
its timestamp is new, and a receiver honouring the engine's zero playout delay
|
|
130
146
|
stamps every frame alike, so the iOS peer declines that hint and keeps the
|
|
131
|
-
receiver's usual jitter buffer. `fitToView()` shows the whole desktop
|
|
147
|
+
receiver's usual jitter buffer. `fitToView()` shows the whole desktop; a new
|
|
148
|
+
desktop size or a rotation reapplies the orientation default from the top-left.
|
|
132
149
|
- **A pointer a phone can see.** Once a touch has sent the desktop's pointer
|
|
133
150
|
somewhere, the view draws an outlined arrow there at a readable size, over a
|
|
134
151
|
picture whose own cursor is a few pixels tall or not captured at all. On iOS,
|
|
@@ -141,7 +158,7 @@ heat and battery.
|
|
|
141
158
|
(`keyboardClearance`). Android follows the keyboard animation and keeps the
|
|
142
159
|
pointer in sight; iOS follows the keyboard show/hide bounds.
|
|
143
160
|
- **Contained geometry.** Touch maps through the picture's actual placement.
|
|
144
|
-
|
|
161
|
+
On the whole desktop, a one-finger drag starting in the letterbox does not move the pointer; a
|
|
145
162
|
second finger on the picture can still start a pinch or scroll. A drag that
|
|
146
163
|
leaves the picture is held to its edge rather than released somewhere unseen.
|
|
147
164
|
- **The gestures remote-desktop viewers settled on,** with a slop threshold:
|
|
@@ -212,11 +229,13 @@ heat and battery.
|
|
|
212
229
|
network is still down. Renegotiation transients from a restart the session
|
|
213
230
|
asked for itself are expected, not a new outage: they neither flip the
|
|
214
231
|
status nor trigger another restart for a short grace window.
|
|
215
|
-
- **
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
a
|
|
232
|
+
- **Input starts disarmed.** This is a local arm against accidental touches,
|
|
233
|
+
not a permission: the session already has input. `setInputEnabled(true)` arms
|
|
234
|
+
pointer, keyboard and clipboard input; `setInputEnabled(false)` blocks new
|
|
235
|
+
input, releases held keys and buttons, and resets pending gestures. The app
|
|
236
|
+
decides when to arm it after a deliberate action and must disarm it on
|
|
237
|
+
background or lost readiness; a clipboard write already in flight may still
|
|
238
|
+
finish.
|
|
220
239
|
- **Released state.** The app disables input on background and unmount;
|
|
221
240
|
session close and a lost control channel release what the desktop was holding.
|
|
222
241
|
- **Explicit clipboard.** Two methods, called on a user action while control is
|
|
@@ -274,11 +293,16 @@ What the consuming app — not this package — must get right before review.
|
|
|
274
293
|
disabling input on background; the engine releases held keys within
|
|
275
294
|
seconds of the path dying, so nothing stays stuck.
|
|
276
295
|
|
|
277
|
-
##
|
|
296
|
+
## Example app proof
|
|
278
297
|
|
|
279
298
|
`example/` is the smallest app that shows a desktop: a native (non-Expo Go)
|
|
280
299
|
build with `react-native-webrtc`, connecting to `desklink-host bridge` with the
|
|
281
|
-
URL it is launched with
|
|
300
|
+
URL it is launched with on iOS, or the connection link it is opened with on
|
|
301
|
+
Android. Its hidden accessibility status shows plain messages: `Connected`
|
|
302
|
+
once a frame is presented, `Can’t reach your desktop. Reconnecting…` while
|
|
303
|
+
retrying, and `Your desktop ended this session` after revocation. It never
|
|
304
|
+
displays the bridge URL, pairing token, raw failure text or frame metrics.
|
|
305
|
+
`test/ios-flow.mjs` builds it for the iOS simulator on
|
|
282
306
|
a Mac over ssh, serves a private Xvfb desktop from this machine through the
|
|
283
307
|
bridge, and checks that the first frame is presented, that the answer puts
|
|
284
308
|
H.264 first with NACK when the host offers it (VP9 otherwise; both descriptions
|
|
@@ -314,6 +338,19 @@ To try the example by hand against any bridge:
|
|
|
314
338
|
xcrun simctl launch <device> dev.desklink.example -desklinkUrl 'ws://HOST:PORT/desktop?token=TOKEN'
|
|
315
339
|
```
|
|
316
340
|
|
|
341
|
+
On Android, build the same example with `npm run android` and open a connection
|
|
342
|
+
link whose `url` query parameter is the URL-encoded bridge WebSocket address:
|
|
343
|
+
`desklink-example://connect?url=ws%3A%2F%2FHOST%3APORT%2Fdesktop%3Ftoken%3DTOKEN`.
|
|
344
|
+
The example allows the bridge's plaintext private-network connection; consumer
|
|
345
|
+
apps choose their own transport policy. A new link replaces the open session.
|
|
346
|
+
The live picture has no diagnostic overlay. Its accessibility status remains
|
|
347
|
+
available as `desklink-status`.
|
|
348
|
+
|
|
349
|
+
For device evidence, an optional `report` query parameter accepts a URL-encoded
|
|
350
|
+
HTTP endpoint. While live, the example posts receiver statistics and connection
|
|
351
|
+
and first-presentation timestamps once per second. Collection failures do not
|
|
352
|
+
interrupt the session. Without that parameter, no evidence is posted.
|
|
353
|
+
|
|
317
354
|
## Licence
|
|
318
355
|
|
|
319
356
|
Apache-2.0. See `NOTICE`.
|
|
@@ -5,6 +5,7 @@ import android.content.pm.ActivityInfo
|
|
|
5
5
|
import android.content.res.Configuration
|
|
6
6
|
import expo.modules.kotlin.modules.Module
|
|
7
7
|
import expo.modules.kotlin.modules.ModuleDefinition
|
|
8
|
+
import expo.modules.kotlin.Promise
|
|
8
9
|
import java.util.concurrent.ConcurrentHashMap
|
|
9
10
|
|
|
10
11
|
/**
|
|
@@ -113,6 +114,12 @@ class DesklinkModule : Module() {
|
|
|
113
114
|
|
|
114
115
|
Function("isAvailable") { true }
|
|
115
116
|
|
|
117
|
+
AsyncFunction("getStats") { id: String, promise: Promise ->
|
|
118
|
+
val session = sessions[id]
|
|
119
|
+
if (session == null) promise.resolve("[]")
|
|
120
|
+
else session.getStats { promise.resolve(it) }
|
|
121
|
+
}
|
|
122
|
+
|
|
116
123
|
View(DesktopView::class) {
|
|
117
124
|
Name("DesklinkSurface")
|
|
118
125
|
|
|
@@ -126,6 +133,14 @@ class DesklinkModule : Module() {
|
|
|
126
133
|
view.setKeyboardClearance(clearance ?: 0f)
|
|
127
134
|
}
|
|
128
135
|
|
|
136
|
+
Prop("insets") { view: DesktopView, insets: List<Double>? ->
|
|
137
|
+
val edges = insets.orEmpty()
|
|
138
|
+
view.setInsets(
|
|
139
|
+
edges.getOrNull(0)?.toFloat() ?: 0f, edges.getOrNull(1)?.toFloat() ?: 0f,
|
|
140
|
+
edges.getOrNull(2)?.toFloat() ?: 0f, edges.getOrNull(3)?.toFloat() ?: 0f,
|
|
141
|
+
)
|
|
142
|
+
}
|
|
143
|
+
|
|
129
144
|
Prop("gestures") { view: DesktopView, gestures: String? ->
|
|
130
145
|
view.setGestures(gestures ?: "desktop")
|
|
131
146
|
}
|
|
@@ -370,6 +370,33 @@ class DesktopSession(
|
|
|
370
370
|
return servers
|
|
371
371
|
}
|
|
372
372
|
|
|
373
|
+
/** A bounded diagnostics snapshot; never expose ICE addresses or control content. */
|
|
374
|
+
@Synchronized
|
|
375
|
+
fun getStats(done: (String) -> Unit) {
|
|
376
|
+
if (closed) {
|
|
377
|
+
done("[]")
|
|
378
|
+
return
|
|
379
|
+
}
|
|
380
|
+
io.execute {
|
|
381
|
+
val connection = peer
|
|
382
|
+
if (closed || connection == null) {
|
|
383
|
+
done("[]")
|
|
384
|
+
return@execute
|
|
385
|
+
}
|
|
386
|
+
connection.getStats { report ->
|
|
387
|
+
val fields = setOf("kind", "mediaType", "codecId", "mimeType", "clockRate", "payloadType",
|
|
388
|
+
"packetsReceived", "packetsLost", "bytesReceived", "framesReceived", "framesDecoded",
|
|
389
|
+
"keyFramesDecoded", "framesDropped", "framesPerSecond", "frameWidth", "frameHeight",
|
|
390
|
+
"nackCount", "pliCount", "firCount", "totalDecodeTime", "decoderImplementation", "jitterBufferDelay")
|
|
391
|
+
val stats = report.statsMap.values.filter { it.type == "inbound-rtp" || it.type == "codec" }
|
|
392
|
+
.map { stat -> JSONObject(stat.members.filterKeys { it in fields }).apply {
|
|
393
|
+
put("id", stat.id); put("type", stat.type); put("timestampUs", stat.timestampUs)
|
|
394
|
+
} }
|
|
395
|
+
done(org.json.JSONArray(stats).toString())
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
|
|
373
400
|
private fun observer(target: Long) = object : PeerConnection.Observer {
|
|
374
401
|
override fun onSignalingChange(state: PeerConnection.SignalingState?) = Unit
|
|
375
402
|
|
|
@@ -65,7 +65,8 @@ private const val MIN_WHEEL_STEP = 0.05f
|
|
|
65
65
|
* - tap: click where the finger lands; two taps: a double click on the same spot;
|
|
66
66
|
* - press and hold: a right click on release, or drag after it to hold the left
|
|
67
67
|
* button (select text, move a window);
|
|
68
|
-
* - one finger: move around a zoomed-in desktop
|
|
68
|
+
* - one finger: move around a zoomed-in desktop — the picture starts filled,
|
|
69
|
+
* covering the view, so this is the default; on the whole desktop,
|
|
69
70
|
* move its pointer, which follows the finger without pressing a button —
|
|
70
71
|
* unless `gestures` is `browser`, where it scrolls the page, or `device`,
|
|
71
72
|
* where it lands with the button down and drags with it held;
|
|
@@ -102,11 +103,23 @@ class DesktopView(context: Context, appContext: AppContext) : ExpoView(context,
|
|
|
102
103
|
|
|
103
104
|
// The picture's placement: surface pixels per desktop pixel, and where the
|
|
104
105
|
// desktop's top-left corner sits in this view. `fitted` keeps a picture that
|
|
105
|
-
// was showing the whole desktop showing it when the view changes size
|
|
106
|
+
// was showing the whole desktop showing it when the view changes size;
|
|
107
|
+
// `filling` keeps the default, a picture covering the view, until the user
|
|
108
|
+
// zooms or asks for the whole desktop. `filledScale` is the fill it was last
|
|
109
|
+
// placed at, so a new fill (a rotation, a new surface) starts at the
|
|
110
|
+
// desktop's top-left corner again.
|
|
106
111
|
private var scale = 1f
|
|
107
112
|
private var originX = 0f
|
|
108
113
|
private var originY = 0f
|
|
109
114
|
private var fitted = true
|
|
115
|
+
private var filling = true
|
|
116
|
+
private var filledScale = 0f
|
|
117
|
+
|
|
118
|
+
/** What the system covers at each edge (bars, cutout), in pixels: left, top, right, bottom. */
|
|
119
|
+
private var insetLeft = 0f
|
|
120
|
+
private var insetTop = 0f
|
|
121
|
+
private var insetRight = 0f
|
|
122
|
+
private var insetBottom = 0f
|
|
110
123
|
|
|
111
124
|
/**
|
|
112
125
|
* How much of this view's bottom the phone's keyboard, and the app's own
|
|
@@ -254,6 +267,8 @@ class DesktopView(context: Context, appContext: AppContext) : ExpoView(context,
|
|
|
254
267
|
surfaceWidth = width
|
|
255
268
|
surfaceHeight = height
|
|
256
269
|
fitted = true
|
|
270
|
+
filling = true
|
|
271
|
+
filledScale = 0f
|
|
257
272
|
// A point on the old geometry says nothing about the new one.
|
|
258
273
|
pointerAt = null
|
|
259
274
|
layoutPicture()
|
|
@@ -296,6 +311,22 @@ class DesktopView(context: Context, appContext: AppContext) : ExpoView(context,
|
|
|
296
311
|
/** Show the whole desktop again. */
|
|
297
312
|
fun fitToView() {
|
|
298
313
|
fitted = true
|
|
314
|
+
filling = false
|
|
315
|
+
layoutPicture()
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* What the system covers at each edge of the view, in points. The picture
|
|
320
|
+
* still reaches under it, but starts and pans inside it.
|
|
321
|
+
*/
|
|
322
|
+
fun setInsets(top: Float, left: Float, bottom: Float, right: Float) {
|
|
323
|
+
val density = resources.displayMetrics.density
|
|
324
|
+
if (top * density == insetTop && left * density == insetLeft && bottom * density == insetBottom && right * density == insetRight) return
|
|
325
|
+
insetTop = top * density
|
|
326
|
+
insetLeft = left * density
|
|
327
|
+
insetBottom = bottom * density
|
|
328
|
+
insetRight = right * density
|
|
329
|
+
filledScale = 0f
|
|
299
330
|
layoutPicture()
|
|
300
331
|
}
|
|
301
332
|
|
|
@@ -347,9 +378,9 @@ class DesktopView(context: Context, appContext: AppContext) : ExpoView(context,
|
|
|
347
378
|
*/
|
|
348
379
|
private fun coverBottom(next: Float) {
|
|
349
380
|
if (abs(next - covered) < 0.5f) return
|
|
350
|
-
val before =
|
|
381
|
+
val before = visibleBottom()
|
|
351
382
|
covered = next
|
|
352
|
-
val visible =
|
|
383
|
+
val visible = visibleBottom()
|
|
353
384
|
val at = pointerAt
|
|
354
385
|
if (at == null) {
|
|
355
386
|
originY += (visible - before) / 2f
|
|
@@ -361,7 +392,13 @@ class DesktopView(context: Context, appContext: AppContext) : ExpoView(context,
|
|
|
361
392
|
layoutPicture()
|
|
362
393
|
}
|
|
363
394
|
|
|
364
|
-
|
|
395
|
+
/** Where the uncovered part of the view ends: above the keyboard, or above what the system covers. */
|
|
396
|
+
private fun visibleBottom(): Float = max(insetTop + 1f, min(height - covered, height - insetBottom))
|
|
397
|
+
|
|
398
|
+
private fun safeWidth(): Float = max(1f, width - insetLeft - insetRight)
|
|
399
|
+
|
|
400
|
+
/** The whole uncovered height, keyboard or not: the keyboard moves the picture rather than shrinking it. */
|
|
401
|
+
private fun safeHeight(): Float = max(1f, height - insetTop - insetBottom)
|
|
365
402
|
|
|
366
403
|
private fun inputMethods() = context.getSystemService(Context.INPUT_METHOD_SERVICE) as InputMethodManager
|
|
367
404
|
|
|
@@ -386,8 +423,10 @@ class DesktopView(context: Context, appContext: AppContext) : ExpoView(context,
|
|
|
386
423
|
|
|
387
424
|
override fun onSizeChanged(w: Int, h: Int, oldw: Int, oldh: Int) {
|
|
388
425
|
super.onSizeChanged(w, h, oldw, oldh)
|
|
389
|
-
// A rotation:
|
|
390
|
-
|
|
426
|
+
// A rotation: a filled picture starts again at its top-left corner;
|
|
427
|
+
// a zoomed one keeps what the user was looking at.
|
|
428
|
+
if (filling) filledScale = 0f
|
|
429
|
+
else if (oldw > 0 && oldh > 0 && !fitted) {
|
|
391
430
|
originX += (w - oldw) / 2f
|
|
392
431
|
originY += (h - oldh) / 2f
|
|
393
432
|
}
|
|
@@ -416,34 +455,54 @@ class DesktopView(context: Context, appContext: AppContext) : ExpoView(context,
|
|
|
416
455
|
*/
|
|
417
456
|
private fun fitScale(): Float {
|
|
418
457
|
if (width == 0 || height == 0 || surfaceWidth == 0 || surfaceHeight == 0) return 1f
|
|
419
|
-
return min(
|
|
458
|
+
return min(safeWidth() / surfaceWidth, safeHeight() / surfaceHeight)
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
private fun fillScale(): Float {
|
|
462
|
+
val cover = if (width <= height) safeWidth() / surfaceWidth
|
|
463
|
+
else max(width.toFloat() / surfaceWidth, (height - insetTop) / surfaceHeight)
|
|
464
|
+
return max(fitScale(), min(cover, MAX_SCALE))
|
|
420
465
|
}
|
|
421
466
|
|
|
422
|
-
/** Re-derive the placement from `fitted`, the zoom limits and the edges. */
|
|
467
|
+
/** Re-derive the placement from `filling`, `fitted`, the zoom limits and the edges. */
|
|
423
468
|
private fun layoutPicture() {
|
|
424
469
|
if (surfaceWidth == 0 || surfaceHeight == 0 || width == 0 || height == 0) return
|
|
425
470
|
val fit = fitScale()
|
|
426
|
-
|
|
471
|
+
if (filling) {
|
|
472
|
+
val fill = fillScale()
|
|
473
|
+
if (fill != filledScale) {
|
|
474
|
+
scale = fill
|
|
475
|
+
filledScale = fill
|
|
476
|
+
// The desktop's top-left corner, where a page's first words are.
|
|
477
|
+
originX = insetLeft
|
|
478
|
+
originY = insetTop
|
|
479
|
+
}
|
|
480
|
+
} else {
|
|
481
|
+
scale = if (fitted) fit else scale.coerceIn(fit, max(fit, MAX_SCALE))
|
|
482
|
+
}
|
|
427
483
|
fitted = scale <= fit * 1.001f
|
|
428
484
|
clampOrigin()
|
|
429
485
|
publishTransform()
|
|
430
486
|
}
|
|
431
487
|
|
|
432
488
|
/**
|
|
433
|
-
* A picture smaller than
|
|
434
|
-
* one
|
|
489
|
+
* A picture smaller than the uncovered part of the view is centred in it on
|
|
490
|
+
* that axis; a larger one can move until either edge reaches that part's
|
|
491
|
+
* edge. Above the keyboard, only the part above it counts.
|
|
435
492
|
*/
|
|
436
493
|
private fun clampOrigin() {
|
|
437
494
|
val pictureWidth = surfaceWidth * scale
|
|
438
495
|
val pictureHeight = surfaceHeight * scale
|
|
439
|
-
val
|
|
440
|
-
|
|
441
|
-
|
|
496
|
+
val right = width - insetRight
|
|
497
|
+
val bottom = visibleBottom()
|
|
498
|
+
originX = if (pictureWidth <= right - insetLeft) insetLeft + (right - insetLeft - pictureWidth) / 2f else originX.coerceIn(right - pictureWidth, insetLeft)
|
|
499
|
+
originY = if (pictureHeight <= bottom - insetTop) insetTop + (if (filling) 0f else (bottom - insetTop - pictureHeight) / 2f) else originY.coerceIn(bottom - pictureHeight, insetTop)
|
|
442
500
|
}
|
|
443
501
|
|
|
444
502
|
private fun zoomAround(focusX: Float, focusY: Float, factor: Float) {
|
|
445
503
|
val fit = fitScale()
|
|
446
504
|
val next = (scale * factor).coerceIn(fit, max(fit, MAX_SCALE))
|
|
505
|
+
filling = false
|
|
447
506
|
val applied = next / scale
|
|
448
507
|
originX = focusX - (focusX - originX) * applied
|
|
449
508
|
originY = focusY - (focusY - originY) * applied
|
|
@@ -623,7 +682,7 @@ class DesktopView(context: Context, appContext: AppContext) : ExpoView(context,
|
|
|
623
682
|
}
|
|
624
683
|
// The whole desktop has nowhere to move to, so the finger moves the
|
|
625
684
|
// desktop's pointer instead, without a button.
|
|
626
|
-
gesture = if (!fitted) Gesture.PAN else if (downOnPicture) Gesture.HOVER else Gesture.LETTERBOX
|
|
685
|
+
gesture = if (filling || !fitted) Gesture.PAN else if (downOnPicture) Gesture.HOVER else Gesture.LETTERBOX
|
|
627
686
|
lastX = event.x
|
|
628
687
|
lastY = event.y
|
|
629
688
|
if (gesture == Gesture.HOVER) hoverTo(active, event.x, event.y)
|
package/package.json
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
2
2
|
import React from 'react';
|
|
3
3
|
import TestRenderer from 'react-test-renderer';
|
|
4
|
+
import { pictureBox } from '../test/picture-box.mjs';
|
|
4
5
|
|
|
5
6
|
/** `act` refuses to flush state updates unless React is told this is a test. */
|
|
6
7
|
(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true;
|
|
@@ -57,14 +58,16 @@ const required: string[] = [];
|
|
|
57
58
|
|
|
58
59
|
const BOUNDS = { width: 390, height: 844 };
|
|
59
60
|
const DESKTOP = { width: 1280, height: 720 };
|
|
60
|
-
//
|
|
61
|
+
// Most of this spec works on the whole desktop (`whole: true`, as after
|
|
62
|
+
// `fitToView()`): fit = min(390/1280, 844/720); the picture is 390 wide,
|
|
63
|
+
// centred vertically.
|
|
61
64
|
const SCALE = Math.min(BOUNDS.width / DESKTOP.width, BOUNDS.height / DESKTOP.height);
|
|
62
65
|
const ORIGIN_Y = (BOUNDS.height - DESKTOP.height * SCALE) / 2;
|
|
63
66
|
|
|
64
67
|
type Touch = { locationX: number; locationY: number; timestamp?: number };
|
|
65
68
|
function touch(x: number, y: number, timestamp?: number): Touch { return { locationX: x, locationY: y, timestamp }; }
|
|
66
69
|
|
|
67
|
-
async function openView(options: { gestures?: 'desktop' | 'browser' | 'device'; accessibilityLabel?: string; native?: boolean } = {}) {
|
|
70
|
+
async function openView(options: { gestures?: 'desktop' | 'browser' | 'device'; accessibilityLabel?: string; native?: boolean; whole?: boolean; desktop?: { width: number; height: number }; bounds?: { width: number; height: number }; insets?: { top: number; left: number; bottom: number; right: number } } = {}) {
|
|
68
71
|
nativeInput.installed = options.native === true;
|
|
69
72
|
const { desktopInputEnabled, nativeDesklink } = await import('./native.ios');
|
|
70
73
|
const { DesktopView } = await import('./DesktopView.ios');
|
|
@@ -79,7 +82,7 @@ async function openView(options: { gestures?: 'desktop' | 'browser' | 'device';
|
|
|
79
82
|
let renderer: TestRenderer.ReactTestRenderer | null = null;
|
|
80
83
|
TestRenderer.act(() => {
|
|
81
84
|
renderer = TestRenderer.create(
|
|
82
|
-
<DesktopView sessionId={id} gestures={options.gestures} accessibilityLabel={options.accessibilityLabel} />,
|
|
85
|
+
<DesktopView sessionId={id} gestures={options.gestures} accessibilityLabel={options.accessibilityLabel} insets={options.insets} />,
|
|
83
86
|
);
|
|
84
87
|
});
|
|
85
88
|
const outer = () => renderer!.root.findByType(options.native ? 'DesklinkInput' as never : 'View').props as {
|
|
@@ -90,10 +93,11 @@ async function openView(options: { gestures?: 'desktop' | 'browser' | 'device';
|
|
|
90
93
|
[key: string]: unknown;
|
|
91
94
|
};
|
|
92
95
|
TestRenderer.act(() => {
|
|
93
|
-
outer().onLayout({ nativeEvent: { layout: { ...BOUNDS } } });
|
|
94
|
-
nativeDesklink.setSurfaceSize(id, DESKTOP.width, DESKTOP.height);
|
|
96
|
+
outer().onLayout({ nativeEvent: { layout: { ...(options.bounds ?? BOUNDS) } } });
|
|
97
|
+
nativeDesklink.setSurfaceSize(id, (options.desktop ?? DESKTOP).width, (options.desktop ?? DESKTOP).height);
|
|
95
98
|
peer.ontrack!({ track: { kind: 'video' }, streams: [] });
|
|
96
99
|
});
|
|
100
|
+
if (options.whole ?? true) TestRenderer.act(() => { nativeDesklink.fitToView(id); });
|
|
97
101
|
const grant = (at: Touch[]) => TestRenderer.act(() => { outer().onResponderGrant({ nativeEvent: { touches: at } }); });
|
|
98
102
|
const move = (at: Touch[]) => TestRenderer.act(() => { outer().onResponderMove({ nativeEvent: { touches: at } }); });
|
|
99
103
|
const release = (at: Touch[] = []) => TestRenderer.act(() => { outer().onResponderRelease({ nativeEvent: { touches: at } }); });
|
|
@@ -130,6 +134,124 @@ beforeEach(() => {
|
|
|
130
134
|
afterEach(() => { vi.unstubAllGlobals(); });
|
|
131
135
|
|
|
132
136
|
describe('DesktopView.ios parity with DesktopView.kt', () => {
|
|
137
|
+
it('fits portrait width, then zooms and pans without distorting the desktop', async () => {
|
|
138
|
+
const { sent, rtcStyle, grant, move, release } = await openView({ whole: false });
|
|
139
|
+
expect(rtcStyle()).toMatchObject({ left: 0, top: 0, width: BOUNDS.width });
|
|
140
|
+
expect(rtcStyle().height).toBeCloseTo(DESKTOP.height * SCALE, 5);
|
|
141
|
+
grant([touch(100, 100)]);
|
|
142
|
+
move([touch(100, 100), touch(140, 100)]);
|
|
143
|
+
move([touch(80, 100), touch(160, 100)]);
|
|
144
|
+
release();
|
|
145
|
+
const before = rtcStyle();
|
|
146
|
+
expect(before.width / before.height).toBeCloseTo(DESKTOP.width / DESKTOP.height, 5);
|
|
147
|
+
grant([touch(260, 400)]);
|
|
148
|
+
move([touch(200, 400)]);
|
|
149
|
+
release();
|
|
150
|
+
expect(sent.filter((message) => message.kind === 'pointer')).toEqual([]);
|
|
151
|
+
expect(rtcStyle().left).toBeCloseTo(before.left - 60, 5);
|
|
152
|
+
const at = rtcStyle();
|
|
153
|
+
grant([touch(100, 300)]);
|
|
154
|
+
release();
|
|
155
|
+
expect(sent.filter((message) => message.kind === 'pointer')).toMatchObject([
|
|
156
|
+
{ kind: 'pointer', phase: 'down', x: Math.floor((100 - at.left) / (SCALE * 2)), y: Math.floor((300 - at.top) / (SCALE * 2)), button: 1 },
|
|
157
|
+
{ kind: 'pointer', phase: 'up', x: Math.floor((100 - at.left) / (SCALE * 2)), y: Math.floor((300 - at.top) / (SCALE * 2)), button: 1 },
|
|
158
|
+
]);
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
it('pans a tall portrait desktop to both vertical edges at fit-width', async () => {
|
|
162
|
+
const insets = { top: 47, left: 0, bottom: 21, right: 0 };
|
|
163
|
+
const { rtcStyle, grant, move, release } = await openView({ whole: false, insets, desktop: { width: 720, height: 1800 } });
|
|
164
|
+
expect(rtcStyle().width).toBeCloseTo(390, 5);
|
|
165
|
+
expect(rtcStyle().top).toBeCloseTo(47, 5);
|
|
166
|
+
grant([touch(200, 400)]);
|
|
167
|
+
move([touch(200, -2000)]);
|
|
168
|
+
release();
|
|
169
|
+
expect(rtcStyle().top + rtcStyle().height).toBeCloseTo(823, 5);
|
|
170
|
+
grant([touch(200, 400)]);
|
|
171
|
+
move([touch(200, 2000)]);
|
|
172
|
+
release();
|
|
173
|
+
expect(rtcStyle().top).toBeCloseTo(47, 5);
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
it('starts again at the top-left corner after a rotation, and keeps the whole desktop when the same size is reported again', async () => {
|
|
177
|
+
const { id, nativeDesklink, outer, rtcStyle, grant, move, release } = await openView({ whole: false });
|
|
178
|
+
grant([touch(260, 400)]);
|
|
179
|
+
move([touch(200, 400)]);
|
|
180
|
+
release();
|
|
181
|
+
expect(rtcStyle().left).toBeCloseTo(0, 5);
|
|
182
|
+
const layout = (bounds: { width: number; height: number }) => TestRenderer.act(() => {
|
|
183
|
+
(outer().onLayout as (event: unknown) => void)({ nativeEvent: { layout: bounds } });
|
|
184
|
+
});
|
|
185
|
+
layout({ width: BOUNDS.height, height: BOUNDS.width });
|
|
186
|
+
// Landscape crops the bottom, never the top.
|
|
187
|
+
expect(rtcStyle().top).toBeCloseTo(0, 5);
|
|
188
|
+
expect(rtcStyle().left).toBeCloseTo(0, 5);
|
|
189
|
+
layout(BOUNDS);
|
|
190
|
+
expect(rtcStyle().left).toBeCloseTo(0, 5);
|
|
191
|
+
TestRenderer.act(() => { nativeDesklink.fitToView(id); });
|
|
192
|
+
TestRenderer.act(() => { nativeDesklink.setSurfaceSize(id, DESKTOP.width, DESKTOP.height); });
|
|
193
|
+
expect(rtcStyle().width).toBeCloseTo(BOUNDS.width, 5);
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
it('resets both default offsets on equal-scale layout and inset changes', async () => {
|
|
197
|
+
const { DesktopView } = await import('./DesktopView.ios');
|
|
198
|
+
const insets = { top: 0, left: 47, bottom: 21, right: 47 };
|
|
199
|
+
const { id, renderer, outer, rtcStyle, grant, move, release } = await openView({ whole: false, bounds: { width: 844, height: 390 }, insets });
|
|
200
|
+
const pan = () => {
|
|
201
|
+
grant([touch(400, 200)]);
|
|
202
|
+
move([touch(340, 140)]);
|
|
203
|
+
release();
|
|
204
|
+
};
|
|
205
|
+
const width = rtcStyle().width;
|
|
206
|
+
pan();
|
|
207
|
+
expect(rtcStyle().left).toBeLessThan(47);
|
|
208
|
+
expect(rtcStyle().top).toBeLessThan(0);
|
|
209
|
+
TestRenderer.act(() => { outer().onLayout({ nativeEvent: { layout: { width: 844, height: 400 } } }); });
|
|
210
|
+
expect(rtcStyle()).toMatchObject({ left: 47, top: 0, width });
|
|
211
|
+
pan();
|
|
212
|
+
TestRenderer.act(() => { renderer.update(<DesktopView sessionId={id} insets={{ top: 10, left: 50, right: 44, bottom: 21 }} />); });
|
|
213
|
+
expect(rtcStyle()).toMatchObject({ left: 50, top: 10, width });
|
|
214
|
+
});
|
|
215
|
+
|
|
216
|
+
it('starts and pans inside the insets while the picture still reaches under them', async () => {
|
|
217
|
+
const landscape = { width: 844, height: 390 };
|
|
218
|
+
const insets = { top: 0, left: 47, bottom: 21, right: 47 };
|
|
219
|
+
const { rtcStyle, grant, move, release } = await openView({ whole: false, bounds: landscape, insets });
|
|
220
|
+
const fill = Math.max(landscape.width / DESKTOP.width, landscape.height / DESKTOP.height);
|
|
221
|
+
expect(rtcStyle().width).toBeCloseTo(DESKTOP.width * fill, 5);
|
|
222
|
+
// The desktop's top-left corner shows just inside the cutout's side.
|
|
223
|
+
expect(rtcStyle().left).toBeCloseTo(47, 5);
|
|
224
|
+
expect(rtcStyle().top).toBeCloseTo(0, 5);
|
|
225
|
+
// Panned all the way, the desktop's right and bottom edges reach the
|
|
226
|
+
// uncovered part's edges, not the screen's.
|
|
227
|
+
grant([touch(600, 300)]);
|
|
228
|
+
move([touch(-2000, -2000)]);
|
|
229
|
+
release();
|
|
230
|
+
expect(rtcStyle().left + rtcStyle().width).toBeCloseTo(landscape.width - 47, 5);
|
|
231
|
+
expect(rtcStyle().top + rtcStyle().height).toBeCloseTo(landscape.height - 21, 5);
|
|
232
|
+
for (const x of [-2000, 2000]) for (const y of [-2000, 2000]) {
|
|
233
|
+
grant([touch(400, 200)]);
|
|
234
|
+
move([touch(x, y)]);
|
|
235
|
+
release();
|
|
236
|
+
const box = rtcStyle();
|
|
237
|
+
const width = Math.min(landscape.width, box.left + box.width) - Math.max(0, box.left);
|
|
238
|
+
const height = Math.min(landscape.height, box.top + box.height) - Math.max(0, box.top);
|
|
239
|
+
expect(1 - width * height / (landscape.width * landscape.height)).toBeLessThan(0.15);
|
|
240
|
+
expect(box.width / box.height).toBeCloseTo(DESKTOP.width / DESKTOP.height, 5);
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
});
|
|
244
|
+
|
|
245
|
+
it('leaves no black bars on a landscape phone, and fitToView shows the whole desktop', async () => {
|
|
246
|
+
const landscape = { width: 844, height: 390 };
|
|
247
|
+
const { id, nativeDesklink, rtcStyle } = await openView({ whole: false, bounds: landscape });
|
|
248
|
+
expect(rtcStyle().width).toBeCloseTo(landscape.width, 5);
|
|
249
|
+
expect(rtcStyle().left).toBeCloseTo(0, 5);
|
|
250
|
+
TestRenderer.act(() => { nativeDesklink.fitToView(id); });
|
|
251
|
+
expect(rtcStyle().height).toBeCloseTo(landscape.height, 5);
|
|
252
|
+
expect(rtcStyle().width).toBeCloseTo(DESKTOP.width * landscape.height / DESKTOP.height, 5);
|
|
253
|
+
});
|
|
254
|
+
|
|
133
255
|
it('ignores a one-finger drag that starts in the letterbox', async () => {
|
|
134
256
|
const { sent, grant, move, release } = await openView();
|
|
135
257
|
// y = 10 is above the fitted picture (it starts at ORIGIN_Y).
|
|
@@ -148,7 +270,8 @@ describe('DesktopView.ios parity with DesktopView.kt', () => {
|
|
|
148
270
|
const { rtcStyle, grant, move } = await openView();
|
|
149
271
|
grant([touch(100, 400)]);
|
|
150
272
|
move([touch(100, 400), touch(140, 400)]);
|
|
151
|
-
move
|
|
273
|
+
// A finger's pinch spreads a few points per move, each step under the slop.
|
|
274
|
+
for (let step = 2; step <= 20; step += 2) move([touch(100 - step, 400), touch(140 + step, 400)]);
|
|
152
275
|
expect(rtcStyle().width).toBeCloseTo(DESKTOP.width * SCALE * 2, 0);
|
|
153
276
|
// The desktop point under the focus (120, 400) stays under it.
|
|
154
277
|
expect(rtcStyle().left).toBeCloseTo(-120, 0);
|
|
@@ -362,3 +485,28 @@ describe('DesktopView.ios hardware keyboard and pointer', () => {
|
|
|
362
485
|
expect(wheels[1].dy).toBeCloseTo(0.01 / SCALE / 120, 5);
|
|
363
486
|
});
|
|
364
487
|
});
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
describe('iOS proof picture bounds', () => {
|
|
491
|
+
it('measures contiguous paper without including the disconnected home indicator', () => {
|
|
492
|
+
for (const density of [1, 3]) {
|
|
493
|
+
const png = { width: 390 * density, height: 844 * density, data: new Uint8Array(390 * 844 * density * density * 4) };
|
|
494
|
+
const paper = (left: number, top: number, right: number, bottom: number) => {
|
|
495
|
+
for (let y = top * density; y < bottom * density; y++) for (let x = left * density; x < right * density; x++) {
|
|
496
|
+
const i = (y * png.width + x) * 4;
|
|
497
|
+
png.data.set([251, 250, 247, 255], i);
|
|
498
|
+
}
|
|
499
|
+
};
|
|
500
|
+
paper(0, 47, 390, 266);
|
|
501
|
+
paper(125, 825, 265, 830);
|
|
502
|
+
expect(pictureBox(png, 390)).toEqual({ top: 47, left: 0, bottom: 266 });
|
|
503
|
+
png.data.fill(0);
|
|
504
|
+
paper(0, 203, 390, 642);
|
|
505
|
+
paper(125, 825, 265, 830);
|
|
506
|
+
expect(pictureBox(png, 390)).toEqual({ top: 203, left: 0, bottom: 642 });
|
|
507
|
+
png.data.fill(0);
|
|
508
|
+
paper(47, 0, 390, 844);
|
|
509
|
+
expect(pictureBox(png, 390)).toEqual({ top: 0, left: 47, bottom: 844 });
|
|
510
|
+
}
|
|
511
|
+
});
|
|
512
|
+
});
|
package/src/DesktopView.ios.tsx
CHANGED
|
@@ -16,6 +16,8 @@ const MIN_WHEEL_STEP = 0.05;
|
|
|
16
16
|
const MAX_WHEEL_STEP = 10;
|
|
17
17
|
/** Desktop pixels one wheel detent stands for. */
|
|
18
18
|
const PIXELS_PER_DETENT = 120;
|
|
19
|
+
/** The closest the picture can be zoomed: view points per desktop pixel. */
|
|
20
|
+
const MAX_SCALE = 2.5;
|
|
19
21
|
/** Where the cursor mark's tip sits inside the mark, in points. */
|
|
20
22
|
const CURSOR_HOTSPOT = 3;
|
|
21
23
|
|
|
@@ -62,13 +64,18 @@ function loadRTCView(): React.ComponentType<Record<string, unknown>> | null {
|
|
|
62
64
|
}
|
|
63
65
|
}
|
|
64
66
|
|
|
65
|
-
|
|
67
|
+
/** An offset that the edge clamp turns into the desktop's top-left corner. */
|
|
68
|
+
const TOP_LEFT: Point = { x: Infinity, y: Infinity };
|
|
69
|
+
|
|
70
|
+
export function DesktopView({ sessionId, style, placeholder, accessibilityLabel, keyboardClearance = 0, insets, gestures = 'desktop' }: DesktopViewProps) {
|
|
66
71
|
const [revision, refresh] = React.useReducer((n: number) => n + 1, 0);
|
|
67
72
|
const [bounds, setBounds] = React.useState({ width: 0, height: 0 });
|
|
68
73
|
const [size, setSize] = React.useState(() => getDesktopSize(sessionId) ?? { width: 0, height: 0 });
|
|
69
74
|
const [keyboardHeight, setKeyboardHeight] = React.useState(0);
|
|
70
|
-
|
|
71
|
-
const [
|
|
75
|
+
/** Zoom over the whole-desktop fit; null fills the view, the default. */
|
|
76
|
+
const [zoom, setZoom] = React.useState<number | null>(null);
|
|
77
|
+
/** From the middle of the uncovered part of the view; clamped to the picture's edges when shown. */
|
|
78
|
+
const [offset, setOffset] = React.useState<Point>(TOP_LEFT);
|
|
72
79
|
const [cursor, setCursor] = React.useState<Point | null>(null);
|
|
73
80
|
const [keyboardText, setKeyboardText] = React.useState('');
|
|
74
81
|
const keyboardTextRef = React.useRef('');
|
|
@@ -123,13 +130,28 @@ export function DesktopView({ sessionId, style, placeholder, accessibilityLabel,
|
|
|
123
130
|
});
|
|
124
131
|
}, [sessionId]);
|
|
125
132
|
|
|
126
|
-
|
|
127
|
-
const
|
|
128
|
-
const
|
|
133
|
+
// The uncovered part of the view: inside what the system covers, and above the keyboard.
|
|
134
|
+
const top = insets?.top ?? 0;
|
|
135
|
+
const left = insets?.left ?? 0;
|
|
136
|
+
const safeWidth = Math.max(1, bounds.width - left - (insets?.right ?? 0));
|
|
137
|
+
const safeHeight = Math.max(1, bounds.height - top - (insets?.bottom ?? 0));
|
|
138
|
+
const visibleBottom = Math.min(top + safeHeight, bounds.height - (keyboardHeight ? keyboardHeight + keyboardClearance : 0));
|
|
139
|
+
const visibleHeight = Math.max(1, visibleBottom - top);
|
|
140
|
+
const fit = size.width && size.height ? Math.min(safeWidth / size.width, visibleHeight / size.height) : 1;
|
|
141
|
+
const defaultScale = bounds.width <= bounds.height
|
|
142
|
+
? safeWidth / size.width
|
|
143
|
+
: Math.max(bounds.width / size.width, (bounds.height - top) / size.height);
|
|
144
|
+
const fill = size.width && size.height ? Math.max(fit, Math.min(MAX_SCALE, defaultScale)) : 1;
|
|
145
|
+
const scale = zoom === null ? fill : fit * zoom;
|
|
146
|
+
/** The zoom over the whole-desktop fit the picture is at now. */
|
|
147
|
+
const zoomed = fit > 0 ? scale / fit : 1;
|
|
148
|
+
// A new desktop size fills the view again, and a new fill (a rotation)
|
|
149
|
+
// starts again, at the desktop's top-left corner, where a page's first words are.
|
|
150
|
+
React.useEffect(() => { setZoom(null); setOffset(TOP_LEFT); }, [size.width, size.height]);
|
|
151
|
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
|
152
|
+
React.useEffect(() => { if (zoom === null) setOffset(TOP_LEFT); }, [fill, bounds.width, bounds.height, top, left, insets?.right, insets?.bottom]);
|
|
129
153
|
const pictureWidth = size.width * scale;
|
|
130
154
|
const pictureHeight = size.height * scale;
|
|
131
|
-
const originX = (bounds.width - pictureWidth) / 2 + offset.x;
|
|
132
|
-
const originY = (visibleHeight - pictureHeight) / 2 + offset.y;
|
|
133
155
|
const point = (x: number, y: number, clamp = false): Point | null => {
|
|
134
156
|
if (!size.width || !size.height || !scale) return null;
|
|
135
157
|
const px = Math.floor((x - originX) / scale);
|
|
@@ -139,10 +161,14 @@ export function DesktopView({ sessionId, style, placeholder, accessibilityLabel,
|
|
|
139
161
|
};
|
|
140
162
|
/** Hold the offset to the picture's edges at a zoom level. */
|
|
141
163
|
const clampOffset = (x: number, y: number, atScale: number = scale): Point => {
|
|
142
|
-
const maxX = Math.max(0, (size.width * atScale -
|
|
164
|
+
const maxX = Math.max(0, (size.width * atScale - safeWidth) / 2);
|
|
143
165
|
const maxY = Math.max(0, (size.height * atScale - visibleHeight) / 2);
|
|
144
166
|
return { x: Math.max(-maxX, Math.min(maxX, x)), y: Math.max(-maxY, Math.min(maxY, y)) };
|
|
145
167
|
};
|
|
168
|
+
// A rotation or the keyboard can leave the offset past the picture's new edges.
|
|
169
|
+
const shown = clampOffset(offset.x, offset.y);
|
|
170
|
+
const originX = left + (safeWidth - pictureWidth) / 2 + shown.x;
|
|
171
|
+
const originY = top + (zoom === null && pictureHeight <= visibleHeight ? 0 : (visibleHeight - pictureHeight) / 2) + shown.y;
|
|
146
172
|
const send = (control: Record<string, unknown>) => { if (sessionId) nativeDesklink.sendControl(sessionId, JSON.stringify(control)); };
|
|
147
173
|
const flushWheel = (force: boolean) => {
|
|
148
174
|
const { x, y } = wheel.current;
|
|
@@ -186,7 +212,7 @@ export function DesktopView({ sessionId, style, placeholder, accessibilityLabel,
|
|
|
186
212
|
// second one, and one finger alone is ignored. A zoomed picture pans
|
|
187
213
|
// from anywhere, clamped to its edges.
|
|
188
214
|
const onPicture = point(at.x, at.y) !== null;
|
|
189
|
-
gesture.current = { stamp, start: at, last: at, time: Date.now(), mode: onPicture ||
|
|
215
|
+
gesture.current = { stamp, start: at, last: at, time: Date.now(), mode: onPicture || zoomed > 1.001 ? 'pending' : 'letterbox', span: 0, focus: at };
|
|
190
216
|
stopTimer();
|
|
191
217
|
longPress.current = setTimeout(() => {
|
|
192
218
|
if (gesture.current?.mode === 'pending' && point(at.x, at.y)) gesture.current.mode = 'armed';
|
|
@@ -223,16 +249,16 @@ export function DesktopView({ sessionId, style, placeholder, accessibilityLabel,
|
|
|
223
249
|
}
|
|
224
250
|
}
|
|
225
251
|
if (g.mode === 'pinch' && g.span) {
|
|
226
|
-
const maxZoom = fit > 0 ?
|
|
227
|
-
const next = Math.max(1, Math.min(maxZoom,
|
|
228
|
-
const applied =
|
|
252
|
+
const maxZoom = fit > 0 ? Math.max(1, MAX_SCALE / fit) : MAX_SCALE;
|
|
253
|
+
const next = Math.max(1, Math.min(maxZoom, zoomed * distance / g.span));
|
|
254
|
+
const applied = zoomed > 0 ? next / zoomed : 1;
|
|
229
255
|
// The desktop point under the focus stays under it while the
|
|
230
256
|
// offset follows the fingers, then everything clamps back.
|
|
231
257
|
const nextScale = fit * next;
|
|
232
|
-
const ox =
|
|
233
|
-
const oy =
|
|
234
|
-
const baseX = (
|
|
235
|
-
const baseY = (visibleHeight - size.height * nextScale) / 2;
|
|
258
|
+
const ox = originX;
|
|
259
|
+
const oy = originY;
|
|
260
|
+
const baseX = left + (safeWidth - size.width * nextScale) / 2;
|
|
261
|
+
const baseY = top + (visibleHeight - size.height * nextScale) / 2;
|
|
236
262
|
setZoom(next);
|
|
237
263
|
setOffset(clampOffset(
|
|
238
264
|
center.x - (center.x - ox) * applied - baseX + (center.x - g.focus.x),
|
|
@@ -245,7 +271,10 @@ export function DesktopView({ sessionId, style, placeholder, accessibilityLabel,
|
|
|
245
271
|
wheel.current.y += -(center.y - g.focus.y) / scale / PIXELS_PER_DETENT;
|
|
246
272
|
flushWheel(false);
|
|
247
273
|
}
|
|
248
|
-
|
|
274
|
+
// Undecided, the spread is measured from where the fingers landed:
|
|
275
|
+
// a slow pinch crosses the slop over many small moves.
|
|
276
|
+
if (g.mode !== 'two') g.span = distance;
|
|
277
|
+
g.focus = center; g.last = center;
|
|
249
278
|
return;
|
|
250
279
|
}
|
|
251
280
|
if (touches.length !== 1 || g.mode === 'spent' || g.mode === 'letterbox' || g.mode === 'two' || g.mode === 'pinch' || (g.mode === 'scroll' && gestures !== 'browser')) return;
|
|
@@ -258,7 +287,7 @@ export function DesktopView({ sessionId, style, placeholder, accessibilityLabel,
|
|
|
258
287
|
g.mode = 'scroll';
|
|
259
288
|
const under = point(at.x, at.y);
|
|
260
289
|
if (under) pointer('move', under);
|
|
261
|
-
} else g.mode = zoom > 1 ? 'pan' : 'hover';
|
|
290
|
+
} else g.mode = zoom === null || zoomed > 1.001 ? 'pan' : 'hover';
|
|
262
291
|
}
|
|
263
292
|
if (g.mode === 'scroll') {
|
|
264
293
|
// Content follows the finger: moving it up scrolls the page down.
|
|
@@ -271,7 +300,7 @@ export function DesktopView({ sessionId, style, placeholder, accessibilityLabel,
|
|
|
271
300
|
if (g.mode === 'pan') {
|
|
272
301
|
const dx = at.x - g.last.x;
|
|
273
302
|
const dy = at.y - g.last.y;
|
|
274
|
-
setOffset((current) => clampOffset(current.x + dx,
|
|
303
|
+
setOffset((current) => { const from = clampOffset(current.x, current.y); return clampOffset(from.x + dx, from.y + dy); });
|
|
275
304
|
}
|
|
276
305
|
if (g.mode === 'armed' && Math.hypot(at.x - g.start.x, at.y - g.start.y) > 8) {
|
|
277
306
|
const from = point(g.start.x, g.start.y, true);
|
package/src/DesktopView.tsx
CHANGED
|
@@ -5,6 +5,9 @@ import { StyleSheet, View, type StyleProp, type ViewStyle } from 'react-native';
|
|
|
5
5
|
import { desktopAvailable } from './native';
|
|
6
6
|
import type { GestureProfile } from './protocol';
|
|
7
7
|
|
|
8
|
+
/** Room at each edge of the view, in points; a missing edge is 0. */
|
|
9
|
+
export type EdgeInsets = { top?: number; left?: number; bottom?: number; right?: number };
|
|
10
|
+
|
|
8
11
|
export interface DesktopViewProps {
|
|
9
12
|
/** The session handle from `useDesktopSession`. */
|
|
10
13
|
sessionId: string | null;
|
|
@@ -19,6 +22,14 @@ export interface DesktopViewProps {
|
|
|
19
22
|
* following the keyboard as it moves.
|
|
20
23
|
*/
|
|
21
24
|
keyboardClearance?: number;
|
|
25
|
+
/**
|
|
26
|
+
* What the system covers at each edge of the view — status and navigation
|
|
27
|
+
* bars, a camera cutout, the home indicator — in points, as a safe-area
|
|
28
|
+
* library reports it. The picture still reaches under these edges, but it
|
|
29
|
+
* starts and pans inside them: its top-left corner first shows just inside,
|
|
30
|
+
* and every edge of the desktop can be brought into the uncovered part.
|
|
31
|
+
*/
|
|
32
|
+
insets?: EdgeInsets;
|
|
22
33
|
/**
|
|
23
34
|
* Which touch meaning the surface uses: a desktop pointer, a browser page
|
|
24
35
|
* (one finger scrolls), or a device screen (one finger presses and drags).
|
|
@@ -32,6 +43,8 @@ interface NativeSurfaceProps {
|
|
|
32
43
|
accessible?: boolean;
|
|
33
44
|
accessibilityLabel?: string;
|
|
34
45
|
keyboardClearance?: number;
|
|
46
|
+
/** `insets` as top, left, bottom, right. */
|
|
47
|
+
insets?: number[];
|
|
35
48
|
gestures?: GestureProfile;
|
|
36
49
|
}
|
|
37
50
|
|
|
@@ -49,7 +62,7 @@ const NativeSurface: React.ComponentType<NativeSurfaceProps> | null = desktopAva
|
|
|
49
62
|
* belong to the application, which mounts this wherever it wants the desktop
|
|
50
63
|
* to appear.
|
|
51
64
|
*/
|
|
52
|
-
export function DesktopView({ sessionId, style, placeholder, accessibilityLabel, keyboardClearance = 0, gestures = 'desktop' }: DesktopViewProps) {
|
|
65
|
+
export function DesktopView({ sessionId, style, placeholder, accessibilityLabel, keyboardClearance = 0, insets, gestures = 'desktop' }: DesktopViewProps) {
|
|
53
66
|
if (NativeSurface == null || sessionId == null) {
|
|
54
67
|
return (
|
|
55
68
|
<View style={[styles.surface, style]}>
|
|
@@ -57,7 +70,8 @@ export function DesktopView({ sessionId, style, placeholder, accessibilityLabel,
|
|
|
57
70
|
</View>
|
|
58
71
|
);
|
|
59
72
|
}
|
|
60
|
-
return <NativeSurface style={[styles.surface, style]} sessionId={sessionId} accessible={accessibilityLabel !== undefined} accessibilityLabel={accessibilityLabel} keyboardClearance={keyboardClearance}
|
|
73
|
+
return <NativeSurface style={[styles.surface, style]} sessionId={sessionId} accessible={accessibilityLabel !== undefined} accessibilityLabel={accessibilityLabel} keyboardClearance={keyboardClearance}
|
|
74
|
+
insets={[insets?.top ?? 0, insets?.left ?? 0, insets?.bottom ?? 0, insets?.right ?? 0]} gestures={gestures} />;
|
|
61
75
|
}
|
|
62
76
|
|
|
63
77
|
const styles = StyleSheet.create({
|
package/src/DesktopView.web.tsx
CHANGED
|
@@ -3,6 +3,9 @@ import { StyleSheet, View, type StyleProp, type ViewStyle } from 'react-native';
|
|
|
3
3
|
|
|
4
4
|
import type { GestureProfile } from './protocol';
|
|
5
5
|
|
|
6
|
+
/** Room at each edge of the view, in points; a missing edge is 0. */
|
|
7
|
+
export type EdgeInsets = { top?: number; left?: number; bottom?: number; right?: number };
|
|
8
|
+
|
|
6
9
|
export interface DesktopViewProps {
|
|
7
10
|
/** The session handle from `useDesktopSession`. */
|
|
8
11
|
sessionId: string | null;
|
|
@@ -16,6 +19,15 @@ export interface DesktopViewProps {
|
|
|
16
19
|
* controls, in points. While the keyboard is up the picture sits above both.
|
|
17
20
|
*/
|
|
18
21
|
keyboardClearance?: number;
|
|
22
|
+
/**
|
|
23
|
+
* What the system covers at each edge of the view — status and navigation
|
|
24
|
+
* bars, a camera cutout, the home indicator — in points, as a safe-area
|
|
25
|
+
* library reports it. The picture still reaches under these edges, but it
|
|
26
|
+
* starts and pans inside them: its top-left corner first shows just inside,
|
|
27
|
+
* and every edge of the desktop can be brought into the uncovered part.
|
|
28
|
+
*/
|
|
29
|
+
/** The browser surface fits its own box and ignores this. */
|
|
30
|
+
insets?: EdgeInsets;
|
|
19
31
|
/**
|
|
20
32
|
* Which touch meaning the surface uses: a desktop pointer, a browser page
|
|
21
33
|
* (one finger scrolls), or a device screen (one finger presses and drags).
|
package/src/index.ts
CHANGED
|
@@ -10,7 +10,7 @@ export * from './protocol';
|
|
|
10
10
|
*/
|
|
11
11
|
export type { NativeDesklinkModule, NativeSessionEvent } from './native';
|
|
12
12
|
export { desktopAvailable } from './availability';
|
|
13
|
-
export { DesktopView, type DesktopViewProps } from './DesktopView';
|
|
13
|
+
export { DesktopView, type DesktopViewProps, type EdgeInsets } from './DesktopView';
|
|
14
14
|
export { observeWebKeyboardMotion } from './webKeyboardMotion';
|
|
15
15
|
export type { StickyModifier, StickyModifiers, StickyState } from './stickyModifiers';
|
|
16
16
|
export {
|
package/src/native.ios.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { MediaStream, RTCPeerConnection } from 'react-native-webrtc';
|
|
2
2
|
import type { NativeDesklinkModule, NativeSessionEvent } from './native';
|
|
3
3
|
import { requireWebRTC } from './webrtc.ios';
|
|
4
|
+
import { videoStats } from './videoStats';
|
|
4
5
|
|
|
5
6
|
/** The WebRTC stack, loaded the first time a session opens rather than with this module. */
|
|
6
7
|
function loadWebRTC(): { RTCPeerConnection: new (config: unknown) => RTCPeerConnection; MediaStream: new () => MediaStream } | null {
|
|
@@ -99,6 +100,10 @@ function failure(id: string, error: unknown) {
|
|
|
99
100
|
}
|
|
100
101
|
|
|
101
102
|
export const nativeDesklink: NativeDesklinkModule = {
|
|
103
|
+
async getStats(id) {
|
|
104
|
+
const session = sessions.get(id);
|
|
105
|
+
return session ? videoStats(await session.peer.getStats()) : '[]';
|
|
106
|
+
},
|
|
102
107
|
createSession(iceServersJson) {
|
|
103
108
|
const id = `desklink-${++nextId}`;
|
|
104
109
|
try {
|
package/src/native.ts
CHANGED
|
@@ -45,6 +45,8 @@ export interface NativeDesklinkModule {
|
|
|
45
45
|
/** Hold the screen in landscape while the desktop is shown, or follow the phone again. */
|
|
46
46
|
setOrientation(mode: 'landscape' | 'auto'): boolean;
|
|
47
47
|
closeSession(id: string): boolean;
|
|
48
|
+
/** Video reception and codec statistics, excluding network addresses and credentials. */
|
|
49
|
+
getStats?(id: string): Promise<string>;
|
|
48
50
|
isAvailable(): boolean;
|
|
49
51
|
addListener?(name: 'onSessionEvent', handler: (event: NativeSessionEvent) => void): { remove: () => void };
|
|
50
52
|
}
|
|
@@ -57,4 +59,3 @@ export const nativeDesklink: NativeDesklinkModule | null = native;
|
|
|
57
59
|
|
|
58
60
|
/** True when this build can show a live desktop surface at all. */
|
|
59
61
|
export const desktopAvailable = native !== null;
|
|
60
|
-
|
package/src/native.web.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { NativeDesklinkModule, NativeEventName, NativeSessionEvent } from './native';
|
|
2
|
+
import { videoStats } from './videoStats';
|
|
2
3
|
import type { GestureProfile } from './protocol';
|
|
3
4
|
import { observeWebKeyboardMotion } from './webKeyboardMotion';
|
|
4
5
|
|
|
@@ -898,6 +899,10 @@ const sequence = new Map<string, number>();
|
|
|
898
899
|
export const desktopAvailable = true;
|
|
899
900
|
|
|
900
901
|
export const nativeDesklink: NativeDesklinkModule = {
|
|
902
|
+
async getStats(id) {
|
|
903
|
+
const session = sessions.get(id);
|
|
904
|
+
return session ? videoStats(await session.peer.getStats()) : '[]';
|
|
905
|
+
},
|
|
901
906
|
isAvailable: () => true,
|
|
902
907
|
|
|
903
908
|
createSession(iceServersJson: string): string | null {
|
package/src/protocol.ts
CHANGED
|
@@ -16,6 +16,11 @@
|
|
|
16
16
|
|
|
17
17
|
export const PROTOCOL_VERSION = 3;
|
|
18
18
|
|
|
19
|
+
/**
|
|
20
|
+
* @deprecated Pairing is the only trust boundary: a paired session always has
|
|
21
|
+
* view, input and clipboard, so there is nothing to ask for. Kept so 0.3 code
|
|
22
|
+
* compiles; removed at 1.0.
|
|
23
|
+
*/
|
|
19
24
|
export type Permission = 'view' | 'control' | 'clipboard';
|
|
20
25
|
|
|
21
26
|
/**
|
|
@@ -42,7 +47,8 @@ export interface SurfaceGeometry {
|
|
|
42
47
|
|
|
43
48
|
export interface SessionOpenRequest {
|
|
44
49
|
cursor?: 'embedded' | 'hidden';
|
|
45
|
-
|
|
50
|
+
/** @deprecated Ignored: a paired session always has view, input and clipboard. */
|
|
51
|
+
permissions?: Permission[];
|
|
46
52
|
maxWidth?: number;
|
|
47
53
|
maxHeight?: number;
|
|
48
54
|
bitrateKbps?: number;
|
|
@@ -127,6 +133,7 @@ export type SessionStatus =
|
|
|
127
133
|
| 'failed';
|
|
128
134
|
|
|
129
135
|
export type SessionFailure =
|
|
136
|
+
/** The app's `authorize()` refused, or the host did: not paired, or the pairing was revoked. */
|
|
130
137
|
| { code: 'permission'; message: string }
|
|
131
138
|
/** Nobody approved the desktop's own screen-sharing prompt in time. */
|
|
132
139
|
| { code: 'consent'; message: string }
|
|
@@ -146,6 +153,12 @@ export interface SessionSnapshot {
|
|
|
146
153
|
geometry: SurfaceGeometry | null;
|
|
147
154
|
/** True once an authorized, current-generation frame has actually rendered. */
|
|
148
155
|
presented: boolean;
|
|
156
|
+
/**
|
|
157
|
+
* `message` is free text from the app's `authorize()`, the host or the
|
|
158
|
+
* engine and may contain a bridge URL or pairing token. Never display it
|
|
159
|
+
* to people; redact sensitive data before logging it. Choose user-facing
|
|
160
|
+
* text from the session state and `code` instead.
|
|
161
|
+
*/
|
|
149
162
|
failure: SessionFailure | null;
|
|
150
163
|
/** Redacted diagnostics: codec, route, counters. Never identities. */
|
|
151
164
|
diagnostics: Record<string, string | number | boolean>;
|
package/src/useDesktopSession.ts
CHANGED
|
@@ -172,10 +172,10 @@ async function loadPlatform(): Promise<boolean> {
|
|
|
172
172
|
|
|
173
173
|
export interface DesktopSessionOptions {
|
|
174
174
|
/**
|
|
175
|
-
*
|
|
176
|
-
*
|
|
177
|
-
*
|
|
178
|
-
*
|
|
175
|
+
* The app's pairing check: return the authenticated channel to the host
|
|
176
|
+
* engine of a paired computer, or throw. Pairing is the only trust
|
|
177
|
+
* boundary, so a paired session has view, input and clipboard. Called again
|
|
178
|
+
* on every reconnect, so a revoked pairing cannot reopen.
|
|
179
179
|
*/
|
|
180
180
|
authorize: () => Promise<{ signaling: Signaling; session: SessionOpenRequest }>;
|
|
181
181
|
onStateChange?: (snapshot: SessionSnapshot) => void;
|
|
@@ -191,6 +191,8 @@ export interface DesktopSessionOptions {
|
|
|
191
191
|
|
|
192
192
|
export interface DesktopSession {
|
|
193
193
|
snapshot: SessionSnapshot;
|
|
194
|
+
/** Video reception and codec diagnostics, without network addresses. */
|
|
195
|
+
getStats: () => Promise<Array<Record<string, unknown>>>;
|
|
194
196
|
/** The native session handle, or null when nothing is open. */
|
|
195
197
|
nativeId: string | null;
|
|
196
198
|
connect: () => Promise<void>;
|
|
@@ -556,6 +558,12 @@ export function useDesktopSession(options: DesktopSessionOptions): DesktopSessio
|
|
|
556
558
|
if (id != null) nativeDesklink?.hideKeyboard(id);
|
|
557
559
|
}, []);
|
|
558
560
|
|
|
561
|
+
const getStats = useCallback(async (): Promise<Array<Record<string, unknown>>> => {
|
|
562
|
+
const id = nativeRef.current;
|
|
563
|
+
if (id == null || !nativeDesklink?.getStats) return [];
|
|
564
|
+
return JSON.parse(await nativeDesklink.getStats(id));
|
|
565
|
+
}, []);
|
|
566
|
+
|
|
559
567
|
const fitToView = useCallback(() => {
|
|
560
568
|
const id = nativeRef.current;
|
|
561
569
|
if (id != null) nativeDesklink?.fitToView(id);
|
|
@@ -717,7 +725,10 @@ export function useDesktopSession(options: DesktopSessionOptions): DesktopSessio
|
|
|
717
725
|
let openedResult: SessionOpenResult;
|
|
718
726
|
try {
|
|
719
727
|
openedResult = await authorization.signaling.request<SessionOpenResult>('session.open', {
|
|
720
|
-
|
|
728
|
+
// Ignored by 0.4+ engines, whatever the app asked for. A 0.3
|
|
729
|
+
// engine refuses an open without `view`, so the full set stays
|
|
730
|
+
// on the wire until 0.3 hosts leave support.
|
|
731
|
+
permissions: ['view', 'control', 'clipboard'],
|
|
721
732
|
max_width: authorization.session.maxWidth,
|
|
722
733
|
...(authorization.session.cursor === undefined ? {} : { cursor: authorization.session.cursor }),
|
|
723
734
|
max_height: authorization.session.maxHeight,
|
|
@@ -1069,6 +1080,7 @@ export function useDesktopSession(options: DesktopSessionOptions): DesktopSessio
|
|
|
1069
1080
|
releaseHeld,
|
|
1070
1081
|
setInputEnabled,
|
|
1071
1082
|
fitToView,
|
|
1083
|
+
getStats,
|
|
1072
1084
|
setOrientation,
|
|
1073
1085
|
send,
|
|
1074
1086
|
modifiers,
|
|
@@ -1086,6 +1098,7 @@ export function useDesktopSession(options: DesktopSessionOptions): DesktopSessio
|
|
|
1086
1098
|
releaseHeld,
|
|
1087
1099
|
setInputEnabled,
|
|
1088
1100
|
fitToView,
|
|
1101
|
+
getStats,
|
|
1089
1102
|
setOrientation,
|
|
1090
1103
|
send,
|
|
1091
1104
|
modifiers,
|
|
@@ -1106,5 +1119,5 @@ function randomId(): string {
|
|
|
1106
1119
|
return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
|
|
1107
1120
|
}
|
|
1108
1121
|
|
|
1109
|
-
/**
|
|
1122
|
+
/** @deprecated Ignored: a paired session always has view, input and clipboard. Removed at 1.0. */
|
|
1110
1123
|
export const CONTROL_PERMISSIONS: Permission[] = ['view', 'control', 'clipboard'];
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/** Keep receiver diagnostics useful without disclosing ICE addresses or credentials. */
|
|
2
|
+
export function videoStats(report: { forEach(callback: (stat: Record<string, unknown>) => void): void }): string {
|
|
3
|
+
const fields = new Set([
|
|
4
|
+
'id', 'type', 'timestamp', 'kind', 'mediaType', 'codecId', 'mimeType', 'clockRate', 'payloadType',
|
|
5
|
+
'packetsReceived', 'packetsLost', 'bytesReceived', 'framesReceived', 'framesDecoded',
|
|
6
|
+
'keyFramesDecoded', 'framesDropped', 'framesPerSecond', 'frameWidth', 'frameHeight',
|
|
7
|
+
'nackCount', 'pliCount', 'firCount', 'totalDecodeTime', 'decoderImplementation', 'jitterBufferDelay',
|
|
8
|
+
]);
|
|
9
|
+
const stats: Array<Record<string, unknown>> = [];
|
|
10
|
+
report.forEach((stat) => {
|
|
11
|
+
if (stat.type === 'inbound-rtp' || stat.type === 'codec') {
|
|
12
|
+
stats.push(Object.fromEntries(Object.entries(stat).filter(([key]) => fields.has(key))));
|
|
13
|
+
}
|
|
14
|
+
});
|
|
15
|
+
return JSON.stringify(stats);
|
|
16
|
+
}
|