@displayxr/inline3d 1.47.6 → 1.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +136 -1
- package/call.d.ts +202 -12
- package/camera.d.ts +44 -7
- package/dist/call.js +11 -6
- package/docs/woven-canvas-rules.md +25 -13
- package/js/call/element.js +9 -3
- package/js/call/options.js +22 -3
- package/js/call/share.js +147 -0
- package/js/call/transport.js +160 -9
- package/js/call/ui.js +11 -1
- package/js/call/wire.js +227 -6
- package/js/camera/app-share.js +54 -0
- package/js/camera/capture.js +196 -32
- package/js/camera/conv-meter.js +152 -0
- package/js/inline3d-call.js +737 -91
- package/js/inline3d-camera.js +34 -7
- package/js/inline3d-player.js +8 -4
- package/package.json +8 -1
- package/signaling/README.md +289 -0
- package/signaling/dev-server.mjs +320 -0
- package/signaling/gate.mjs +112 -0
- package/signaling/keys.mjs +123 -0
- package/signaling/limits.mjs +221 -0
- package/signaling/room.mjs +322 -0
- package/signaling/turn-budget.mjs +254 -0
package/js/inline3d-call.js
CHANGED
|
@@ -76,6 +76,14 @@
|
|
|
76
76
|
// visibility (paused), the depth slider drives lift's convergence, and at most `maxLifted`
|
|
77
77
|
// streams run at once.
|
|
78
78
|
//
|
|
79
|
+
// CONTENT (3D app share, docs/call.md § Sharing a 3D app): a participant may share a second
|
|
80
|
+
// video track — a running DisplayXR app's side-by-side views (`openAppShare()`), or any SBS
|
|
81
|
+
// track — with `shareContent()`. It rides its own transceiver (call/transport.js, "THE CONTENT
|
|
82
|
+
// TRACK") and is announced by a `content` message; each receiver shows it in a ContentTile: the
|
|
83
|
+
// remote tile's woven path with `kind: 'content'`, but DISPLAY-REFERRED — the frames already put
|
|
84
|
+
// the sender's display plane at a known disparity (`convergence.frac`, 0 for an app in v1), so
|
|
85
|
+
// that is applied as the shift, nothing is measured, and only the depth slider adds to it.
|
|
86
|
+
//
|
|
79
87
|
// SESSION LOSS: a call must outlive a service restart. A layer that fails to build is retried
|
|
80
88
|
// with backoff (flat in between); a session that ENDS without the page closing the wall is
|
|
81
89
|
// replaced with a fresh `createInline3D()` and every tile re-registers on it — no reload. To tell
|
|
@@ -85,7 +93,7 @@
|
|
|
85
93
|
|
|
86
94
|
import { sharedInline3D } from './inline3d.js';
|
|
87
95
|
import * as wire from './call/wire.js';
|
|
88
|
-
import { DEFAULT_MAX_PEERS, HINT_MAX_HZ, newPeerId, parseInviteLink, normalizeHint, normalizeState, rateGate, createConvergence, eyeOutputSize, backoffMs } from './call/wire.js';
|
|
96
|
+
import { DEFAULT_MAX_PEERS, HINT_MAX_HZ, CONV_MAX_HZ, CONV_RECV_MAX_HZ, CONV_HEARTBEAT_MS, CONV_FRESH_MS, APP_MAX_PER_SEC, APP_RECV_MAX_PER_SEC, newPeerId, parseInviteLink, normalizeHint, normalizeState, normalizeConv, makeConv, makeAppMessage, normalizeApp, windowGate, selectedPath, sanitizeName, rateGate, createConvergence, eyeOutputSize, backoffMs } from './call/wire.js';
|
|
89
97
|
import { MeshTransport as MeshTransportImpl, clampMaxPeers as clampMaxPeersImpl } from './call/transport.js';
|
|
90
98
|
import { openMic } from './camera/capture.js';
|
|
91
99
|
import { openCamera, addCameraView, isStereoCamera } from './inline3d-camera.js';
|
|
@@ -98,13 +106,17 @@ import * as signalingMod from './call/signaling.js';
|
|
|
98
106
|
import { injectCallStyle, ICONS, el, show, CALL_ACCENTS as CALL_ACCENTS_IMPL, createStrings, CALL_STRINGS } from './call/ui.js';
|
|
99
107
|
import { normalizeCallOptions as normalizeOptions } from './call/options.js';
|
|
100
108
|
import { defineCallElement } from './call/element.js';
|
|
101
|
-
import {
|
|
109
|
+
import { createDisparityTrack, createFocusTracker } from './call/disparity.js';
|
|
110
|
+
import { grabLumaFrame, createConvergenceMeter } from './camera/conv-meter.js';
|
|
102
111
|
import { dxrSignaling } from './call/signaling.js';
|
|
112
|
+
import { describeAppShare, isAppShareDevice } from './call/share.js';
|
|
103
113
|
|
|
104
114
|
// ── the public surface (RFC 0003 §2) ────────────────────────────────────────────────────────
|
|
105
115
|
export { DxrCallElement, defineCallElement, attrsToOpts, CALL_EVENT_PREFIX } from './call/element.js';
|
|
106
116
|
export { dxrSignaling, DXR_SIGNAL_DEFAULT } from './call/signaling.js';
|
|
107
117
|
export { parseInviteLink };
|
|
118
|
+
// 3D screen share (runtime ADR-061): find and open a running DisplayXR app as content.
|
|
119
|
+
export { isAppShareDevice, listShareableApps, watchShareableApps, openAppShare } from './call/share.js';
|
|
108
120
|
|
|
109
121
|
const TAG = '[inline3d/call]';
|
|
110
122
|
const HELLO_WAIT_MS = 2000;
|
|
@@ -119,8 +131,6 @@ const AUTO_CONV_INTERVAL_MS = 200;
|
|
|
119
131
|
const AUTO_CONV_EYE_WIDTH = 240;
|
|
120
132
|
// While a depth cursor hovers a stereo tile, its source is copied this often (the cursor measures it).
|
|
121
133
|
const CURSOR_SAMPLE_MS = 66;
|
|
122
|
-
// WebCodecs pixel formats whose plane 0 is luma (Y): read directly, no colour conversion.
|
|
123
|
-
const Y_PLANE_FORMATS = new Set(['I420', 'I420A', 'I422', 'I444', 'NV12']);
|
|
124
134
|
/**
|
|
125
135
|
* Put a 3D video call in `container`. Resolves once the camera is open and (with `ui:true`) the
|
|
126
136
|
* lobby is showing; with `ui:false` (or `autoJoin:true`) once the call is joined.
|
|
@@ -170,6 +180,13 @@ class Call {
|
|
|
170
180
|
this.room = o.room === 'auto' ? parseInviteLink(globalThis.location) : o.room;
|
|
171
181
|
this.state = 'idle'; // idle | lobby | joining | in-call | full | left
|
|
172
182
|
this.tiles = new Map();
|
|
183
|
+
// Content tiles (3D app share): one per sharing participant, keyed by its id; `'self'` is the
|
|
184
|
+
// optional preview of what this side shares (`selfContentView`). `contentStreams`: each peer's
|
|
185
|
+
// content track as it arrived (it can come before or after its `content` message).
|
|
186
|
+
this.contentTiles = new Map();
|
|
187
|
+
this.contentStreams = new Map();
|
|
188
|
+
this.content = null; // what THIS side shares: { track, stream, desc, off }
|
|
189
|
+
this._contentWarned = new Set();
|
|
173
190
|
this.listeners = new Map();
|
|
174
191
|
this.local = null; // { stream, video, audio, format, width, height, calibration, owned, label }
|
|
175
192
|
this.muted = false;
|
|
@@ -177,9 +194,25 @@ class Call {
|
|
|
177
194
|
this.depth = 0;
|
|
178
195
|
this.speakerId = null;
|
|
179
196
|
this.transport = null;
|
|
197
|
+
// Headless hooks (docs/call.md § Headless mode): this side's name, the signalling state, the
|
|
198
|
+
// self view switch, the audio output every tile plays to, and the app channel's per-peer gates.
|
|
199
|
+
this.name = o.name;
|
|
200
|
+
this.signalState = null;
|
|
201
|
+
this.selfViewOn = o.selfView;
|
|
202
|
+
this.sinkId = null;
|
|
203
|
+
this._appOut = new Map();
|
|
204
|
+
this._appIn = new Map();
|
|
180
205
|
this._raf = 0;
|
|
181
206
|
this._statsTimer = 0;
|
|
182
207
|
this._hintGate = rateGate(HINT_MAX_HZ);
|
|
208
|
+
// Sender-side auto-convergence (docs/call.md): this side measures its OWN raw camera frames
|
|
209
|
+
// (before encoding) and broadcasts `conv`, so every receiver applies one good measurement
|
|
210
|
+
// instead of each measuring a compressed, possibly downscaled copy. The timer paces it.
|
|
211
|
+
this._convGate = rateGate(CONV_MAX_HZ, 20); // paced by a 5 Hz timer: 20 ms of slack for its jitter (rateGate)
|
|
212
|
+
this._convMeter = createConvergenceMeter({ intervalMs: 0, grab: o._convGrab });
|
|
213
|
+
this._convOut = { frac: null, sentAt: 0, msg: null };
|
|
214
|
+
this._convGen = 0;
|
|
215
|
+
this._convTimer = 0;
|
|
183
216
|
// mono→3D (call/lift.js): the option, whether it is on right now (setMono3D), the resolved
|
|
184
217
|
// lift (null until resolved), the pool of lifted streams, and the web-provider frame watch.
|
|
185
218
|
this.mono3D = o.mono3D;
|
|
@@ -256,7 +289,7 @@ class Call {
|
|
|
256
289
|
let camError = null;
|
|
257
290
|
const given = isStereoCamera(want) ? want : null;
|
|
258
291
|
try {
|
|
259
|
-
cam = given || (await openCamera({ prefer: want, format: format || this.o.format, calibration: this.o.calibration, rectify: this.o.rectify || undefined, log }));
|
|
292
|
+
cam = given || (await openCamera({ prefer: want, format: format || this.o.format, calibration: this.o.calibration, rectify: this.o.rectify || undefined, exclude: this.o.cameraExclude, log }));
|
|
260
293
|
if (given && given.state !== 'live') throw Object.assign(new Error('the camera given is not live'), { code: 'no-camera', skipped: [] });
|
|
261
294
|
} catch (err) {
|
|
262
295
|
camError = err;
|
|
@@ -330,6 +363,9 @@ class Call {
|
|
|
330
363
|
baselineMm: l.calibration?.baselineMm,
|
|
331
364
|
hfovDeg: l.calibration?.hfovDeg,
|
|
332
365
|
rectified: !!l.calibration?.rectified,
|
|
366
|
+
conv: this.o.sendConvergence && l.format === 'sbs',
|
|
367
|
+
name: this.name,
|
|
368
|
+
content: true, // this SDK carries the content track (3D app share): see call/transport.js
|
|
333
369
|
});
|
|
334
370
|
}
|
|
335
371
|
|
|
@@ -370,9 +406,12 @@ class Call {
|
|
|
370
406
|
room: this.room,
|
|
371
407
|
localStream: this.local.stream,
|
|
372
408
|
sendFormat: this.local.format,
|
|
409
|
+
contentTrack: this.content ? this.content.track : null,
|
|
410
|
+
contentStream: this.content ? this.content.stream : null,
|
|
373
411
|
onPeer: (id) => this._onPeer(id),
|
|
374
412
|
onPeerLeft: (id, reason) => this._onPeerLeft(id, reason),
|
|
375
413
|
onStream: (id, stream) => this.tiles.get(id)?.setStream(stream),
|
|
414
|
+
onContentStream: (id, stream) => this._onContentStream(id, stream),
|
|
376
415
|
onMessage: (id, msg) => this._onMessage(id, msg),
|
|
377
416
|
onPeerState: (id, st) => {
|
|
378
417
|
this.tiles.get(id)?.setConn(st);
|
|
@@ -382,6 +421,7 @@ class Call {
|
|
|
382
421
|
onGhost: (gid, on) => this._onGhost(gid, on),
|
|
383
422
|
onSignalingState: (st, err) => {
|
|
384
423
|
this.log('signaling', { state: st });
|
|
424
|
+
this._setSignal(st, err);
|
|
385
425
|
if (st === 'closed' && err) this.error(err.code || 'signaling-closed', err.message, err);
|
|
386
426
|
},
|
|
387
427
|
});
|
|
@@ -397,6 +437,7 @@ class Call {
|
|
|
397
437
|
throw err;
|
|
398
438
|
}
|
|
399
439
|
this._setState('in-call');
|
|
440
|
+
this._setSignal('connected');
|
|
400
441
|
// The grid is sized so the whole call block fits ONE viewport, but only if it starts at the
|
|
401
442
|
// top: a tile below the fold is withheld from the weave (browser#167), so its layer goes
|
|
402
443
|
// live late. Bring the block into view once, on join.
|
|
@@ -409,6 +450,7 @@ class Call {
|
|
|
409
450
|
}
|
|
410
451
|
}
|
|
411
452
|
this._startStats();
|
|
453
|
+
this._startConvSend();
|
|
412
454
|
this._pagehide = () => this.leave('pagehide');
|
|
413
455
|
globalThis.addEventListener?.('pagehide', this._pagehide);
|
|
414
456
|
this.log('joined', { id: this.id, peers: this.tiles.size });
|
|
@@ -476,6 +518,8 @@ class Call {
|
|
|
476
518
|
_onPeerLeft(id, reason) {
|
|
477
519
|
const tile = this.tiles.get(id);
|
|
478
520
|
if (!tile) return;
|
|
521
|
+
this._dropRemoteContent(id);
|
|
522
|
+
this.contentStreams.delete(id);
|
|
479
523
|
tile.setConn('left');
|
|
480
524
|
this.emit('peerleft', { id, reason });
|
|
481
525
|
tile.leaveTimer = setTimeout(() => {
|
|
@@ -492,15 +536,351 @@ class Call {
|
|
|
492
536
|
if (msg.type === '__open') {
|
|
493
537
|
this.transport.send(id, this.hello());
|
|
494
538
|
this.transport.send(id, this.stateMsg());
|
|
539
|
+
// A peer that joins mid-call gets the current convergence at once, not at the next heartbeat.
|
|
540
|
+
if (this._convOut.msg && this._convSending()) this.transport.send(id, this._convOut.msg);
|
|
541
|
+
// ...and what this side is sharing, if anything (a late joiner, a rebuilt connection).
|
|
542
|
+
if (this.content) this.transport.send(id, wire.makeContent(this.content.desc));
|
|
495
543
|
tile.onChannelOpen();
|
|
496
544
|
return;
|
|
497
545
|
}
|
|
498
546
|
const h = wire.normalizeHello(msg);
|
|
499
|
-
if (h)
|
|
547
|
+
if (h) {
|
|
548
|
+
tile.setHello(h);
|
|
549
|
+
this._onPeerHello(id, h);
|
|
550
|
+
return;
|
|
551
|
+
}
|
|
552
|
+
const cm = wire.normalizeContent(msg);
|
|
553
|
+
if (cm) return this._onRemoteContent(id, cm);
|
|
500
554
|
const hint = normalizeHint(msg);
|
|
501
555
|
if (hint) return tile.setHint(hint);
|
|
502
556
|
const st = normalizeState(msg);
|
|
503
557
|
if (st) return tile.setRemoteState(st);
|
|
558
|
+
const cv = normalizeConv(msg);
|
|
559
|
+
if (cv) return tile.setConv(cv);
|
|
560
|
+
// An app message (handle.send / broadcast). The ONLY type that reaches the page: everything
|
|
561
|
+
// above is the SDK's own and never surfaces. Rate-capped per peer on receipt too.
|
|
562
|
+
const app = normalizeApp(msg);
|
|
563
|
+
if (app) {
|
|
564
|
+
let gate = this._appIn.get(id);
|
|
565
|
+
if (!gate) this._appIn.set(id, (gate = windowGate(APP_RECV_MAX_PER_SEC)));
|
|
566
|
+
if (gate(performance.now())) this.emit('message', { from: id, data: app.data });
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
// ── content (3D app share) ──────────────────────────────────────────────────────────────
|
|
571
|
+
|
|
572
|
+
/**
|
|
573
|
+
* A peer's hello: both sides carry content → the pair gets its content transceiver (the offerer
|
|
574
|
+
* adds it; call/transport.js). A peer that cannot (an older SDK) while this side shares: one
|
|
575
|
+
* `warning { code: 'content-unsupported' }` per peer and share.
|
|
576
|
+
*/
|
|
577
|
+
_onPeerHello(id, h) {
|
|
578
|
+
if (h.content) this.transport?.addContentTransceiver?.(id);
|
|
579
|
+
else this._warnContentPeer(id);
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
_warnContentPeer(id) {
|
|
583
|
+
const tile = this.tiles.get(id);
|
|
584
|
+
if (!this.content || !tile || !tile.hello || tile.hello.content || this._contentWarned.has(id)) return;
|
|
585
|
+
this._contentWarned.add(id);
|
|
586
|
+
this.warning('content-unsupported', `participant ${id} is on an older SDK that cannot receive a shared app; they keep seeing your camera only`);
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
_onContentStream(id, stream) {
|
|
590
|
+
if (stream) this.contentStreams.set(id, stream);
|
|
591
|
+
else this.contentStreams.delete(id);
|
|
592
|
+
const ct = this.contentTiles.get(id);
|
|
593
|
+
if (ct && stream) ct.setStream(stream);
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
/** A peer's `content` message (wire.normalizeContent validated it). */
|
|
597
|
+
_onRemoteContent(id, cm) {
|
|
598
|
+
const tile = this.tiles.get(id);
|
|
599
|
+
if (!tile) return;
|
|
600
|
+
const prev = tile.content;
|
|
601
|
+
if (!cm.on) {
|
|
602
|
+
if (prev.on) this._dropRemoteContent(id);
|
|
603
|
+
return;
|
|
604
|
+
}
|
|
605
|
+
tile.content = { on: true, format: cm.format, appName: cm.appName };
|
|
606
|
+
let ct = this.contentTiles.get(id);
|
|
607
|
+
if (!ct) {
|
|
608
|
+
ct = new ContentTile(this, id);
|
|
609
|
+
this.contentTiles.set(id, ct);
|
|
610
|
+
const s = this.contentStreams.get(id);
|
|
611
|
+
if (s) ct.setStream(s);
|
|
612
|
+
}
|
|
613
|
+
ct.setContent(cm);
|
|
614
|
+
this.log('content', { peer: id, on: true, format: cm.format, appName: cm.appName, convergence: cm.convergence });
|
|
615
|
+
if (!prev.on || prev.appName !== cm.appName) this.emit('content', { id, on: true, appName: cm.appName });
|
|
616
|
+
this._layout();
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
_dropRemoteContent(id) {
|
|
620
|
+
const tile = this.tiles.get(id);
|
|
621
|
+
const ct = this.contentTiles.get(id);
|
|
622
|
+
const was = tile ? tile.content : null;
|
|
623
|
+
if (tile) tile.content = { on: false, format: null, appName: null };
|
|
624
|
+
if (ct) {
|
|
625
|
+
ct.destroy();
|
|
626
|
+
this.contentTiles.delete(id);
|
|
627
|
+
}
|
|
628
|
+
if (was && was.on) {
|
|
629
|
+
this.log('content', { peer: id, on: false });
|
|
630
|
+
this.emit('content', { id, on: false, appName: was.appName });
|
|
631
|
+
}
|
|
632
|
+
this._layout();
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
/**
|
|
636
|
+
* Share `src` (a MediaStream or a video MediaStreamTrack — `openAppShare()`'s, or any
|
|
637
|
+
* side-by-side track) as this side's content. `opts` override what the track says about itself
|
|
638
|
+
* (`openAppShare()`'s `descriptor` fits as is). Replaces a previous share. Resolves with what is
|
|
639
|
+
* announced to everyone.
|
|
640
|
+
*/
|
|
641
|
+
async shareContent(src, opts = {}) {
|
|
642
|
+
const isStream = !!(src && typeof src.getVideoTracks === 'function');
|
|
643
|
+
const track = isStream ? src.getVideoTracks()[0] || null : src && src.kind === 'video' ? src : null;
|
|
644
|
+
if (!track) throw new TypeError(`${TAG} shareContent(src) needs a MediaStream with a video track, or a video MediaStreamTrack`);
|
|
645
|
+
if (track.readyState === 'ended') throw Object.assign(new Error(`${TAG} shareContent: that track has ended`), { code: 'content-ended' });
|
|
646
|
+
const stream = isStream ? src : new MediaStream([track]);
|
|
647
|
+
const o = opts && typeof opts === 'object' ? opts : {};
|
|
648
|
+
const auto = describeAppShare(track);
|
|
649
|
+
const app = isAppShareDevice(track);
|
|
650
|
+
const c = o.convergence;
|
|
651
|
+
const desc = {
|
|
652
|
+
format: o.format === 'mono' || o.format === 'sbs' ? o.format : auto.format,
|
|
653
|
+
width: o.width > 0 ? o.width : auto.width,
|
|
654
|
+
height: o.height > 0 ? o.height : auto.height,
|
|
655
|
+
// A DisplayXR app's frames are display-referred (its hint, 0 when the browser has none); any
|
|
656
|
+
// other track's convergence is unknown unless the page says so.
|
|
657
|
+
convergence: c && typeof c === 'object' ? { known: !!c.known, frac: c.known && Number.isFinite(c.frac) ? c.frac : c.known ? 0 : null } : app ? auto.convergence : { known: false, frac: null },
|
|
658
|
+
appName: sanitizeName(o.appName) || auto.appName,
|
|
659
|
+
};
|
|
660
|
+
const announced = wire.makeContent(desc);
|
|
661
|
+
desc.convergence = announced.convergence;
|
|
662
|
+
desc.width = announced.width;
|
|
663
|
+
desc.height = announced.height;
|
|
664
|
+
try {
|
|
665
|
+
track.contentHint = 'detail';
|
|
666
|
+
} catch {
|
|
667
|
+
/* older engines */
|
|
668
|
+
}
|
|
669
|
+
if (this.content && this.content.track !== track) this.content.off();
|
|
670
|
+
const onEnded = () => {
|
|
671
|
+
if (this.content && this.content.track === track) this.stopContent({}, 'ended').catch(() => {});
|
|
672
|
+
};
|
|
673
|
+
if (!this.content || this.content.track !== track) track.addEventListener?.('ended', onEnded);
|
|
674
|
+
const off = () => track.removeEventListener?.('ended', onEnded);
|
|
675
|
+
const was = this.content;
|
|
676
|
+
this.content = { track, stream, desc, off: was && was.track === track ? was.off : off };
|
|
677
|
+
this._contentWarned.clear();
|
|
678
|
+
if (this.transport) {
|
|
679
|
+
await this.transport.replaceTrack('video', track, stream, { slot: 'content' });
|
|
680
|
+
this.transport.broadcast(announced);
|
|
681
|
+
for (const id of this.tiles.keys()) this._warnContentPeer(id);
|
|
682
|
+
}
|
|
683
|
+
this._refreshSelfContent();
|
|
684
|
+
this.log('share-content', { format: desc.format, width: desc.width, height: desc.height, appName: desc.appName, convergence: desc.convergence });
|
|
685
|
+
if (!was || was.desc.appName !== desc.appName) this.emit('content', { id: 'self', on: true, appName: desc.appName });
|
|
686
|
+
return this.contentInfo();
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
/** Stop sharing: the content sender goes empty, everyone gets `content { on: false }`. */
|
|
690
|
+
async stopContent({ stopTrack = false } = {}, reason = 'stopped') {
|
|
691
|
+
const c = this.content;
|
|
692
|
+
if (!c) return false;
|
|
693
|
+
this.content = null;
|
|
694
|
+
c.off();
|
|
695
|
+
this._refreshSelfContent();
|
|
696
|
+
if (this.transport) {
|
|
697
|
+
this.transport.broadcast(wire.makeContent({ on: false }));
|
|
698
|
+
await this.transport.replaceTrack('video', null, null, { slot: 'content' });
|
|
699
|
+
}
|
|
700
|
+
if (stopTrack) c.track.stop?.();
|
|
701
|
+
this.log('stop-content', { reason });
|
|
702
|
+
this.emit('content', { id: 'self', on: false, appName: c.desc.appName, reason });
|
|
703
|
+
return true;
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
/** `handle.content`: what this side shares, or null. */
|
|
707
|
+
contentInfo() {
|
|
708
|
+
const c = this.content;
|
|
709
|
+
if (!c) return null;
|
|
710
|
+
const d = c.desc;
|
|
711
|
+
return Object.freeze({ on: true, format: d.format, width: d.width, height: d.height, appName: d.appName, convergence: Object.freeze({ ...d.convergence }) });
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
/** The optional preview of what this side shares (`selfContentView`). */
|
|
715
|
+
_refreshSelfContent() {
|
|
716
|
+
const want = !!(this.content && this.o.selfContentView && this.state !== 'left');
|
|
717
|
+
let ct = this.contentTiles.get('self');
|
|
718
|
+
if (!want) {
|
|
719
|
+
if (ct) {
|
|
720
|
+
ct.destroy();
|
|
721
|
+
this.contentTiles.delete('self');
|
|
722
|
+
this._layout();
|
|
723
|
+
}
|
|
724
|
+
return;
|
|
725
|
+
}
|
|
726
|
+
if (!ct) {
|
|
727
|
+
ct = new ContentTile(this, 'self');
|
|
728
|
+
this.contentTiles.set('self', ct);
|
|
729
|
+
}
|
|
730
|
+
ct.setStream(this.content.stream);
|
|
731
|
+
ct.setContent({ on: true, ...this.content.desc });
|
|
732
|
+
this._layout();
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
/** The signalling connection's state for `handle.signal` + the `signal` event (transitions only). */
|
|
736
|
+
_setSignal(st, err) {
|
|
737
|
+
if (st === this.signalState) return;
|
|
738
|
+
this.signalState = st;
|
|
739
|
+
const ev = { state: st };
|
|
740
|
+
if (err && err.code) ev.code = err.code;
|
|
741
|
+
this.emit('signal', ev);
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
// ── headless hooks: names, app messages, devices, the self view ─────────────────────────
|
|
745
|
+
|
|
746
|
+
setName(n) {
|
|
747
|
+
this.name = sanitizeName(n);
|
|
748
|
+
this.transport?.broadcast(this.hello());
|
|
749
|
+
return this.name;
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
/** One app message to one peer: false when not connected, too big, not serialisable, or over its rate. */
|
|
753
|
+
sendApp(peerId, data) {
|
|
754
|
+
if (!this.transport) return false;
|
|
755
|
+
const m = makeAppMessage(data);
|
|
756
|
+
if (!m) return false;
|
|
757
|
+
return this._sendAppTo(peerId, m.json);
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
/** The same message to every peer; true only if every connected peer got it. */
|
|
761
|
+
broadcastApp(data) {
|
|
762
|
+
if (!this.transport) return false;
|
|
763
|
+
const m = makeAppMessage(data);
|
|
764
|
+
if (!m) return false;
|
|
765
|
+
let any = false;
|
|
766
|
+
let all = true;
|
|
767
|
+
for (const id of this.tiles.keys()) {
|
|
768
|
+
if (this.transport.peerState?.(id) === null) continue; // a ghost / a peer the mesh does not hold
|
|
769
|
+
any = true;
|
|
770
|
+
if (!this._sendAppTo(id, m.json)) all = false;
|
|
771
|
+
}
|
|
772
|
+
return any && all;
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
_sendAppTo(id, json) {
|
|
776
|
+
if (!this.tiles.has(id)) return false;
|
|
777
|
+
let gate = this._appOut.get(id);
|
|
778
|
+
if (!gate) this._appOut.set(id, (gate = windowGate(APP_MAX_PER_SEC)));
|
|
779
|
+
if (!gate(performance.now())) return false;
|
|
780
|
+
return this.transport.sendRaw ? this.transport.sendRaw(id, json) : this.transport.send(id, JSON.parse(json));
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
/** Switch the microphone to `deviceId` (null = the default). False (old mic kept) when it cannot open. */
|
|
784
|
+
async setMic(deviceId) {
|
|
785
|
+
const track = await openMic({ deviceId: deviceId || undefined, log: (t, x) => this.log(t, x) });
|
|
786
|
+
if (!track) return false;
|
|
787
|
+
const l = this.local;
|
|
788
|
+
if (!l) {
|
|
789
|
+
track.stop();
|
|
790
|
+
return false;
|
|
791
|
+
}
|
|
792
|
+
const old = l.audioTrack;
|
|
793
|
+
track.enabled = !this.muted;
|
|
794
|
+
l.audioTrack = track;
|
|
795
|
+
l.stream = new MediaStream([l.videoTrack, track].filter(Boolean));
|
|
796
|
+
if (this.transport) await this.transport.replaceTrack('audio', track, l.stream);
|
|
797
|
+
if (old && old !== track) old.stop();
|
|
798
|
+
this.log('mic', { deviceId: deviceId || null });
|
|
799
|
+
return true;
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
/** Every participant's audio to `sinkId` (also tiles that join later). True if every tile took it. */
|
|
803
|
+
async setAudioOutput(sinkId) {
|
|
804
|
+
this.sinkId = sinkId || '';
|
|
805
|
+
const results = await Promise.all([...this.tiles.values()].map((t) => t.applySink()));
|
|
806
|
+
return results.every(Boolean);
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
setSelfView(on) {
|
|
810
|
+
const want = on === undefined ? !this.selfViewOn : !!on;
|
|
811
|
+
if (want === this.selfViewOn) return want;
|
|
812
|
+
this.selfViewOn = want;
|
|
813
|
+
if (want) {
|
|
814
|
+
if (!this.self) this.self = new SelfTile(this, this.ui.selfSlot);
|
|
815
|
+
show(this.ui.selfSlot, true);
|
|
816
|
+
show(this.self.el, true);
|
|
817
|
+
this.self.update();
|
|
818
|
+
} else if (this.self) {
|
|
819
|
+
// Remove the LAYER first (never display:none over a live registration), then hide the box.
|
|
820
|
+
this.self.detach();
|
|
821
|
+
show(this.self.el, false);
|
|
822
|
+
}
|
|
823
|
+
this.log('self-view', { on: want });
|
|
824
|
+
return want;
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
// ── sender-side auto-convergence ─────────────────────────────────────────────────────────
|
|
828
|
+
|
|
829
|
+
_startConvSend() {
|
|
830
|
+
this._stopConvSend();
|
|
831
|
+
this._convTimer = setInterval(() => this._tickConv(), AUTO_CONV_INTERVAL_MS);
|
|
832
|
+
}
|
|
833
|
+
|
|
834
|
+
_stopConvSend() {
|
|
835
|
+
clearInterval(this._convTimer);
|
|
836
|
+
this._convTimer = 0;
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
/** Forget the measurement (a new camera, leaving): the next `conv` is a fresh one. */
|
|
840
|
+
_resetConvSend() {
|
|
841
|
+
this._convGen++;
|
|
842
|
+
this._convMeter.reset();
|
|
843
|
+
this._convOut = { frac: null, sentAt: 0, msg: null };
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
/** Is this side measuring and sending its convergence right now? */
|
|
847
|
+
_convSending() {
|
|
848
|
+
const l = this.local;
|
|
849
|
+
return !!(this.transport && this.o.sendConvergence && l && l.format === 'sbs' && l.videoTrack && !this.camOff && l.video);
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
/**
|
|
853
|
+
* One sample of this side's raw camera frame (`local.video` plays the camera's own stream: the
|
|
854
|
+
* pre-encode, unmirrored pair; the self view's mirroring is a paint, never on this video). Runs
|
|
855
|
+
* whatever the self view does, including `selfView: false`. Broadcasts `conv` when the gate
|
|
856
|
+
* passes and the value moved by a quarter source pixel, or the heartbeat is due.
|
|
857
|
+
*/
|
|
858
|
+
_tickConv() {
|
|
859
|
+
if (!this._convSending() || this._convMeter.busy) return;
|
|
860
|
+
const v = this.local.video;
|
|
861
|
+
const gen = this._convGen;
|
|
862
|
+
this._convMeter
|
|
863
|
+
.sample(v, performance.now())
|
|
864
|
+
.then((r) => {
|
|
865
|
+
if (!r || gen !== this._convGen || !this._convSending()) return;
|
|
866
|
+
// Gate on the SEND time (what a receiver's own gate sees). A send the gate refuses is not
|
|
867
|
+
// lost: the value still differs from the last one sent, so the next tick sends it.
|
|
868
|
+
const now = performance.now();
|
|
869
|
+
const out = this._convOut;
|
|
870
|
+
const eyeW = (v.videoWidth || 0) / 2;
|
|
871
|
+
const minStep = eyeW > 0 ? 0.25 / eyeW : 0;
|
|
872
|
+
const moved = (r.frac === null) !== (out.frac === null) || (r.frac !== null && Math.abs(r.frac - out.frac) >= minStep);
|
|
873
|
+
const due = !out.msg || now - out.sentAt >= CONV_HEARTBEAT_MS;
|
|
874
|
+
if ((!moved && !due) || !this._convGate(now)) return;
|
|
875
|
+
const msg = makeConv(r.frac, r.ncc);
|
|
876
|
+
this.transport.broadcast(msg);
|
|
877
|
+
this._convOut = { frac: r.frac, sentAt: now, msg };
|
|
878
|
+
if (r.frac !== null && !this._convLogged) {
|
|
879
|
+
this._convLogged = true;
|
|
880
|
+
this.log('send-conv', { frac: Math.round(r.frac * 1e4) / 1e4, disparityPx: Math.round(r.d * 10) / 10, method: r.method, via: r.via, ms: Math.round(r.ms) });
|
|
881
|
+
}
|
|
882
|
+
})
|
|
883
|
+
.catch(() => {});
|
|
504
884
|
}
|
|
505
885
|
|
|
506
886
|
// ── handle ops ─────────────────────────────────────────────────────────────────────────
|
|
@@ -533,6 +913,7 @@ class Call {
|
|
|
533
913
|
const n = v === null || v === undefined || !Number.isFinite(+v) ? 0 : Math.max(-1, Math.min(1, +v));
|
|
534
914
|
this.depth = n;
|
|
535
915
|
for (const t of this.tiles.values()) t.conv.depth = n;
|
|
916
|
+
for (const t of this.contentTiles.values()) t.conv.depth = n; // added to content's known convergence
|
|
536
917
|
this.liftPool?.setDepth(n); // the same control is lift's convergence on lifted tiles
|
|
537
918
|
if (this.ui?.depth && +this.ui.depth.value !== n) this.ui.depth.value = String(n);
|
|
538
919
|
return n;
|
|
@@ -541,6 +922,7 @@ class Call {
|
|
|
541
922
|
async setCamera(idOrStream, { format } = {}) {
|
|
542
923
|
const before = this.local?.format;
|
|
543
924
|
const l = await this._openMedia(idOrStream, format, { keepOnFail: true });
|
|
925
|
+
this._resetConvSend(); // a new camera: measure afresh, before the hello that announces it
|
|
544
926
|
if (this.transport) {
|
|
545
927
|
await this.transport.replaceTrack('video', l.videoTrack, l.stream);
|
|
546
928
|
await this.transport.setSendFormat(l.format);
|
|
@@ -569,8 +951,20 @@ class Call {
|
|
|
569
951
|
globalThis.removeEventListener?.('pagehide', this._pagehide);
|
|
570
952
|
this.transport?.stop();
|
|
571
953
|
this.transport = null;
|
|
954
|
+
this._stopConvSend();
|
|
955
|
+
this._resetConvSend();
|
|
572
956
|
for (const t of this.tiles.values()) t.destroy();
|
|
573
957
|
this.tiles.clear();
|
|
958
|
+
for (const t of this.contentTiles.values()) t.destroy();
|
|
959
|
+
this.contentTiles.clear();
|
|
960
|
+
this.contentStreams.clear();
|
|
961
|
+
if (this.content) {
|
|
962
|
+
// Leaving ends the share (the track is the page's; it is left running).
|
|
963
|
+
const c = this.content;
|
|
964
|
+
this.content = null;
|
|
965
|
+
c.off();
|
|
966
|
+
this.emit('content', { id: 'self', on: false, appName: c.desc.appName, reason: 'left' });
|
|
967
|
+
}
|
|
574
968
|
this.liftPool?.releaseAll();
|
|
575
969
|
clearInterval(this._statsTimer);
|
|
576
970
|
this._setSpeaker(null);
|
|
@@ -585,6 +979,9 @@ class Call {
|
|
|
585
979
|
this.local.audioTrack = null;
|
|
586
980
|
}
|
|
587
981
|
this.self?.detach(); // a rejoin re-attaches it
|
|
982
|
+
this.signalState = null;
|
|
983
|
+
this._appOut.clear();
|
|
984
|
+
this._appIn.clear();
|
|
588
985
|
this._unhookWall();
|
|
589
986
|
this._setState('left');
|
|
590
987
|
this._layout();
|
|
@@ -648,6 +1045,7 @@ class Call {
|
|
|
648
1045
|
// Re-create only what was woven before the session was live (#172); flat tiles are untouched.
|
|
649
1046
|
// A lifted tile is a woven window too (lift's addScene canvas): release + lift again.
|
|
650
1047
|
for (const t of this.tiles.values()) if (t.route === 'woven-sbs' || t.route === 'lifted') t.reroute(true);
|
|
1048
|
+
for (const t of this.contentTiles.values()) if (t.route === 'woven-sbs') t.reroute(true);
|
|
651
1049
|
if (this.self && this.self.route === 'woven-sbs') this.self.reroute(true);
|
|
652
1050
|
this._layout();
|
|
653
1051
|
};
|
|
@@ -676,6 +1074,7 @@ class Call {
|
|
|
676
1074
|
this.weaveLive = false;
|
|
677
1075
|
this.log('session-ended', { byPage: hook.pageClosed });
|
|
678
1076
|
for (const t of this.tiles.values()) t.onWallLost();
|
|
1077
|
+
for (const t of this.contentTiles.values()) t.onWallLost();
|
|
679
1078
|
this.self?.onWallLost();
|
|
680
1079
|
if (hook.pageClosed || this.state === 'left') {
|
|
681
1080
|
this._unhookWall();
|
|
@@ -704,6 +1103,7 @@ class Call {
|
|
|
704
1103
|
this._hookWall(w);
|
|
705
1104
|
this.log('session-recovered', { attempt });
|
|
706
1105
|
for (const t of this.tiles.values()) t.reroute(true);
|
|
1106
|
+
for (const t of this.contentTiles.values()) t.reroute(true);
|
|
707
1107
|
this.self?.reroute(true);
|
|
708
1108
|
}, backoffMs(attempt, { baseMs: 1000, maxMs: 15000 }));
|
|
709
1109
|
}
|
|
@@ -715,6 +1115,7 @@ class Call {
|
|
|
715
1115
|
const tick = (now) => {
|
|
716
1116
|
this._raf = requestAnimationFrame(tick);
|
|
717
1117
|
for (const t of this.tiles.values()) t.paint();
|
|
1118
|
+
for (const t of this.contentTiles.values()) t.paint();
|
|
718
1119
|
if (last && this.liftPool && this.liftPool.anyWeb) this._watchFrames(now - last);
|
|
719
1120
|
last = now;
|
|
720
1121
|
};
|
|
@@ -863,10 +1264,16 @@ class Call {
|
|
|
863
1264
|
if (s.type === 'codec') codecs[s.id] = s.mimeType;
|
|
864
1265
|
});
|
|
865
1266
|
const q = { in: null, out: null };
|
|
1267
|
+
if (withQuality) Object.assign(q, selectedPath(report));
|
|
1268
|
+
// The content track (3D app share) has its own inbound / outbound video: not the camera's quality.
|
|
1269
|
+
const contentIds = new Set();
|
|
1270
|
+
for (const t of this.contentStreams.get(id)?.getTracks?.() || []) contentIds.add(t.id);
|
|
1271
|
+
if (this.content && this.content.track) contentIds.add(this.content.track.id);
|
|
866
1272
|
report.forEach((s) => {
|
|
867
1273
|
if (s.type === 'inbound-rtp' && s.kind === 'audio' && typeof s.audioLevel === 'number') tile.level = Math.max(s.audioLevel, (tile.level || 0) * 0.6);
|
|
868
1274
|
if (s.type === 'media-source' && s.kind === 'audio' && typeof s.audioLevel === 'number') localLevel = Math.max(localLevel, s.audioLevel);
|
|
869
1275
|
if (!withQuality) return;
|
|
1276
|
+
if (s.trackIdentifier && contentIds.has(s.trackIdentifier)) return;
|
|
870
1277
|
if (s.type === 'inbound-rtp' && s.kind === 'video') {
|
|
871
1278
|
const prev = tile._prevIn;
|
|
872
1279
|
q.in = {
|
|
@@ -893,7 +1300,7 @@ class Call {
|
|
|
893
1300
|
tile._prevOut = { bytes: s.bytesSent, t: s.timestamp };
|
|
894
1301
|
}
|
|
895
1302
|
});
|
|
896
|
-
if (withQuality && (q.in || q.out)) {
|
|
1303
|
+
if (withQuality && (q.in || q.out || q.path)) {
|
|
897
1304
|
tile.quality = q;
|
|
898
1305
|
this.emit('quality', { id, ...q });
|
|
899
1306
|
}
|
|
@@ -916,6 +1323,7 @@ class Call {
|
|
|
916
1323
|
this._selfSpeaking = speaking;
|
|
917
1324
|
this.transport?.broadcast(this.stateMsg());
|
|
918
1325
|
this.self?.setSpeaking(speaking);
|
|
1326
|
+
this.emit('speaking', { id: 'self', on: speaking, kind: 'camera' });
|
|
919
1327
|
}
|
|
920
1328
|
}
|
|
921
1329
|
|
|
@@ -962,6 +1370,9 @@ class Call {
|
|
|
962
1370
|
const t = this.t;
|
|
963
1371
|
injectCallStyle(c.ownerDocument || document);
|
|
964
1372
|
c.classList.add('dxr-call-host');
|
|
1373
|
+
// layout:'none': the footer and self slot step aside too (display:contents), so the page can
|
|
1374
|
+
// place the self tile ([data-dxr-peer="self"]) — style it in place, never move it.
|
|
1375
|
+
c.classList.toggle('dxr-call-host--none', this.o.layout === 'none');
|
|
965
1376
|
// theme → CSS custom properties on the host (chrome only: no variable can reach a tile).
|
|
966
1377
|
for (const [prop, v] of Object.entries(this.o.cssVars)) c.style.setProperty(prop, v);
|
|
967
1378
|
const ui = (this.ui = {});
|
|
@@ -979,7 +1390,7 @@ class Call {
|
|
|
979
1390
|
show(ui.bar, false);
|
|
980
1391
|
}
|
|
981
1392
|
c.append(ui.banner, ui.panel, ui.grid, ui.invite, ui.foot);
|
|
982
|
-
if (this.
|
|
1393
|
+
if (this.selfViewOn) this.self = new SelfTile(this, ui.selfSlot);
|
|
983
1394
|
else show(ui.selfSlot, false);
|
|
984
1395
|
if (this.o.chrome) this._buildBar();
|
|
985
1396
|
}
|
|
@@ -1062,11 +1473,16 @@ class Call {
|
|
|
1062
1473
|
c.classList.toggle('dxr-call-host--lobby', lobby);
|
|
1063
1474
|
show(ui.banner, this.o.chrome && !this.woven);
|
|
1064
1475
|
const n = this.tiles.size;
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1476
|
+
const nc = this.contentTiles.size;
|
|
1477
|
+
ui.grid.dataset.n = String(Math.min(n + nc, 4));
|
|
1478
|
+
ui.grid.dataset.content = nc ? '1' : '0';
|
|
1479
|
+
show(ui.grid, n + nc > 0 && this.state !== 'left');
|
|
1480
|
+
// Shared content is the main tile (it spans the row; the people form a strip under it).
|
|
1481
|
+
// Otherwise, speaker layout: the active speaker (or the first tile) spans the row. layout:'none' positions nothing.
|
|
1482
|
+
const contentMain = nc && this.o.layout !== 'none' ? [...this.contentTiles.values()][0] : null;
|
|
1483
|
+
const main = contentMain ? null : this.o.layout === 'speaker' ? this.speakerId || [...this.tiles.keys()][0] : null;
|
|
1069
1484
|
for (const [id, tile] of this.tiles) tile.el.classList.toggle('dxr-call-tile--main', id === main);
|
|
1485
|
+
for (const ct of this.contentTiles.values()) ct.el.classList.toggle('dxr-call-tile--main', ct === contentMain);
|
|
1070
1486
|
if (!this.o.chrome) return;
|
|
1071
1487
|
this._refreshBar();
|
|
1072
1488
|
queueMicrotask(() => this._fitGrid());
|
|
@@ -1129,6 +1545,9 @@ class Call {
|
|
|
1129
1545
|
hello: this.hello(),
|
|
1130
1546
|
transport: this.transport,
|
|
1131
1547
|
peers: [...this.tiles.values()].map((tile) => tile.diag()),
|
|
1548
|
+
// Content tiles (3D app share), by participant ('self' = the preview of your own share).
|
|
1549
|
+
content: [...this.contentTiles.values()].map((tile) => tile.diag()),
|
|
1550
|
+
sharing: this.contentInfo(),
|
|
1132
1551
|
// The self view's ACTUAL weave state (web#131): route, woven, the fallback reason and the
|
|
1133
1552
|
// tile's firstWoven result — null without a self view.
|
|
1134
1553
|
self: this.self ? this.self.diag() : null,
|
|
@@ -1203,8 +1622,37 @@ class Call {
|
|
|
1203
1622
|
retryCamera: () => call.retryCamera(),
|
|
1204
1623
|
setMono3D: (on) => call.setMono3D(on),
|
|
1205
1624
|
leave: () => call.leave('left'),
|
|
1206
|
-
/**
|
|
1207
|
-
|
|
1625
|
+
/**
|
|
1626
|
+
* A participant's tile element (for `layout: 'none'`), or null; `'self'` is your self view.
|
|
1627
|
+
* `kind: 'content'`: their shared-app tile instead (`'self'`: the preview of yours).
|
|
1628
|
+
*/
|
|
1629
|
+
tile: (peerId, kind = 'camera') =>
|
|
1630
|
+
kind === 'content' ? call.contentTiles.get(peerId)?.el || null : peerId === 'self' ? call.self?.el || null : call.tiles.get(peerId)?.el || null,
|
|
1631
|
+
/** 3D app share: share a track (a running app's views from openAppShare(), or any SBS track). */
|
|
1632
|
+
shareContent: (src, o) => call.shareContent(src, o),
|
|
1633
|
+
stopContent: (o) => call.stopContent(o),
|
|
1634
|
+
/** What you share right now, or null. */
|
|
1635
|
+
get content() {
|
|
1636
|
+
return call.contentInfo();
|
|
1637
|
+
},
|
|
1638
|
+
/** Your display name (sent in `hello`), or null. */
|
|
1639
|
+
get name() {
|
|
1640
|
+
return call.name;
|
|
1641
|
+
},
|
|
1642
|
+
setName: (n) => call.setName(n),
|
|
1643
|
+
send: (peerId, data) => call.sendApp(peerId, data),
|
|
1644
|
+
broadcast: (data) => call.broadcastApp(data),
|
|
1645
|
+
/** The signalling connection: 'connected' | 'reconnecting' | 'closed', null before join / after leave. */
|
|
1646
|
+
get signal() {
|
|
1647
|
+
return call.signalState;
|
|
1648
|
+
},
|
|
1649
|
+
setSelfView: (on) => call.setSelfView(on),
|
|
1650
|
+
setMic: (deviceId) => call.setMic(deviceId),
|
|
1651
|
+
setAudioOutput: (sinkId) => call.setAudioOutput(sinkId),
|
|
1652
|
+
/** What this side sends (camera + microphone), read-only; null before a camera / mic opened. */
|
|
1653
|
+
get localStream() {
|
|
1654
|
+
return call.local ? call.local.stream : null;
|
|
1655
|
+
},
|
|
1208
1656
|
diagnostics: () => call.diagnostics(),
|
|
1209
1657
|
on(type, cb) {
|
|
1210
1658
|
if (typeof cb !== 'function') throw new TypeError(`${TAG} on() takes a function`);
|
|
@@ -1247,6 +1695,27 @@ class Call {
|
|
|
1247
1695
|
|
|
1248
1696
|
// ── paint helpers ──────────────────────────────────────────────────────────────────────────
|
|
1249
1697
|
|
|
1698
|
+
/**
|
|
1699
|
+
* Observe a tile's stage box (`ResizeObserver`, where there is one): `onBox(w, h)` with its
|
|
1700
|
+
* `clientWidth` / `clientHeight` on every change. Returns the disconnect, or null without one.
|
|
1701
|
+
*/
|
|
1702
|
+
function observeBox(el, onBox) {
|
|
1703
|
+
if (typeof ResizeObserver !== 'function') return null;
|
|
1704
|
+
const ro = new ResizeObserver(() => onBox(el.clientWidth || 0, el.clientHeight || 0));
|
|
1705
|
+
ro.observe(el);
|
|
1706
|
+
return () => ro.disconnect();
|
|
1707
|
+
}
|
|
1708
|
+
|
|
1709
|
+
/** The `weave` event payload of a settled `firstWoven` / `rewoven()`. `kind`: which tile of `id`. */
|
|
1710
|
+
function weaveEvent(id, res, kind = 'camera') {
|
|
1711
|
+
const ev = { id, state: res.woven ? 'woven' : 'flat', ms: Math.round(res.ms || 0), kind };
|
|
1712
|
+
if (res.reason) ev.reason = res.reason;
|
|
1713
|
+
return ev;
|
|
1714
|
+
}
|
|
1715
|
+
|
|
1716
|
+
/** `theme.tileAspect: 'fill'`: a stage box's aspect, clamped to what a woven tile may take. */
|
|
1717
|
+
const clampAspect = (w, h) => Math.max(0.3, Math.min(4, w / h));
|
|
1718
|
+
|
|
1250
1719
|
/** Size a FLAT (never woven) canvas to its box × dpr. Returns false if it has no box yet. */
|
|
1251
1720
|
function sizeFlat(canvas) {
|
|
1252
1721
|
const dpr = Math.min(globalThis.devicePixelRatio || 1, 2);
|
|
@@ -1284,9 +1753,17 @@ function paintFlat(canvas, video, { eyeHalf = null, mirror = false } = {}) {
|
|
|
1284
1753
|
// ── a remote participant ──────────────────────────────────────────────────────────────────
|
|
1285
1754
|
|
|
1286
1755
|
class Tile {
|
|
1287
|
-
|
|
1756
|
+
/**
|
|
1757
|
+
* @param {Call} call
|
|
1758
|
+
* @param {string} id the participant
|
|
1759
|
+
* @param {'camera'|'content'} [kind] `'content'`: a ContentTile (their shared app), see below
|
|
1760
|
+
*/
|
|
1761
|
+
constructor(call, id, kind = 'camera') {
|
|
1288
1762
|
this.call = call;
|
|
1289
1763
|
this.id = id;
|
|
1764
|
+
this.kind = kind;
|
|
1765
|
+
// What this participant shares (their `content` messages): { on, format, appName }.
|
|
1766
|
+
this.content = { on: false, format: null, appName: null };
|
|
1290
1767
|
this.hello = null;
|
|
1291
1768
|
this.format = null; // null until hello or its timeout
|
|
1292
1769
|
this.route = null;
|
|
@@ -1304,6 +1781,7 @@ class Tile {
|
|
|
1304
1781
|
this.autoCanvas = null;
|
|
1305
1782
|
this.autoLocked = false;
|
|
1306
1783
|
this.autoStats = { n: 0, ms: 0, readMs: 0, method: null, via: null };
|
|
1784
|
+
this.autoGrabs = 0; // local copies made: the receiver fallback, or the depth cursor's
|
|
1307
1785
|
this.autoPending = false;
|
|
1308
1786
|
// Bumped on every stream change and on destroy(): an async copyTo() that resolves after either
|
|
1309
1787
|
// belongs to a stream this tile no longer shows, and is dropped (review of #92).
|
|
@@ -1312,6 +1790,10 @@ class Tile {
|
|
|
1312
1790
|
this.destroyed = false;
|
|
1313
1791
|
this.conv.depth = call.depth;
|
|
1314
1792
|
this.hintGate = rateGate(HINT_MAX_HZ);
|
|
1793
|
+
// The sender's own measurement (`conv`, when its hello says `conv: true`): `{ frac, c, at }`,
|
|
1794
|
+
// `at` = when it ARRIVED here. Fresh for CONV_FRESH_MS; after that this side measures again.
|
|
1795
|
+
this.convGate = rateGate(CONV_RECV_MAX_HZ);
|
|
1796
|
+
this.senderConv = null;
|
|
1315
1797
|
this.layerFails = 0;
|
|
1316
1798
|
this.forceFlat = false;
|
|
1317
1799
|
// lifted route (mono→3D): the lift handle once it lands, whether it is showing (live), and a
|
|
@@ -1323,7 +1805,8 @@ class Tile {
|
|
|
1323
1805
|
this.liftFails = 0;
|
|
1324
1806
|
this.visible = true;
|
|
1325
1807
|
this.video = Object.assign(document.createElement('video'), { muted: true, playsInline: true, autoplay: true });
|
|
1326
|
-
|
|
1808
|
+
// A content tile is video only: the participant's voice stays on their camera tile.
|
|
1809
|
+
this.audio = kind === 'content' ? null : Object.assign(document.createElement('audio'), { autoplay: true });
|
|
1327
1810
|
this.video.addEventListener('playing', () => this._maybeRoute());
|
|
1328
1811
|
this.video.addEventListener('loadeddata', () => this._maybeRoute());
|
|
1329
1812
|
this.video.addEventListener('resize', () => this._maybeRoute());
|
|
@@ -1332,17 +1815,29 @@ class Tile {
|
|
|
1332
1815
|
this.badge = el('div', { class: 'dxr-call-badge', part: 'badge' });
|
|
1333
1816
|
this.plate = el('div', { class: 'dxr-call-state', part: 'plate', text: call.t('connecting') });
|
|
1334
1817
|
this.stage = el('div', { class: 'dxr-call-stage' }, [this.canvas, this.cover, this.badge, this.plate]);
|
|
1335
|
-
|
|
1818
|
+
// 'fill' (theme.tileAspect): no aspect-ratio — the page sizes the stage and the crop follows it.
|
|
1819
|
+
if (call.o.tileAspect !== 'fill') this.stage.style.aspectRatio = String(call.o.tileAspect);
|
|
1820
|
+
/** The stage box `{ w, h }` from the ResizeObserver; null until it first reports (or without one). */
|
|
1821
|
+
this._box = null;
|
|
1822
|
+
this._aspect = null; // 'fill': the stage's own aspect, once measured
|
|
1823
|
+
this._rwP = null; // the rewoven() promise a `weave` event is waiting on
|
|
1824
|
+
this._unobserve = observeBox(this.stage, (w, h) => this._onBox(w, h));
|
|
1336
1825
|
// `data-dxr-peer` is the documented hook (layout:'none', handle.tile()); `data-peer` is the 1.29 spelling.
|
|
1337
|
-
|
|
1826
|
+
// `data-dxr-kind`: 'camera' (the person) or 'content' (what they share) — tile(id, kind).
|
|
1827
|
+
const attrs = { class: `dxr-call-tile${kind === 'content' ? ' dxr-call-tile--content' : ''}`, part: 'tile', 'data-dxr-peer': id, 'data-dxr-kind': kind };
|
|
1828
|
+
if (kind !== 'content') attrs['data-peer'] = id;
|
|
1829
|
+
this.el = el('div', attrs, [this.stage, el('div', { class: 'dxr-call-talk' })]);
|
|
1338
1830
|
if (!call.o.ui) {
|
|
1339
1831
|
show(this.badge, false);
|
|
1340
1832
|
}
|
|
1341
|
-
|
|
1833
|
+
// Shared content goes FIRST in the grid (created there: a later move would be a teardown).
|
|
1834
|
+
if (kind === 'content') call.ui.grid.insertBefore(this.el, call.ui.grid.firstChild);
|
|
1835
|
+
else call.ui.grid.appendChild(this.el);
|
|
1836
|
+
if (call.sinkId) this.applySink();
|
|
1342
1837
|
this._renderBadge();
|
|
1343
1838
|
/** The depth cursor (`cursor: 'depth'`) once its module has loaded; null otherwise. */
|
|
1344
1839
|
this.cursor = null;
|
|
1345
|
-
if (call.o.cursor) {
|
|
1840
|
+
if (call.o.cursor && kind !== 'content') {
|
|
1346
1841
|
call._pairCursorMod = call._pairCursorMod || import('./camera/pair-cursor.js');
|
|
1347
1842
|
call._pairCursorMod.then((m) => {
|
|
1348
1843
|
if (!this.destroyed) this.cursor = new m.PairCursor(this.canvas, call.o.cursor);
|
|
@@ -1365,9 +1860,41 @@ class Tile {
|
|
|
1365
1860
|
muted: this.remote.muted,
|
|
1366
1861
|
cameraOff: this.remote.cameraOff,
|
|
1367
1862
|
speaking: this.speaking,
|
|
1863
|
+
name: (this.hello && this.hello.name) || null,
|
|
1864
|
+
// What they share (3D app share): `on`, its format and the app's name.
|
|
1865
|
+
content: Object.freeze({ ...this.content }),
|
|
1368
1866
|
});
|
|
1369
1867
|
}
|
|
1370
1868
|
|
|
1869
|
+
/** The stage box changed: the 'fill' aspect, and a woven tile's `weave` event once it re-joins. */
|
|
1870
|
+
_onBox(w, h) {
|
|
1871
|
+
const prev = this._box;
|
|
1872
|
+
this._box = { w, h };
|
|
1873
|
+
if (w > 0 && h > 0) this._aspect = clampAspect(w, h);
|
|
1874
|
+
if (!prev || !w || !h || (prev.w === w && prev.h === h)) return;
|
|
1875
|
+
const handle = this.handle;
|
|
1876
|
+
if (!handle || !this._fwSettled || typeof handle.rewoven !== 'function') return;
|
|
1877
|
+
const p = handle.rewoven();
|
|
1878
|
+
if (p === this._rwP || !p || typeof p.then !== 'function') return;
|
|
1879
|
+
this._rwP = p;
|
|
1880
|
+
p.then((res) => {
|
|
1881
|
+
if (this._rwP !== p || this.handle !== handle || !res) return;
|
|
1882
|
+
this._rwP = null;
|
|
1883
|
+
this.call.emit('weave', weaveEvent(this.id, res, this.kind));
|
|
1884
|
+
});
|
|
1885
|
+
}
|
|
1886
|
+
|
|
1887
|
+
/** Play this participant's audio to the call's output device (`handle.setAudioOutput`). */
|
|
1888
|
+
applySink() {
|
|
1889
|
+
const a = this.audio;
|
|
1890
|
+
if (!a) return Promise.resolve(true); // a content tile has no audio of its own
|
|
1891
|
+
if (typeof a.setSinkId !== 'function') return Promise.resolve(false);
|
|
1892
|
+
return a.setSinkId(this.call.sinkId || '').then(
|
|
1893
|
+
() => true,
|
|
1894
|
+
() => false
|
|
1895
|
+
);
|
|
1896
|
+
}
|
|
1897
|
+
|
|
1371
1898
|
/** Everything else (UNSTABLE): `handle.diagnostics().peers[i]`. */
|
|
1372
1899
|
diag() {
|
|
1373
1900
|
return {
|
|
@@ -1387,6 +1914,12 @@ class Tile {
|
|
|
1387
1914
|
readMs: this.autoStats.n ? this.autoStats.readMs / this.autoStats.n : null,
|
|
1388
1915
|
method: this.autoStats.method,
|
|
1389
1916
|
via: this.autoStats.via,
|
|
1917
|
+
// Local copies made (the receiver fallback, or the depth cursor's).
|
|
1918
|
+
samples: this.autoGrabs,
|
|
1919
|
+
// Where the applied disparity comes from: 'sender' (its `conv`, fresh), 'local' (measured
|
|
1920
|
+
// here: an older or silent sender), null (autoConverge off, or nothing measured yet).
|
|
1921
|
+
source: !this.call.o.autoConverge ? null : this._senderFrac(performance.now()) !== null ? 'sender' : this.conv.measuredPx !== null ? 'local' : null,
|
|
1922
|
+
senderFrac: this.senderConv ? this.senderConv.frac : null,
|
|
1390
1923
|
},
|
|
1391
1924
|
lift: this.lifted
|
|
1392
1925
|
? {
|
|
@@ -1403,11 +1936,11 @@ class Tile {
|
|
|
1403
1936
|
setStream(stream) {
|
|
1404
1937
|
if (this.video.srcObject !== stream) {
|
|
1405
1938
|
this.video.srcObject = stream;
|
|
1406
|
-
this.audio.srcObject = stream;
|
|
1939
|
+
if (this.audio) this.audio.srcObject = stream;
|
|
1407
1940
|
this._resetAutoConverge(); // a new stream may be a new camera: measure afresh
|
|
1408
1941
|
}
|
|
1409
1942
|
this.video.play().catch(() => {});
|
|
1410
|
-
this.audio
|
|
1943
|
+
this.audio?.play().catch(() => {});
|
|
1411
1944
|
}
|
|
1412
1945
|
|
|
1413
1946
|
onChannelOpen() {
|
|
@@ -1423,8 +1956,10 @@ class Tile {
|
|
|
1423
1956
|
|
|
1424
1957
|
setHello(h) {
|
|
1425
1958
|
clearTimeout(this.helloTimer);
|
|
1959
|
+
const prevName = this.hello ? this.hello.name || null : null;
|
|
1426
1960
|
this.hello = h;
|
|
1427
1961
|
this.conv.hello = h;
|
|
1962
|
+
if ((h.name || null) !== prevName) this.call.emit('name', { id: this.id, name: h.name || null });
|
|
1428
1963
|
const changed = this.format !== h.format;
|
|
1429
1964
|
this.format = h.format;
|
|
1430
1965
|
this.call.log('hello', { peer: this.id, format: h.format, width: h.width, height: h.height, rectified: h.rectified });
|
|
@@ -1437,6 +1972,28 @@ class Tile {
|
|
|
1437
1972
|
this.conv.subjectZmm = h.subjectZmm;
|
|
1438
1973
|
}
|
|
1439
1974
|
|
|
1975
|
+
/**
|
|
1976
|
+
* A `conv` from the sender (wire.normalizeConv already validated it). Rate-limited here too, so a
|
|
1977
|
+
* chatty or hostile sender cannot drive this tile at frame rate. `d: null` (nothing measured
|
|
1978
|
+
* yet) keeps a previous value alive and is otherwise ignored.
|
|
1979
|
+
*/
|
|
1980
|
+
setConv(cv) {
|
|
1981
|
+
const now = performance.now();
|
|
1982
|
+
if (!this.convGate(now)) return;
|
|
1983
|
+
if (cv.d === null) {
|
|
1984
|
+
if (this.senderConv) this.senderConv.at = now;
|
|
1985
|
+
return;
|
|
1986
|
+
}
|
|
1987
|
+
this.senderConv = { frac: cv.d, c: cv.c === undefined ? null : cv.c, at: now };
|
|
1988
|
+
}
|
|
1989
|
+
|
|
1990
|
+
/** The sender's measured disparity as a fraction of its eye width, when fresh and announced; else null. */
|
|
1991
|
+
_senderFrac(now) {
|
|
1992
|
+
const sc = this.senderConv;
|
|
1993
|
+
if (!sc || !this.hello || !this.hello.conv || now - sc.at >= CONV_FRESH_MS) return null;
|
|
1994
|
+
return sc.frac;
|
|
1995
|
+
}
|
|
1996
|
+
|
|
1440
1997
|
setRemoteState(st) {
|
|
1441
1998
|
this.remote = st;
|
|
1442
1999
|
this._renderBadge();
|
|
@@ -1448,6 +2005,7 @@ class Tile {
|
|
|
1448
2005
|
if (on === this.speaking) return;
|
|
1449
2006
|
this.speaking = on;
|
|
1450
2007
|
this.el.classList.toggle('dxr-call-tile--speaking', on);
|
|
2008
|
+
this.call.emit('speaking', { id: this.id, on, kind: this.kind });
|
|
1451
2009
|
}
|
|
1452
2010
|
|
|
1453
2011
|
setConn(st) {
|
|
@@ -1499,10 +2057,14 @@ class Tile {
|
|
|
1499
2057
|
reroute(force) {
|
|
1500
2058
|
const call = this.call;
|
|
1501
2059
|
const pool = call.liftPool;
|
|
2060
|
+
const content = this.kind === 'content';
|
|
1502
2061
|
const r =
|
|
1503
2062
|
this.forceFlat && this.format === 'sbs'
|
|
1504
2063
|
? { route: 'flat-left' }
|
|
1505
|
-
:
|
|
2064
|
+
: content
|
|
2065
|
+
? // Shared content: woven when it is a pair on a woven wall, else flat. Never lifted.
|
|
2066
|
+
{ route: this.format === 'sbs' ? (call.woven ? 'woven-sbs' : 'flat-left') : 'flat' }
|
|
2067
|
+
: wire.routeFor({
|
|
1506
2068
|
format: this.format,
|
|
1507
2069
|
woven: call.woven,
|
|
1508
2070
|
mono3D: call.mono3DOn ? 'auto' : 'off',
|
|
@@ -1511,9 +2073,9 @@ class Tile {
|
|
|
1511
2073
|
failed: this.liftFailed,
|
|
1512
2074
|
});
|
|
1513
2075
|
// First mono peer on a woven wall and lift not asked for yet (e.g. setMono3D(true) later).
|
|
1514
|
-
if (this.format !== 'sbs' && call.woven && call.mono3DOn && !call._liftP) call._ensureLift();
|
|
2076
|
+
if (!content && this.format !== 'sbs' && call.woven && call.mono3DOn && !call._liftP) call._ensureLift();
|
|
1515
2077
|
// A mono peer on a 3D display that stays flat because the lift MODULE is missing: say so.
|
|
1516
|
-
if (this.format !== 'sbs' && call.woven && call.liftMissing) call._warnLiftMissing();
|
|
2078
|
+
if (!content && this.format !== 'sbs' && call.woven && call.liftMissing) call._warnLiftMissing();
|
|
1517
2079
|
const key = `${r.route}|${this.format}`;
|
|
1518
2080
|
if (!force && key === this.routedKey) return;
|
|
1519
2081
|
this.routedKey = key;
|
|
@@ -1521,10 +2083,10 @@ class Tile {
|
|
|
1521
2083
|
this.route = r.route;
|
|
1522
2084
|
if (r.route === 'woven-sbs') this._registerWoven();
|
|
1523
2085
|
else if (r.route === 'lifted') this._registerLifted();
|
|
1524
|
-
call.log('route', { peer: this.id, route: r.route, format: this.format, mono3d: r.mono3d || null });
|
|
1525
|
-
call.emit('display', { id: this.id, display: this.display() });
|
|
2086
|
+
call.log('route', { peer: this.id, kind: this.kind, route: r.route, format: this.format, mono3d: r.mono3d || null });
|
|
2087
|
+
call.emit('display', { id: this.id, display: this.display(), kind: this.kind });
|
|
1526
2088
|
// 1.29's 'format' event (route / mono3d detail → diagnostics()): one release of compatibility.
|
|
1527
|
-
if (call.listeners.get('format')?.size) call.emit('format', { id: this.id, format: this.format, route: r.route, mono3d: r.mono3d || null, hello: this.hello });
|
|
2089
|
+
if (!content && call.listeners.get('format')?.size) call.emit('format', { id: this.id, format: this.format, route: r.route, mono3d: r.mono3d || null, hello: this.hello });
|
|
1528
2090
|
this._renderBadge();
|
|
1529
2091
|
}
|
|
1530
2092
|
|
|
@@ -1533,12 +2095,18 @@ class Tile {
|
|
|
1533
2095
|
if (!this.convCanvas) this.convCanvas = document.createElement('canvas');
|
|
1534
2096
|
this._paintConv(); // the first frame exists before the layer does
|
|
1535
2097
|
show(this.cover, true);
|
|
1536
|
-
|
|
2098
|
+
// tileCover 'auto': the core's rect cover takes over after the first join, across every later
|
|
2099
|
+
// resize / move (layout: 'none' — the page moves the tiles). No cover machinery here.
|
|
2100
|
+
const handle = call.o.tileCover === 'auto' ? call.wall.addImage(this.canvas, this.convCanvas, { rectCover: 'auto' }) : call.wall.addImage(this.canvas, this.convCanvas);
|
|
1537
2101
|
this.handle = handle;
|
|
2102
|
+
this._fwSettled = false;
|
|
2103
|
+
this._rwP = null;
|
|
1538
2104
|
handle.firstWoven?.then((res) => {
|
|
1539
2105
|
if (this.handle !== handle) return;
|
|
2106
|
+
this._fwSettled = true;
|
|
1540
2107
|
show(this.cover, false); // a cut, never a fade
|
|
1541
|
-
call.log('first-woven', { peer: this.id, woven: res.woven, reason: res.reason, ms: Math.round(res.ms) });
|
|
2108
|
+
call.log('first-woven', { peer: this.id, kind: this.kind, woven: res.woven, reason: res.reason, ms: Math.round(res.ms) });
|
|
2109
|
+
call.emit('weave', weaveEvent(this.id, res, this.kind));
|
|
1542
2110
|
if (!res.woven && res.reason === 'layer-failed') this._layerFailed();
|
|
1543
2111
|
});
|
|
1544
2112
|
}
|
|
@@ -1685,6 +2253,9 @@ class Tile {
|
|
|
1685
2253
|
|
|
1686
2254
|
paint() {
|
|
1687
2255
|
if (!this.route) return;
|
|
2256
|
+
// Hidden (display:none) or collapsed: nothing to see, so no copy, no measurement. Known from the
|
|
2257
|
+
// ResizeObserver; without one (or before it reports) the tile paints as before.
|
|
2258
|
+
if (this._box && (!this._box.w || !this._box.h)) return;
|
|
1688
2259
|
if (this.route !== 'woven-sbs') this.cursor?.idle(); // only a woven pair has depth to measure
|
|
1689
2260
|
if (this.route === 'woven-sbs') this._paintConv();
|
|
1690
2261
|
else if (this.route === 'lifted') {
|
|
@@ -1699,7 +2270,7 @@ class Tile {
|
|
|
1699
2270
|
const H = v.videoHeight;
|
|
1700
2271
|
if (!W || !H || (v.readyState || 0) < 2 || !this.convCanvas) return;
|
|
1701
2272
|
const eyeW = W / 2;
|
|
1702
|
-
const A = this.call.o.tileAspect;
|
|
2273
|
+
const A = this.call.o.tileAspect === 'fill' ? this._aspect || 16 / 9 : this.call.o.tileAspect;
|
|
1703
2274
|
const { w: outW, h: outH } = eyeOutputSize(eyeW, H, A);
|
|
1704
2275
|
const c = this.convCanvas;
|
|
1705
2276
|
if (c.width !== 2 * outW || c.height !== outH) {
|
|
@@ -1707,7 +2278,18 @@ class Tile {
|
|
|
1707
2278
|
c.height = outH;
|
|
1708
2279
|
}
|
|
1709
2280
|
const hover = !!(this.cursor && this.cursor.hovering);
|
|
1710
|
-
if (this.
|
|
2281
|
+
if (this.kind === 'content') {
|
|
2282
|
+
// DISPLAY-REFERRED (3D app share): the sender's display plane sits at a known disparity, so
|
|
2283
|
+
// that IS the convergence — applied directly, never measured (the depth slider still adds).
|
|
2284
|
+
this.conv.measuredPx = this.contentFrac() * eyeW;
|
|
2285
|
+
} else {
|
|
2286
|
+
// The sender's measurement, when it sends one (recomputed per paint: a WebRTC resolution change
|
|
2287
|
+
// keeps it right, since it is a fraction of the eye width). Otherwise measure here, as before.
|
|
2288
|
+
const auto = this.call.o.autoConverge;
|
|
2289
|
+
const frac = auto ? this._senderFrac(performance.now()) : null;
|
|
2290
|
+
if (frac !== null) this.conv.measuredPx = frac * eyeW;
|
|
2291
|
+
if (hover || (auto && frac === null)) this._sampleDisparity(v, W, H, hover);
|
|
2292
|
+
}
|
|
1711
2293
|
const shift = this.conv.step(eyeW);
|
|
1712
2294
|
const g = c.getContext('2d');
|
|
1713
2295
|
const rects = [];
|
|
@@ -1729,61 +2311,20 @@ class Tile {
|
|
|
1729
2311
|
// ~15 Hz while a depth cursor hovers: it measures the same copy, under the pointer.
|
|
1730
2312
|
if (this.autoPending || now - this.autoAt < (hover ? CURSOR_SAMPLE_MS : AUTO_CONV_INTERVAL_MS)) return;
|
|
1731
2313
|
this.autoAt = now;
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
// readback, not the matching, was ~25 ms of a ~30 ms measurement).
|
|
1735
|
-
if (typeof VideoFrame === 'function') {
|
|
1736
|
-
let frame = null;
|
|
1737
|
-
try {
|
|
1738
|
-
frame = new VideoFrame(v);
|
|
1739
|
-
} catch {
|
|
1740
|
-
frame = null;
|
|
1741
|
-
}
|
|
1742
|
-
if (frame && Y_PLANE_FORMATS.has(frame.format)) {
|
|
1743
|
-
this.autoPending = true;
|
|
1744
|
-
const gen = this.autoGen;
|
|
1745
|
-
const fw = frame.visibleRect ? frame.visibleRect.width : frame.displayWidth;
|
|
1746
|
-
const fh = frame.visibleRect ? frame.visibleRect.height : frame.displayHeight;
|
|
1747
|
-
const buf = new Uint8Array(frame.allocationSize());
|
|
1748
|
-
frame
|
|
1749
|
-
.copyTo(buf)
|
|
1750
|
-
.then((layout) => {
|
|
1751
|
-
if (gen !== this.autoGen) return; // the stream changed or the tile is gone: stale frame
|
|
1752
|
-
const t0 = performance.now();
|
|
1753
|
-
const f = Math.max(1, Math.ceil(fw / 2 / AUTO_CONV_EYE_WIDTH));
|
|
1754
|
-
const ds = downsampleLuma(buf, layout[0].offset, layout[0].stride, fw, fh, f);
|
|
1755
|
-
this._applyMeasurement(ds.img, ds.w, ds.h, ds.w / fw, now, performance.now() - t0, 'videoframe');
|
|
1756
|
-
})
|
|
1757
|
-
.catch(() => {
|
|
1758
|
-
if (gen === this.autoGen) this._applyMeasurement(null, 0, 0, 1, now, 0, 'videoframe');
|
|
1759
|
-
})
|
|
1760
|
-
.finally(() => {
|
|
1761
|
-
frame.close();
|
|
1762
|
-
this.autoPending = false;
|
|
1763
|
-
});
|
|
1764
|
-
return;
|
|
1765
|
-
}
|
|
1766
|
-
if (frame) frame.close();
|
|
1767
|
-
}
|
|
1768
|
-
// Fallback: a small canvas + getImageData (synchronous readback).
|
|
1769
|
-
const s = Math.min(1, AUTO_CONV_EYE_WIDTH / (W / 2));
|
|
1770
|
-
const w = Math.max(2, Math.round((W * s) / 2) * 2);
|
|
1771
|
-
const h = Math.max(2, Math.round(H * s));
|
|
2314
|
+
this.autoPending = true;
|
|
2315
|
+
const gen = this.autoGen;
|
|
1772
2316
|
if (!this.autoCanvas) this.autoCanvas = document.createElement('canvas');
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
img = null; // a frame that cannot be read (tainted, not decoded yet): skip it
|
|
1785
|
-
}
|
|
1786
|
-
this._applyMeasurement(img, w, h, w / W, now, performance.now() - now, 'canvas');
|
|
2317
|
+
// camera/conv-meter.js: WebCodecs Y plane (async copyTo, no canvas readback) when the decoded
|
|
2318
|
+
// frame has one, else a small canvas + getImageData.
|
|
2319
|
+
grabLumaFrame(v, { eyeMax: AUTO_CONV_EYE_WIDTH, scratch: this.autoCanvas })
|
|
2320
|
+
.then((g) => {
|
|
2321
|
+
if (gen !== this.autoGen) return; // the stream changed or the tile is gone: stale frame
|
|
2322
|
+
if (g) this._applyMeasurement(g.img, g.w, g.h, g.scale, now, g.readMs, g.via);
|
|
2323
|
+
else this._applyMeasurement(null, 0, 0, 1, now, 0, this.autoStats.via);
|
|
2324
|
+
})
|
|
2325
|
+
.finally(() => {
|
|
2326
|
+
this.autoPending = false;
|
|
2327
|
+
});
|
|
1787
2328
|
}
|
|
1788
2329
|
|
|
1789
2330
|
/**
|
|
@@ -1793,12 +2334,16 @@ class Tile {
|
|
|
1793
2334
|
*/
|
|
1794
2335
|
_applyMeasurement(img, w, h, s, at, readMs, via) {
|
|
1795
2336
|
if (this.destroyed) return;
|
|
2337
|
+
this.autoGrabs++;
|
|
1796
2338
|
if (img) this.luma = { img, w, h, scale: s };
|
|
1797
2339
|
if (!this.call.o.autoConverge) return; // the copy was for the depth cursor only
|
|
2340
|
+
if (this._senderFrac(performance.now()) !== null) return; // the sender's measurement wins; the copy was for the cursor
|
|
1798
2341
|
const t = performance.now();
|
|
1799
2342
|
const m = img ? this.autoFocus.measure(img, w, h, at) : null;
|
|
1800
2343
|
const d = this.autoTrack.push(m ? m.d / s : null);
|
|
1801
|
-
|
|
2344
|
+
// A failed measurement holds: the track holds its last value, and with an empty track (the
|
|
2345
|
+
// sender's `conv` just went stale) the sender's last value stays until this side locks.
|
|
2346
|
+
if (d !== null) this.conv.measuredPx = d;
|
|
1802
2347
|
this.autoStats.n++;
|
|
1803
2348
|
this.autoStats.readMs += readMs;
|
|
1804
2349
|
this.autoStats.ms += readMs + (performance.now() - t);
|
|
@@ -1820,6 +2365,7 @@ class Tile {
|
|
|
1820
2365
|
|
|
1821
2366
|
_resetAutoConverge() {
|
|
1822
2367
|
this.autoGen++;
|
|
2368
|
+
this.senderConv = null;
|
|
1823
2369
|
this.luma = null;
|
|
1824
2370
|
this.autoTrack.reset();
|
|
1825
2371
|
this.autoFocus.reset();
|
|
@@ -1837,10 +2383,74 @@ class Tile {
|
|
|
1837
2383
|
clearTimeout(this.leaveTimer);
|
|
1838
2384
|
clearTimeout(this.liftTimer);
|
|
1839
2385
|
this._unregister();
|
|
2386
|
+
this._unobserve?.();
|
|
2387
|
+
this._unobserve = null;
|
|
1840
2388
|
this.video.srcObject = null;
|
|
1841
|
-
this.audio.srcObject = null;
|
|
2389
|
+
if (this.audio) this.audio.srcObject = null;
|
|
1842
2390
|
this.el.remove();
|
|
1843
2391
|
}
|
|
2392
|
+
|
|
2393
|
+
/** Overridden by ContentTile; a camera tile has no known convergence. */
|
|
2394
|
+
contentFrac() {
|
|
2395
|
+
return 0;
|
|
2396
|
+
}
|
|
2397
|
+
}
|
|
2398
|
+
|
|
2399
|
+
// ── shared content (3D app share) ─────────────────────────────────────────────────────────
|
|
2400
|
+
|
|
2401
|
+
/**
|
|
2402
|
+
* What a participant shares (`shareContent`): a Tile with `kind: 'content'` — the same detached
|
|
2403
|
+
* <video>, woven canvas, cover, layer retries, session recovery and #172 re-registration as a
|
|
2404
|
+
* camera tile, but DISPLAY-REFERRED: `convergence.frac` from the `content` message is the shift
|
|
2405
|
+
* (half of it per eye, like a measured disparity), nothing is sampled, the sender's `conv` does not
|
|
2406
|
+
* apply, and only the depth slider adds to it. An unknown convergence is shown as sent (0). No
|
|
2407
|
+
* audio, no depth cursor, never lifted. `id: 'self'` is the optional preview of your own share.
|
|
2408
|
+
*/
|
|
2409
|
+
class ContentTile extends Tile {
|
|
2410
|
+
constructor(call, id) {
|
|
2411
|
+
super(call, id, 'content');
|
|
2412
|
+
this.desc = null;
|
|
2413
|
+
this.conn = 'connected';
|
|
2414
|
+
this._renderPlate();
|
|
2415
|
+
}
|
|
2416
|
+
|
|
2417
|
+
/** A `content` message (or this side's own descriptor, for the self preview). */
|
|
2418
|
+
setContent(cm) {
|
|
2419
|
+
const changed = this.format !== cm.format;
|
|
2420
|
+
this.desc = cm;
|
|
2421
|
+
this.format = cm.format;
|
|
2422
|
+
this.content = { on: true, format: cm.format, appName: cm.appName || null };
|
|
2423
|
+
if (changed) this._maybeRoute();
|
|
2424
|
+
this._renderBadge();
|
|
2425
|
+
}
|
|
2426
|
+
|
|
2427
|
+
contentFrac() {
|
|
2428
|
+
const c = this.desc && this.desc.convergence;
|
|
2429
|
+
return c && c.known && typeof c.frac === 'number' && Number.isFinite(c.frac) ? c.frac : 0;
|
|
2430
|
+
}
|
|
2431
|
+
|
|
2432
|
+
_renderBadge() {
|
|
2433
|
+
super._renderBadge();
|
|
2434
|
+
const name = this.desc && this.desc.appName;
|
|
2435
|
+
if (name && this.route) this.badge.replaceChildren(el('b', { text: `${name} · ${this.display() === '3D' ? this.call.t('badge3D') : this.call.t('badge2D')}` }));
|
|
2436
|
+
}
|
|
2437
|
+
|
|
2438
|
+
diag() {
|
|
2439
|
+
return {
|
|
2440
|
+
id: this.id,
|
|
2441
|
+
kind: 'content',
|
|
2442
|
+
route: this.route,
|
|
2443
|
+
display: this.display(),
|
|
2444
|
+
format: this.format,
|
|
2445
|
+
appName: this.desc ? this.desc.appName : null,
|
|
2446
|
+
width: this.desc ? this.desc.width : null,
|
|
2447
|
+
height: this.desc ? this.desc.height : null,
|
|
2448
|
+
convergence: this.desc ? this.desc.convergence : null,
|
|
2449
|
+
convergencePx: this.conv.current,
|
|
2450
|
+
samples: this.autoGrabs, // always 0: content is never measured
|
|
2451
|
+
element: this.el,
|
|
2452
|
+
};
|
|
2453
|
+
}
|
|
1844
2454
|
}
|
|
1845
2455
|
|
|
1846
2456
|
// ── the self view ─────────────────────────────────────────────────────────────────────────
|
|
@@ -1859,9 +2469,13 @@ class SelfTile {
|
|
|
1859
2469
|
this.badge = el('div', { class: 'dxr-call-badge', part: 'badge' });
|
|
1860
2470
|
this.plate = el('div', { class: 'dxr-call-state dxr-call-hidden', part: 'plate' });
|
|
1861
2471
|
this.stage = el('div', { class: 'dxr-call-stage' }, [this.canvas, this.badge, this.plate]);
|
|
1862
|
-
this.stage.style.aspectRatio = String(call.o.tileAspect);
|
|
1863
|
-
|
|
2472
|
+
if (call.o.tileAspect !== 'fill') this.stage.style.aspectRatio = String(call.o.tileAspect);
|
|
2473
|
+
// `data-dxr-peer="self"`: handle.tile('self'), and the page's CSS under layout: 'none'.
|
|
2474
|
+
this.el = el('div', { class: 'dxr-call-tile dxr-call-tile--self', part: 'tile', 'data-dxr-peer': 'self' }, [this.stage, el('div', { class: 'dxr-call-talk' })]);
|
|
1864
2475
|
slot.appendChild(this.el);
|
|
2476
|
+
this._box = null;
|
|
2477
|
+
this._rwP = null;
|
|
2478
|
+
this._unobserve = observeBox(this.stage, (w, h) => this._onBox(w, h));
|
|
1865
2479
|
this.view = null; // the CameraView over the current camera
|
|
1866
2480
|
this.cam = null;
|
|
1867
2481
|
}
|
|
@@ -1884,7 +2498,7 @@ class SelfTile {
|
|
|
1884
2498
|
const l = this.call.local;
|
|
1885
2499
|
const cam = l && l.videoTrack ? l.cam : null;
|
|
1886
2500
|
const wall = this.call.woven ? this.call.wall : { supported: false };
|
|
1887
|
-
if (!cam || cam.state !== 'live') {
|
|
2501
|
+
if (!cam || cam.state !== 'live' || !this.call.selfViewOn) {
|
|
1888
2502
|
this.detach();
|
|
1889
2503
|
return this._badge();
|
|
1890
2504
|
}
|
|
@@ -1899,16 +2513,48 @@ class SelfTile {
|
|
|
1899
2513
|
autoConverge: !!this.call.o.autoConverge, // the call option (default on) governs the self view too
|
|
1900
2514
|
cursor: this.call.o.cursor || undefined, // and so does `cursor: 'depth'`
|
|
1901
2515
|
aspect: this.call.o.tileAspect,
|
|
2516
|
+
...(this.call.o.tileCover === 'auto' ? { rectCover: 'auto' } : {}),
|
|
1902
2517
|
onRouteChange: (route, st) => {
|
|
1903
2518
|
if (this.cam !== cam) return; // a view this tile already replaced
|
|
1904
2519
|
this.call.log('self-route', { route, woven: st.woven, reason: st.reason });
|
|
1905
2520
|
this._badge();
|
|
2521
|
+
this._weave(st);
|
|
1906
2522
|
},
|
|
1907
2523
|
});
|
|
1908
2524
|
}
|
|
1909
2525
|
this._badge();
|
|
1910
2526
|
}
|
|
1911
2527
|
|
|
2528
|
+
/**
|
|
2529
|
+
* The self view's `weave` event (`id: 'self'`): its tile's `firstWoven` settled (woven or not),
|
|
2530
|
+
* or the view dropped flat because the wall said the tile will not weave.
|
|
2531
|
+
*/
|
|
2532
|
+
_weave(st) {
|
|
2533
|
+
const fw = st.firstWoven;
|
|
2534
|
+
let ev = null;
|
|
2535
|
+
if (fw && typeof fw === 'object') ev = weaveEvent('self', fw);
|
|
2536
|
+
else if (st.route !== 'woven-sbs' && st.reason) ev = { id: 'self', state: 'flat', reason: st.reason, ms: 0, kind: 'camera' };
|
|
2537
|
+
if (ev) this.call.emit('weave', ev);
|
|
2538
|
+
}
|
|
2539
|
+
|
|
2540
|
+
/** The self stage resized: once woven, the `weave` event of its re-join. */
|
|
2541
|
+
_onBox(w, h) {
|
|
2542
|
+
const prev = this._box;
|
|
2543
|
+
this._box = { w, h };
|
|
2544
|
+
if (!prev || !w || !h || (prev.w === w && prev.h === h)) return;
|
|
2545
|
+
const handle = this.view && this.view.woven ? this.view.handle : null;
|
|
2546
|
+
const fw = this.view ? this.view.weaveState().firstWoven : null;
|
|
2547
|
+
if (!handle || !fw || typeof fw !== 'object' || typeof handle.rewoven !== 'function') return;
|
|
2548
|
+
const p = handle.rewoven();
|
|
2549
|
+
if (p === this._rwP || !p || typeof p.then !== 'function') return;
|
|
2550
|
+
this._rwP = p;
|
|
2551
|
+
p.then((res) => {
|
|
2552
|
+
if (this._rwP !== p || !this.view || this.view.handle !== handle || !res) return;
|
|
2553
|
+
this._rwP = null;
|
|
2554
|
+
this.call.emit('weave', weaveEvent('self', res));
|
|
2555
|
+
});
|
|
2556
|
+
}
|
|
2557
|
+
|
|
1912
2558
|
/** 3D only while the view is on the woven route — a layer the wall failed has already taken it flat. */
|
|
1913
2559
|
_badge() {
|
|
1914
2560
|
const t = this.call.t;
|