@displayxr/inline3d 1.48.0 → 1.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +55 -0
- package/call.d.ts +107 -7
- package/camera.d.ts +16 -8
- package/dist/call.js +9 -6
- package/js/call/element.js +3 -1
- package/js/call/options.js +2 -0
- package/js/call/share.js +147 -0
- package/js/call/transport.js +150 -9
- package/js/call/ui.js +3 -0
- package/js/call/wire.js +65 -1
- package/js/camera/app-share.js +54 -0
- package/js/camera/capture.js +100 -23
- package/js/inline3d-call.js +361 -36
- package/js/inline3d-camera.js +9 -5
- package/package.json +1 -1
package/js/call/element.js
CHANGED
|
@@ -37,7 +37,7 @@ import { mountCall, dxrSignaling } from '../inline3d-call.js';
|
|
|
37
37
|
|
|
38
38
|
/** Every call event is re-dispatched as `dxr-call:<type>`; `dxr-call:ready` carries the handle. */
|
|
39
39
|
export const CALL_EVENT_PREFIX = 'dxr-call:';
|
|
40
|
-
const CALL_EVENTS = ['joined', 'left', 'peer', 'peerleft', 'state', 'display', 'speaker', 'quality', 'error', 'warning', 'name', 'message', 'signal', 'weave', 'speaking'];
|
|
40
|
+
const CALL_EVENTS = ['joined', 'left', 'peer', 'peerleft', 'state', 'display', 'speaker', 'quality', 'error', 'warning', 'name', 'message', 'signal', 'weave', 'speaking', 'content'];
|
|
41
41
|
const TAG = '[inline3d/call] <dxr-call>';
|
|
42
42
|
|
|
43
43
|
/** A boolean attribute: present is true, unless its value spells "false"/"0"/"off"/"no". */
|
|
@@ -70,6 +70,7 @@ const str = (v) => (v === null || v === undefined || String(v).trim() === '' ? u
|
|
|
70
70
|
* | `mono3d="off"` | `mono3D: 'off'` |
|
|
71
71
|
* | `no-audio` | `audio: false` |
|
|
72
72
|
* | `no-self-view` | `selfView: false` |
|
|
73
|
+
* | `self-content-view` | `selfContentView: true` |
|
|
73
74
|
* | `no-auto-converge` | `autoConverge: false` |
|
|
74
75
|
* | `no-send-convergence`| `sendConvergence: false` |
|
|
75
76
|
* | `cursor="depth"` | `cursor: 'depth'` |
|
|
@@ -103,6 +104,7 @@ export function attrsToOpts(source) {
|
|
|
103
104
|
if (m3 && /^(off|false|0|no)$/i.test(m3)) o.mono3D = 'off';
|
|
104
105
|
if (isOn(get('no-audio'))) o.audio = false;
|
|
105
106
|
if (isOn(get('no-self-view'))) o.selfView = false;
|
|
107
|
+
if (isOn(get('self-content-view'))) o.selfContentView = true;
|
|
106
108
|
if (isOn(get('no-auto-converge'))) o.autoConverge = false;
|
|
107
109
|
if (isOn(get('no-send-convergence'))) o.sendConvergence = false;
|
|
108
110
|
if (str(get('cursor')) === 'depth') o.cursor = 'depth';
|
package/js/call/options.js
CHANGED
|
@@ -123,6 +123,8 @@ export function normalizeCallOptions(opts = {}) {
|
|
|
123
123
|
chrome,
|
|
124
124
|
autoJoin: opts.autoJoin === undefined ? !chrome : !!opts.autoJoin,
|
|
125
125
|
selfView: opts.selfView === undefined ? true : !!opts.selfView,
|
|
126
|
+
// A woven preview of what this side shares (`shareContent`), as a content tile with id 'self'.
|
|
127
|
+
selfContentView: !!opts.selfContentView,
|
|
126
128
|
tileAspect: aspect,
|
|
127
129
|
// theme → the CSS custom properties written on the host, and the string table.
|
|
128
130
|
cssVars,
|
package/js/call/share.js
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
// call/share.js — 3D screen share, the page side: find the running DisplayXR apps the browser
|
|
2
|
+
// offers as share sources, and open one as a side-by-side content track for `handle.shareContent()`.
|
|
3
|
+
//
|
|
4
|
+
// PREVIEW tier (docs/sdk-stability.md). The device contract is runtime ADR-061 /
|
|
5
|
+
// `XR_DXR_app_share`: the DisplayXR Browser lists each shareable app as a `videoinput` labelled
|
|
6
|
+
// `DisplayXR: <app> (3D)` with a `dxr-app:` device id, and the opened track describes itself in
|
|
7
|
+
// `getSettings().displayxrStereo` (`source: 'app'`, `appName`, `zeroDisparityShiftFrac`). Browsers
|
|
8
|
+
// that predate those hint fields still use the label and the id, so detection takes any one of the
|
|
9
|
+
// three (camera/app-share.js) and a missing `zeroDisparityShiftFrac` reads as 0 — which is what the
|
|
10
|
+
// runtime sends in v1 anyway: the views are DISPLAY-REFERRED (zero disparity at the sender's display
|
|
11
|
+
// plane), so a receiver applies that known value and never measures the frames.
|
|
12
|
+
//
|
|
13
|
+
// App devices are listed only once the page holds camera permission (ADR-061 decision 17). The
|
|
14
|
+
// runtime asks the user on the SENDER's machine, every start ("Share <app> with <browser>?"), when
|
|
15
|
+
// `openAppShare()` opens the device; a Deny rejects it (`code: 'share-denied'`). The track ends
|
|
16
|
+
// (`ended`) when the app exits or the share is stopped from the runtime side.
|
|
17
|
+
|
|
18
|
+
import { isAppShareDevice, appNameFromLabel, APP_SHARE_ID_PREFIX, APP_SHARE_LABEL_RE } from '../camera/app-share.js';
|
|
19
|
+
import { sanitizeName, CONV_MAX_FRAC } from './wire.js';
|
|
20
|
+
|
|
21
|
+
export { isAppShareDevice, appNameFromLabel, APP_SHARE_ID_PREFIX, APP_SHARE_LABEL_RE };
|
|
22
|
+
|
|
23
|
+
/** What `openAppShare` asks for by default: the runtime's default stream (ADR-061 decision 5). */
|
|
24
|
+
const SHARE_CONSTRAINTS = Object.freeze({ width: 2560, height: 720, frameRate: 30 });
|
|
25
|
+
|
|
26
|
+
const mdOf = (o) => (o && o.mediaDevices) || (globalThis.navigator && navigator.mediaDevices) || null;
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* The running apps this browser offers to share: `[{ deviceId, label, appName }]`, in the
|
|
30
|
+
* browser's order. Empty without camera permission (the browser lists none before it), on any
|
|
31
|
+
* other browser, or when nothing is shareable. Never rejects.
|
|
32
|
+
* @param {{ mediaDevices?: MediaDevices }} [o]
|
|
33
|
+
*/
|
|
34
|
+
export async function listShareableApps(o = {}) {
|
|
35
|
+
const md = mdOf(o);
|
|
36
|
+
if (!md || typeof md.enumerateDevices !== 'function') return [];
|
|
37
|
+
let list = [];
|
|
38
|
+
try {
|
|
39
|
+
list = await md.enumerateDevices();
|
|
40
|
+
} catch {
|
|
41
|
+
return [];
|
|
42
|
+
}
|
|
43
|
+
return list
|
|
44
|
+
.filter((d) => d && d.kind === 'videoinput' && isAppShareDevice(d))
|
|
45
|
+
.map((d) => Object.freeze({ deviceId: d.deviceId, label: d.label || '', appName: sanitizeName(appNameFromLabel(d.label)) || sanitizeName(d.label) || null }));
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* `cb(apps)` now and whenever the list changes (`devicechange`: an app started, exited, or the
|
|
50
|
+
* page was just granted the camera). Returns the unsubscribe.
|
|
51
|
+
* @param {(apps: Array<{deviceId: string, label: string, appName: string|null}>) => void} cb
|
|
52
|
+
* @param {{ mediaDevices?: MediaDevices }} [o]
|
|
53
|
+
*/
|
|
54
|
+
export function watchShareableApps(cb, o = {}) {
|
|
55
|
+
if (typeof cb !== 'function') throw new TypeError('@displayxr/inline3d/call: watchShareableApps(cb) takes a function');
|
|
56
|
+
const md = mdOf(o);
|
|
57
|
+
let last = null;
|
|
58
|
+
let live = true;
|
|
59
|
+
let seq = 0;
|
|
60
|
+
const run = async () => {
|
|
61
|
+
const mine = ++seq;
|
|
62
|
+
const apps = await listShareableApps({ mediaDevices: md });
|
|
63
|
+
if (!live || mine !== seq) return; // a newer read is in flight (devicechange bursts)
|
|
64
|
+
const key = JSON.stringify(apps.map((a) => [a.deviceId, a.label]));
|
|
65
|
+
if (key === last) return;
|
|
66
|
+
last = key;
|
|
67
|
+
try {
|
|
68
|
+
cb(apps);
|
|
69
|
+
} catch (err) {
|
|
70
|
+
console.error('[inline3d/call] watchShareableApps listener threw', err);
|
|
71
|
+
}
|
|
72
|
+
};
|
|
73
|
+
const onChange = () => void run();
|
|
74
|
+
if (md && typeof md.addEventListener === 'function') md.addEventListener('devicechange', onChange);
|
|
75
|
+
void run();
|
|
76
|
+
return () => {
|
|
77
|
+
live = false;
|
|
78
|
+
if (md && typeof md.removeEventListener === 'function') md.removeEventListener('devicechange', onChange);
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* The content descriptor of an opened share track (what `shareContent` sends): side-by-side, its
|
|
84
|
+
* size, the app name, and the KNOWN convergence — `frac` is the hint's `zeroDisparityShiftFrac`
|
|
85
|
+
* (the disparity of the sender's display plane as a fraction of the per-eye width; 0 in v1, and 0
|
|
86
|
+
* when the browser does not expose it yet). Pure.
|
|
87
|
+
* @param {MediaStreamTrack} track
|
|
88
|
+
* @param {string} [fallbackLabel]
|
|
89
|
+
*/
|
|
90
|
+
export function describeAppShare(track, fallbackLabel = '') {
|
|
91
|
+
let st = {};
|
|
92
|
+
try {
|
|
93
|
+
st = (track && typeof track.getSettings === 'function' && track.getSettings()) || {};
|
|
94
|
+
} catch {
|
|
95
|
+
st = {};
|
|
96
|
+
}
|
|
97
|
+
const hint = st.displayxrStereo && typeof st.displayxrStereo === 'object' ? st.displayxrStereo : {};
|
|
98
|
+
const z = hint.zeroDisparityShiftFrac;
|
|
99
|
+
const frac = typeof z === 'number' && Number.isFinite(z) ? Math.max(-CONV_MAX_FRAC, Math.min(CONV_MAX_FRAC, z)) : 0;
|
|
100
|
+
const appName = sanitizeName(hint.appName) || sanitizeName(appNameFromLabel(track && track.label)) || sanitizeName(appNameFromLabel(fallbackLabel)) || null;
|
|
101
|
+
return {
|
|
102
|
+
kind: 'content',
|
|
103
|
+
format: hint.layout && hint.layout !== 'side-by-side' ? 'mono' : 'sbs',
|
|
104
|
+
width: st.width > 0 ? st.width : 0,
|
|
105
|
+
height: st.height > 0 ? st.height : 0,
|
|
106
|
+
convergence: { known: true, frac },
|
|
107
|
+
appName,
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Open one shareable app (a `deviceId` from `listShareableApps`) as a side-by-side track. This is
|
|
113
|
+
* when the runtime asks the user on this machine; Deny rejects with `code: 'share-denied'`. A device
|
|
114
|
+
* that turns out not to be an app share is closed again (`code: 'not-app-share'`). The track is
|
|
115
|
+
* yours: stop it when you stop sharing (`track.stop()`), or the runtime keeps the share open.
|
|
116
|
+
*
|
|
117
|
+
* @param {string} deviceId
|
|
118
|
+
* @param {{ width?: number, height?: number, frameRate?: number, mediaDevices?: MediaDevices }} [o]
|
|
119
|
+
* @returns {Promise<{ stream: MediaStream, track: MediaStreamTrack, descriptor: ReturnType<typeof describeAppShare> }>}
|
|
120
|
+
*/
|
|
121
|
+
export async function openAppShare(deviceId, o = {}) {
|
|
122
|
+
if (typeof deviceId !== 'string' || !deviceId) throw Object.assign(new TypeError('@displayxr/inline3d/call: openAppShare(deviceId) needs a deviceId from listShareableApps()'), { code: 'not-app-share' });
|
|
123
|
+
const md = mdOf(o);
|
|
124
|
+
if (!md || typeof md.getUserMedia !== 'function') throw Object.assign(new Error('getUserMedia is not available (insecure context?)'), { code: 'share-failed' });
|
|
125
|
+
const want = (v, d) => ({ ideal: Number.isFinite(v) && v > 0 ? v : d });
|
|
126
|
+
let stream;
|
|
127
|
+
try {
|
|
128
|
+
stream = await md.getUserMedia({
|
|
129
|
+
video: { deviceId: { exact: deviceId }, width: want(o.width, SHARE_CONSTRAINTS.width), height: want(o.height, SHARE_CONSTRAINTS.height), frameRate: want(o.frameRate, SHARE_CONSTRAINTS.frameRate) },
|
|
130
|
+
audio: false,
|
|
131
|
+
});
|
|
132
|
+
} catch (err) {
|
|
133
|
+
const denied = !!err && (err.name === 'NotAllowedError' || err.name === 'PermissionDeniedError' || err.name === 'SecurityError');
|
|
134
|
+
throw Object.assign(new Error(`could not share the app: ${(err && err.message) || err}`), { code: denied ? 'share-denied' : 'share-failed', cause: err });
|
|
135
|
+
}
|
|
136
|
+
const track = stream.getVideoTracks()[0] || null;
|
|
137
|
+
if (!track || !(isAppShareDevice(track) || deviceId.startsWith(APP_SHARE_ID_PREFIX))) {
|
|
138
|
+
stream.getTracks().forEach((t) => t.stop());
|
|
139
|
+
throw Object.assign(new Error('that device is not a shared DisplayXR app'), { code: 'not-app-share' });
|
|
140
|
+
}
|
|
141
|
+
try {
|
|
142
|
+
track.contentHint = 'detail'; // resolution over frame rate: halving an SBS width halves each eye
|
|
143
|
+
} catch {
|
|
144
|
+
/* older engines */
|
|
145
|
+
}
|
|
146
|
+
return { stream, track, descriptor: describeAppShare(track) };
|
|
147
|
+
}
|
package/js/call/transport.js
CHANGED
|
@@ -6,7 +6,8 @@
|
|
|
6
6
|
//
|
|
7
7
|
// transport.start({ room, localStream, onPeer, onPeerLeft, onStream, onMessage, onPeerState })
|
|
8
8
|
// transport.send(peerId, msg) / broadcast(msg)
|
|
9
|
-
// transport.replaceTrack(kind, track, stream) / setSendFormat(format)
|
|
9
|
+
// transport.replaceTrack(kind, track, stream, { slot }) / setSendFormat(format)
|
|
10
|
+
// transport.addContentTransceiver(peerId) // the content track (3D app share), see below
|
|
10
11
|
// transport.peerConnection(peerId) // for getStats (null on an SFU)
|
|
11
12
|
// transport.stop()
|
|
12
13
|
//
|
|
@@ -24,6 +25,19 @@
|
|
|
24
25
|
// and a late ICE candidate or answer from a dead connection is dropped instead of poisoning the
|
|
25
26
|
// new one.
|
|
26
27
|
//
|
|
28
|
+
// THE CONTENT TRACK (3D app share, docs/call.md § Sharing a 3D app): a SECOND video transceiver
|
|
29
|
+
// per pair, sendrecv, so either side can share into it. It exists only when BOTH hellos said
|
|
30
|
+
// `content: true` — the call module calls addContentTransceiver(pid) when the peer's hello says
|
|
31
|
+
// so (its own always does) — so an older peer is never offered a second video m-line and never
|
|
32
|
+
// gets a second camera sender. The offerer adds it (null track until someone shares) and
|
|
33
|
+
// renegotiates with a same-generation re-offer (the answerer's 'offer' case takes it on the same
|
|
34
|
+
// connection); a rebuild re-adds it in the first offer. It is identified as the SECOND video
|
|
35
|
+
// transceiver in getTransceivers() order on both ends: that order is m-line order (the offerer
|
|
36
|
+
// created them in that order; the answerer's are created from the offer's m-lines, and it never
|
|
37
|
+
// adds its own), and m-lines are never reordered or reused here — so both ends agree with no mid
|
|
38
|
+
// or msid bookkeeping. Sharing is then a replaceTrack on that sender only (no renegotiation), and
|
|
39
|
+
// its frames reach the far end's onContentStream, never the camera stream.
|
|
40
|
+
//
|
|
27
41
|
// RECOVERY LADDER on a pair that stops working: 'disconnected' for 4 s, or 'failed' → ICE restart
|
|
28
42
|
// on the SAME connection (cheap: keeps the transceivers, renegotiates candidates) → if that has
|
|
29
43
|
// not connected within 10 s, tear down and rebuild with exponential backoff (1 s, 2 s, 4 s … 30 s,
|
|
@@ -44,6 +58,8 @@ const ANSWERER_WAIT_MS = 8000;
|
|
|
44
58
|
* keeps retrying underneath it the whole time.
|
|
45
59
|
*/
|
|
46
60
|
export const UNREACHABLE_MS = 10000;
|
|
61
|
+
/** The camera's share of its bitrate while this side shares content (tuneSenders). */
|
|
62
|
+
export const CAMERA_SHARE_FACTOR = 0.6;
|
|
47
63
|
|
|
48
64
|
/**
|
|
49
65
|
* Clamp the requested room size to what a full mesh can carry.
|
|
@@ -86,6 +102,9 @@ export class MeshTransport {
|
|
|
86
102
|
this.stopped = false;
|
|
87
103
|
this.sendFormat = 'mono';
|
|
88
104
|
this.localStream = null;
|
|
105
|
+
// The content track this side shares (null = not sharing) and the stream it rides in.
|
|
106
|
+
this.contentTrack = null;
|
|
107
|
+
this.contentStream = null;
|
|
89
108
|
}
|
|
90
109
|
|
|
91
110
|
/** Remote peers currently in the mesh (not counting this one). */
|
|
@@ -93,10 +112,12 @@ export class MeshTransport {
|
|
|
93
112
|
return this.peers.size;
|
|
94
113
|
}
|
|
95
114
|
|
|
96
|
-
async start({ room, localStream, sendFormat, onPeer, onPeerLeft, onStream, onMessage, onPeerState, onRefused, onSignalingState, onGhost }) {
|
|
115
|
+
async start({ room, localStream, sendFormat, contentTrack, contentStream, onPeer, onPeerLeft, onStream, onContentStream, onMessage, onPeerState, onRefused, onSignalingState, onGhost }) {
|
|
97
116
|
this.localStream = localStream;
|
|
98
117
|
this.sendFormat = sendFormat || 'mono';
|
|
99
|
-
|
|
118
|
+
if (contentTrack !== undefined) this.contentTrack = contentTrack || null;
|
|
119
|
+
if (contentStream !== undefined) this.contentStream = contentStream || null;
|
|
120
|
+
this.cb = { onPeer, onPeerLeft, onStream, onContentStream, onMessage, onPeerState, onRefused, onSignalingState, onGhost };
|
|
100
121
|
// Hooks can fire before join() resolves (the PeerJS adapter discovers peers while joining);
|
|
101
122
|
// queue them so every one runs against a fully-initialised transport.
|
|
102
123
|
const later = (fn) => (...a) => (this.ready ? fn(...a) : this._pending.push(() => fn(...a)));
|
|
@@ -178,6 +199,11 @@ export class MeshTransport {
|
|
|
178
199
|
gen: 0,
|
|
179
200
|
state: 'new',
|
|
180
201
|
stream: null,
|
|
202
|
+
// Content (3D app share): both hellos said `content` (contentCap), and the far end's
|
|
203
|
+
// content track once it arrived, in its own stream.
|
|
204
|
+
contentCap: false,
|
|
205
|
+
contentStream: null,
|
|
206
|
+
renegotiate: false,
|
|
181
207
|
attempts: 0,
|
|
182
208
|
iceRestarted: false,
|
|
183
209
|
absent: false,
|
|
@@ -201,6 +227,11 @@ export class MeshTransport {
|
|
|
201
227
|
rec.iceRestarted = false;
|
|
202
228
|
rec.pendingCandidates = [];
|
|
203
229
|
rec.stream = null; // a new connection's tracks arrive in a new stream (a fresh srcObject)
|
|
230
|
+
rec.renegotiate = false;
|
|
231
|
+
if (rec.contentStream) {
|
|
232
|
+
rec.contentStream = null;
|
|
233
|
+
this.cb.onContentStream?.(rec.id, null);
|
|
234
|
+
}
|
|
204
235
|
const pc = new this.PC({ iceServers: this.iceServers });
|
|
205
236
|
rec.pc = pc;
|
|
206
237
|
const dc = pc.createDataChannel('dxr-call', { negotiated: true, id: 0, ordered: true });
|
|
@@ -213,11 +244,20 @@ export class MeshTransport {
|
|
|
213
244
|
pc.addTransceiver(audio || 'audio', { direction: 'sendrecv', streams });
|
|
214
245
|
const vt = pc.addTransceiver(video || 'video', { direction: 'sendrecv', streams });
|
|
215
246
|
this._preferCodecs(vt);
|
|
247
|
+
// A rebuild of a pair that already agreed on content offers its transceiver at once.
|
|
248
|
+
if (rec.contentCap) this._addContent(rec, pc);
|
|
216
249
|
}
|
|
217
250
|
pc.onicecandidate = (e) => {
|
|
218
251
|
if (e.candidate) this._signal(rec, { kind: 'candidate', candidate: e.candidate.toJSON ? e.candidate.toJSON() : e.candidate });
|
|
219
252
|
};
|
|
220
253
|
pc.ontrack = (e) => {
|
|
254
|
+
if (rec.pc !== pc) return;
|
|
255
|
+
if (this._isContent(pc, e.transceiver, e.track)) {
|
|
256
|
+
// The far end's content track: its own stream, never the camera's (no eviction).
|
|
257
|
+
rec.contentStream = new MediaStream([e.track]);
|
|
258
|
+
this.cb.onContentStream?.(rec.id, rec.contentStream);
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
221
261
|
if (!rec.stream) rec.stream = new MediaStream();
|
|
222
262
|
if (!rec.stream.getTracks().includes(e.track)) {
|
|
223
263
|
for (const t of rec.stream.getTracks()) if (t.kind === e.track.kind) rec.stream.removeTrack(t);
|
|
@@ -269,12 +309,18 @@ export class MeshTransport {
|
|
|
269
309
|
await pc.setLocalDescription(answer);
|
|
270
310
|
this._signal(rec, { kind: 'answer', sdp: pc.localDescription.sdp });
|
|
271
311
|
this._flushCandidates(rec);
|
|
312
|
+
if (rec.state === 'connected') this.tuneSenders(); // a re-offer may have added the content sender
|
|
272
313
|
break;
|
|
273
314
|
}
|
|
274
315
|
case 'answer':
|
|
275
316
|
if (!rec.offerer || gen !== rec.gen || !rec.pc || rec.pc.signalingState !== 'have-local-offer') return;
|
|
276
317
|
await rec.pc.setRemoteDescription({ type: 'answer', sdp: data.sdp });
|
|
277
318
|
this._flushCandidates(rec);
|
|
319
|
+
if (rec.renegotiate) {
|
|
320
|
+
// A content transceiver was added while this offer was out: offer again.
|
|
321
|
+
rec.renegotiate = false;
|
|
322
|
+
this._offer(rec).catch((err) => this.log('offer-error', { peer: rec.id, err: String(err) }));
|
|
323
|
+
} else if (rec.state === 'connected') this.tuneSenders();
|
|
278
324
|
break;
|
|
279
325
|
case 'candidate':
|
|
280
326
|
if (gen !== rec.gen || !rec.pc) return;
|
|
@@ -305,11 +351,25 @@ export class MeshTransport {
|
|
|
305
351
|
for (const c of list) rec.pc.addIceCandidate(c).catch(() => {});
|
|
306
352
|
}
|
|
307
353
|
|
|
308
|
-
/**
|
|
354
|
+
/**
|
|
355
|
+
* Answerer side: put our tracks on the transceivers the offer created, and make them sendrecv.
|
|
356
|
+
* The content transceiver (the second video one) never gets the camera: it gets this side's
|
|
357
|
+
* content track when it is sharing, else stays empty (but sendrecv, so it can share later).
|
|
358
|
+
*/
|
|
309
359
|
_attachAnswererTracks(rec) {
|
|
310
360
|
const pc = rec.pc;
|
|
311
361
|
const tracks = this.localStream ? this.localStream.getTracks() : [];
|
|
362
|
+
const content = this._contentTransceiver(pc);
|
|
363
|
+
if (content) {
|
|
364
|
+
if (this.contentTrack && content.sender.track !== this.contentTrack) {
|
|
365
|
+
content.sender.replaceTrack(this.contentTrack).catch(() => {});
|
|
366
|
+
if (this.contentStream && content.sender.setStreams) content.sender.setStreams(this.contentStream);
|
|
367
|
+
}
|
|
368
|
+
content.direction = 'sendrecv';
|
|
369
|
+
this._preferCodecs(content);
|
|
370
|
+
}
|
|
312
371
|
for (const tr of pc.getTransceivers()) {
|
|
372
|
+
if (tr === content) continue;
|
|
313
373
|
const kind = tr.receiver && tr.receiver.track ? tr.receiver.track.kind : null;
|
|
314
374
|
if (!kind || tr.sender.track) continue;
|
|
315
375
|
const t = tracks.find((x) => x.kind === kind);
|
|
@@ -322,6 +382,61 @@ export class MeshTransport {
|
|
|
322
382
|
}
|
|
323
383
|
}
|
|
324
384
|
|
|
385
|
+
// ── the content transceiver ─────────────────────────────────────────────────────────────
|
|
386
|
+
|
|
387
|
+
/** The video transceivers of a connection, in getTransceivers() (= m-line) order. */
|
|
388
|
+
_videoTransceivers(pc) {
|
|
389
|
+
if (!pc || typeof pc.getTransceivers !== 'function') return [];
|
|
390
|
+
return pc.getTransceivers().filter((tr) => tr && tr.receiver && tr.receiver.track && tr.receiver.track.kind === 'video');
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
/** The content transceiver: the SECOND video transceiver (see the header), or null. */
|
|
394
|
+
_contentTransceiver(pc) {
|
|
395
|
+
return this._videoTransceivers(pc)[1] || null;
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/** Is this incoming track (ontrack) the far end's content track? */
|
|
399
|
+
_isContent(pc, transceiver, track) {
|
|
400
|
+
const ct = this._contentTransceiver(pc);
|
|
401
|
+
if (!ct) return false;
|
|
402
|
+
if (transceiver) return transceiver === ct;
|
|
403
|
+
return !!(track && ct.receiver && ct.receiver.track === track);
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
_addContent(rec, pc) {
|
|
407
|
+
if (this._contentTransceiver(pc)) return this._contentTransceiver(pc);
|
|
408
|
+
const streams = this.contentStream ? [this.contentStream] : [];
|
|
409
|
+
const tr = pc.addTransceiver(this.contentTrack || 'video', { direction: 'sendrecv', streams });
|
|
410
|
+
this._preferCodecs(tr);
|
|
411
|
+
return tr;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
/**
|
|
415
|
+
* Both hellos of this pair said `content`: give it the content transceiver. Offerer only (the
|
|
416
|
+
* answerer gets it in the offer); idempotent. Renegotiates with a same-generation re-offer, or
|
|
417
|
+
* after the offer in flight is answered. False when this side cannot (answerer, unknown peer).
|
|
418
|
+
*/
|
|
419
|
+
addContentTransceiver(pid) {
|
|
420
|
+
const rec = this.peers.get(pid);
|
|
421
|
+
if (!rec || this.stopped) return false;
|
|
422
|
+
rec.contentCap = true;
|
|
423
|
+
if (!rec.offerer) return false;
|
|
424
|
+
const pc = rec.pc;
|
|
425
|
+
if (!pc) return true; // the next build adds it
|
|
426
|
+
if (this._contentTransceiver(pc)) return true;
|
|
427
|
+
this._addContent(rec, pc);
|
|
428
|
+
this.log('content-transceiver', { peer: rec.id, gen: rec.gen });
|
|
429
|
+
if (pc.signalingState === 'stable') this._offer(rec).catch((err) => this.log('offer-error', { peer: rec.id, err: String(err) }));
|
|
430
|
+
else rec.renegotiate = true;
|
|
431
|
+
return true;
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
/** Does this pair carry a content transceiver right now? */
|
|
435
|
+
hasContent(pid) {
|
|
436
|
+
const rec = this.peers.get(pid);
|
|
437
|
+
return !!(rec && rec.pc && this._contentTransceiver(rec.pc));
|
|
438
|
+
}
|
|
439
|
+
|
|
325
440
|
_preferCodecs(transceiver) {
|
|
326
441
|
const caps = globalThis.RTCRtpReceiver && RTCRtpReceiver.getCapabilities && RTCRtpReceiver.getCapabilities('video');
|
|
327
442
|
this._canPreferCodecs = !!(transceiver && transceiver.setCodecPreferences && caps);
|
|
@@ -402,6 +517,10 @@ export class MeshTransport {
|
|
|
402
517
|
clearTimeout(rec.waitTimer);
|
|
403
518
|
this._closePc(rec);
|
|
404
519
|
this.peers.delete(rec.id);
|
|
520
|
+
if (rec.contentStream) {
|
|
521
|
+
rec.contentStream = null;
|
|
522
|
+
this.cb.onContentStream?.(rec.id, null);
|
|
523
|
+
}
|
|
405
524
|
this._setState(rec, 'left');
|
|
406
525
|
this.cb.onPeerLeft?.(rec.id, reason);
|
|
407
526
|
this.tuneSenders();
|
|
@@ -523,12 +642,29 @@ export class MeshTransport {
|
|
|
523
642
|
|
|
524
643
|
// ── sending ─────────────────────────────────────────────────────────────────────────────
|
|
525
644
|
|
|
526
|
-
/**
|
|
527
|
-
|
|
528
|
-
|
|
645
|
+
/**
|
|
646
|
+
* Swap an outgoing track. No renegotiation: replaceTrack on the senders of that slot.
|
|
647
|
+
* `slot: 'camera'` (default — setCamera / setMic): every sender of that kind EXCEPT the content
|
|
648
|
+
* one. `slot: 'content'` (shareContent / stopContent, video only): the content sender of every
|
|
649
|
+
* pair that has one; `track: null` stops sharing. A pair that gets its content transceiver later
|
|
650
|
+
* picks the current content track up then.
|
|
651
|
+
*/
|
|
652
|
+
async replaceTrack(kind, track, stream, { slot = 'camera' } = {}) {
|
|
653
|
+
if (slot === 'content') {
|
|
654
|
+
this.contentTrack = track || null;
|
|
655
|
+
this.contentStream = track ? stream || null : null;
|
|
656
|
+
} else if (stream) this.localStream = stream;
|
|
529
657
|
for (const rec of this.peers.values()) {
|
|
530
658
|
if (!rec.pc) continue;
|
|
659
|
+
const content = this._contentTransceiver(rec.pc);
|
|
660
|
+
if (slot === 'content') {
|
|
661
|
+
if (!content) continue;
|
|
662
|
+
await content.sender.replaceTrack(track || null).catch(() => {});
|
|
663
|
+
if (track && stream && content.sender.setStreams) content.sender.setStreams(stream);
|
|
664
|
+
continue;
|
|
665
|
+
}
|
|
531
666
|
for (const tr of rec.pc.getTransceivers()) {
|
|
667
|
+
if (tr === content) continue;
|
|
532
668
|
const k = tr.receiver && tr.receiver.track ? tr.receiver.track.kind : null;
|
|
533
669
|
if (k === kind) await tr.sender.replaceTrack(track).catch(() => {});
|
|
534
670
|
}
|
|
@@ -547,16 +683,21 @@ export class MeshTransport {
|
|
|
547
683
|
*/
|
|
548
684
|
async tuneSenders() {
|
|
549
685
|
const n = this.peers.size;
|
|
550
|
-
const
|
|
686
|
+
const sharing = !!this.contentTrack;
|
|
687
|
+
// While content is on, the camera yields: the shared app is what people look at.
|
|
688
|
+
const kbps = Math.round(maxBitrateKbps(this.sendFormat, n) * (sharing ? CAMERA_SHARE_FACTOR : 1));
|
|
689
|
+
const contentKbps = maxBitrateKbps('sbs', n);
|
|
551
690
|
for (const rec of this.peers.values()) {
|
|
552
691
|
if (!rec.pc) continue;
|
|
692
|
+
const ct = this._contentTransceiver(rec.pc);
|
|
553
693
|
for (const s of rec.pc.getSenders()) {
|
|
554
694
|
if (!s.track || s.track.kind !== 'video') continue;
|
|
695
|
+
const isContent = !!(ct && ct.sender === s);
|
|
555
696
|
try {
|
|
556
697
|
const p = s.getParameters();
|
|
557
698
|
p.degradationPreference = 'maintain-resolution';
|
|
558
699
|
if (!p.encodings || !p.encodings.length) p.encodings = [{}];
|
|
559
|
-
p.encodings[0].maxBitrate = kbps * 1000;
|
|
700
|
+
p.encodings[0].maxBitrate = (isContent ? contentKbps : kbps) * 1000;
|
|
560
701
|
await s.setParameters(p);
|
|
561
702
|
} catch (err) {
|
|
562
703
|
this.log('setParameters-error', { peer: rec.id, err: String(err) });
|
package/js/call/ui.js
CHANGED
|
@@ -148,6 +148,9 @@ const CSS = `
|
|
|
148
148
|
.dxr-call-grid[data-layout="speaker"]{grid-template-columns:repeat(3,1fr);}
|
|
149
149
|
.dxr-call-grid[data-layout="speaker"] .dxr-call-tile--main{grid-column:1/-1;}
|
|
150
150
|
.dxr-call-grid[data-layout="speaker"][data-n="1"]{grid-template-columns:1fr;}
|
|
151
|
+
/* Shared content (3D app share): the content tile spans the row, the people form a strip under it. */
|
|
152
|
+
.dxr-call-grid[data-content="1"]:not([data-n="1"]){grid-template-columns:repeat(3,1fr);max-width:calc(var(--dxr-call-h)*4/3);}
|
|
153
|
+
.dxr-call-grid[data-content="1"] .dxr-call-tile--main{grid-column:1/-1;}
|
|
151
154
|
/* layout:'none' — the module creates the tiles but does not position them: the grid box steps
|
|
152
155
|
aside (display:contents) so each [data-dxr-peer] tile is laid out by the PAGE's own CSS on the
|
|
153
156
|
host. A tile is never moved; it is styled in place (woven-canvas rule 2). */
|
package/js/call/wire.js
CHANGED
|
@@ -168,6 +168,9 @@ export function normalizeHello(msg) {
|
|
|
168
168
|
conv: msg.conv === true,
|
|
169
169
|
// The sender's display name (`name` option, 1.48+), sanitised; null when absent or empty.
|
|
170
170
|
name: sanitizeName(msg.name),
|
|
171
|
+
// This side can carry a SECOND video track, the content tile (3D app share). Absent (an older
|
|
172
|
+
// SDK) = false: the offerer then never adds the content transceiver for this pair.
|
|
173
|
+
content: msg.content === true,
|
|
171
174
|
};
|
|
172
175
|
}
|
|
173
176
|
|
|
@@ -192,7 +195,7 @@ export function sanitizeName(v) {
|
|
|
192
195
|
* Build the `hello` this side sends. `conv: true` (advertised for a side-by-side sender only)
|
|
193
196
|
* says this side sends `conv` messages.
|
|
194
197
|
*/
|
|
195
|
-
export function makeHello({ format, width, height, baselineMm, hfovDeg, rectified, conv, name } = {}) {
|
|
198
|
+
export function makeHello({ format, width, height, baselineMm, hfovDeg, rectified, conv, name, content } = {}) {
|
|
196
199
|
const h = {
|
|
197
200
|
type: 'hello',
|
|
198
201
|
v: WIRE_VERSION,
|
|
@@ -207,9 +210,70 @@ export function makeHello({ format, width, height, baselineMm, hfovDeg, rectifie
|
|
|
207
210
|
if (h.format === 'sbs' && conv) h.conv = true;
|
|
208
211
|
const n = sanitizeName(name);
|
|
209
212
|
if (n) h.name = n;
|
|
213
|
+
if (content) h.content = true;
|
|
210
214
|
return h;
|
|
211
215
|
}
|
|
212
216
|
|
|
217
|
+
// ── content (3D app share: a second, per-peer video track) ────────────────────────────────
|
|
218
|
+
|
|
219
|
+
/** A content stream's size is at most this on either side (a receiver ignores anything bigger). */
|
|
220
|
+
export const CONTENT_MAX_DIM = 16384;
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* `content {on, format, width, height, convergence: {known, frac}, appName?}`: what this side is
|
|
224
|
+
* sharing on its content track (docs/call.md § Sharing a 3D app). `on: false` = stopped (only `on`
|
|
225
|
+
* is sent). `convergence.known` = the frames are display-referred and `frac` IS the disparity of
|
|
226
|
+
* the sender's display plane as a fraction of the per-eye width (`zeroDisparityShiftFrac`, 0 for
|
|
227
|
+
* a DisplayXR app in v1): a receiver applies it and never measures. Pure.
|
|
228
|
+
*/
|
|
229
|
+
export function makeContent({ on = true, format, width, height, convergence, appName } = {}) {
|
|
230
|
+
if (!on) return { type: 'content', on: false };
|
|
231
|
+
const fmt = CALL_FORMATS.includes(format) ? format : 'sbs';
|
|
232
|
+
const dim = (v) => (typeof v === 'number' && Number.isFinite(v) && v >= 1 && v <= CONTENT_MAX_DIM ? Math.round(v) : 0);
|
|
233
|
+
const known = !!(convergence && convergence.known);
|
|
234
|
+
const f = known ? convergence.frac : null;
|
|
235
|
+
const m = {
|
|
236
|
+
type: 'content',
|
|
237
|
+
on: true,
|
|
238
|
+
format: fmt,
|
|
239
|
+
width: dim(width),
|
|
240
|
+
height: dim(height),
|
|
241
|
+
convergence: known ? { known: true, frac: typeof f === 'number' && Number.isFinite(f) ? Math.max(-CONV_MAX_FRAC, Math.min(CONV_MAX_FRAC, f)) : 0 } : { known: false, frac: null },
|
|
242
|
+
};
|
|
243
|
+
const n = sanitizeName(appName);
|
|
244
|
+
if (n) m.appName = n;
|
|
245
|
+
return m;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* Validate an incoming `content`. Null for anything that is not one. `on` must be `true` to mean
|
|
250
|
+
* on; a bad `format` reads as `'mono'` (never assume more than was said); sizes outside
|
|
251
|
+
* 1..16384 read as null; a `convergence.frac` that is not finite or beyond ±CONV_MAX_FRAC makes
|
|
252
|
+
* the convergence UNKNOWN; `appName` is sanitised (≤ 64 characters). Never throws: this reads data
|
|
253
|
+
* from another browser.
|
|
254
|
+
*/
|
|
255
|
+
export function normalizeContent(msg) {
|
|
256
|
+
if (!msg || typeof msg !== 'object' || msg.type !== 'content') return null;
|
|
257
|
+
if (msg.on !== true) return { type: 'content', on: false, format: null, width: null, height: null, convergence: { known: false, frac: null }, appName: null };
|
|
258
|
+
const num = (v) => (typeof v === 'number' && Number.isFinite(v) && v >= 1 && v <= CONTENT_MAX_DIM ? v : null);
|
|
259
|
+
let convergence = { known: false, frac: null };
|
|
260
|
+
try {
|
|
261
|
+
const c = msg.convergence;
|
|
262
|
+
if (c && typeof c === 'object' && c.known === true && typeof c.frac === 'number' && Number.isFinite(c.frac) && Math.abs(c.frac) <= CONV_MAX_FRAC) convergence = { known: true, frac: c.frac };
|
|
263
|
+
} catch {
|
|
264
|
+
/* a hostile getter */
|
|
265
|
+
}
|
|
266
|
+
return {
|
|
267
|
+
type: 'content',
|
|
268
|
+
on: true,
|
|
269
|
+
format: CALL_FORMATS.includes(msg.format) ? msg.format : 'mono',
|
|
270
|
+
width: num(msg.width),
|
|
271
|
+
height: num(msg.height),
|
|
272
|
+
convergence,
|
|
273
|
+
appName: sanitizeName(msg.appName),
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
|
|
213
277
|
/**
|
|
214
278
|
* Wrap a page's app payload: `{ type: 'app', d: data }` serialised, or null when `data` cannot be
|
|
215
279
|
* serialised (undefined, a function, a cycle, a BigInt) or the message is over `APP_MAX_BYTES`
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// camera/app-share.js — recognising a "3D screen share" device: a running DisplayXR app's
|
|
2
|
+
// pre-weave side-by-side views, which the DisplayXR Browser lists as an ordinary `videoinput`
|
|
3
|
+
// (runtime ADR-061, `XR_DXR_app_share`). Pure, no DOM.
|
|
4
|
+
//
|
|
5
|
+
// It lives under /camera, not /call, because the camera probe must know it too: such a device is
|
|
6
|
+
// NOT a camera. Its label ("DisplayXR: <app> (3D)") matches stereoLabelHint's `\b3d\b`, so without
|
|
7
|
+
// this rule `camera: 'auto'` would open somebody's running app and send it as their face. Every
|
|
8
|
+
// probe path in capture.js leaves it out, whatever `exclude` says. `/call`'s share.js builds the
|
|
9
|
+
// discovery API on top of it.
|
|
10
|
+
//
|
|
11
|
+
// The device contract (ADR-061 decision 13): label `DisplayXR: <app> (3D)`, device id prefix
|
|
12
|
+
// `dxr-app:`, and on the opened track `getSettings().displayxrStereo.source === 'app'` (plus
|
|
13
|
+
// `appName`, `zeroDisparityShiftFrac`). Browsers that predate the hint fields still use the label
|
|
14
|
+
// and the id, so any ONE of the three is enough.
|
|
15
|
+
|
|
16
|
+
/** `deviceId` prefix of an app-share device (cameras use `dxr-stereo:`). */
|
|
17
|
+
export const APP_SHARE_ID_PREFIX = 'dxr-app:';
|
|
18
|
+
|
|
19
|
+
/** The label of an app-share device; group 1 is the app's name. */
|
|
20
|
+
export const APP_SHARE_LABEL_RE = /^DisplayXR: (.+) \(3D\)$/;
|
|
21
|
+
|
|
22
|
+
/** The app name an app-share label carries, or null. Pure. */
|
|
23
|
+
export function appNameFromLabel(label) {
|
|
24
|
+
const m = typeof label === 'string' ? APP_SHARE_LABEL_RE.exec(label.trim()) : null;
|
|
25
|
+
return m ? m[1].trim() || null : null;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Is this an app-share device rather than a camera? Takes a `MediaDeviceInfo` (or any
|
|
30
|
+
* `{ label, deviceId }`), a label string, a track's settings (`{ displayxrStereo }`), or a
|
|
31
|
+
* `MediaStreamTrack` (its label and settings). True on ANY of: the label shape, the `dxr-app:` id
|
|
32
|
+
* prefix, or `displayxrStereo.source === 'app'`. Never throws. Pure.
|
|
33
|
+
* @param {unknown} x
|
|
34
|
+
* @returns {boolean}
|
|
35
|
+
*/
|
|
36
|
+
export function isAppShareDevice(x) {
|
|
37
|
+
if (!x) return false;
|
|
38
|
+
if (typeof x === 'string') return APP_SHARE_LABEL_RE.test(x.trim()) || x.startsWith(APP_SHARE_ID_PREFIX);
|
|
39
|
+
if (typeof x !== 'object') return false;
|
|
40
|
+
const o = /** @type {any} */ (x);
|
|
41
|
+
try {
|
|
42
|
+
if (typeof o.label === 'string' && APP_SHARE_LABEL_RE.test(o.label.trim())) return true;
|
|
43
|
+
if (typeof o.deviceId === 'string' && o.deviceId.startsWith(APP_SHARE_ID_PREFIX)) return true;
|
|
44
|
+
const hint = o.displayxrStereo || (typeof o.getSettings === 'function' ? (o.getSettings() || {}).displayxrStereo : null);
|
|
45
|
+
if (hint && typeof hint === 'object' && hint.source === 'app') return true;
|
|
46
|
+
if (typeof o.getSettings === 'function') {
|
|
47
|
+
const id = (o.getSettings() || {}).deviceId;
|
|
48
|
+
if (typeof id === 'string' && id.startsWith(APP_SHARE_ID_PREFIX)) return true;
|
|
49
|
+
}
|
|
50
|
+
} catch {
|
|
51
|
+
/* a hostile getter: not an app */
|
|
52
|
+
}
|
|
53
|
+
return false;
|
|
54
|
+
}
|