@displayxr/inline3d 1.47.6 → 1.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +136 -1
- package/call.d.ts +202 -12
- package/camera.d.ts +44 -7
- package/dist/call.js +11 -6
- package/docs/woven-canvas-rules.md +25 -13
- package/js/call/element.js +9 -3
- package/js/call/options.js +22 -3
- package/js/call/share.js +147 -0
- package/js/call/transport.js +160 -9
- package/js/call/ui.js +11 -1
- package/js/call/wire.js +227 -6
- package/js/camera/app-share.js +54 -0
- package/js/camera/capture.js +196 -32
- package/js/camera/conv-meter.js +152 -0
- package/js/inline3d-call.js +737 -91
- package/js/inline3d-camera.js +34 -7
- package/js/inline3d-player.js +8 -4
- package/package.json +8 -1
- package/signaling/README.md +289 -0
- package/signaling/dev-server.mjs +320 -0
- package/signaling/gate.mjs +112 -0
- package/signaling/keys.mjs +123 -0
- package/signaling/limits.mjs +221 -0
- package/signaling/room.mjs +322 -0
- package/signaling/turn-budget.mjs +254 -0
package/js/camera/capture.js
CHANGED
|
@@ -5,16 +5,30 @@
|
|
|
5
5
|
// 1280x480, i.e. 640x480 per eye, grayscale and UNRECTIFIED). Found → sent as `sbs`. Otherwise the
|
|
6
6
|
// default webcam, sent as `mono`. Per-eye width is whatever the camera gives; nothing is upscaled.
|
|
7
7
|
//
|
|
8
|
-
//
|
|
8
|
+
// Three field facts shape the probing:
|
|
9
9
|
// - On some laptops the stereo camera is held EXCLUSIVELY by the eye tracker, so opening it
|
|
10
10
|
// fails with NotReadableError ("Device in use"). That device is skipped silently — a call must
|
|
11
11
|
// never fail because an optional camera is busy.
|
|
12
|
+
// - Not every `videoinput` is a camera a person should be seen through. A hand-tracking camera
|
|
13
|
+
// (and an eye tracker's or a depth sensor's IR stream) enumerates as an ordinary video input
|
|
14
|
+
// and can even look like a pair by shape; on a Windows box with no stereo camera open, the
|
|
15
|
+
// last-resort "try every device" loop opened one and sent it as the call's 3D picture. Labels
|
|
16
|
+
// matching CAMERA_EXCLUDE_BUILTIN, plus the page's `exclude` list, are never opened by any probe
|
|
17
|
+
// path (an excluded default is closed again), and are reported in `skipped` as `excluded`.
|
|
12
18
|
// - Labels are empty until one getUserMedia has succeeded. Without them, permission is primed on
|
|
13
|
-
// the DEFAULT camera with no constraints
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
//
|
|
19
|
+
// the DEFAULT camera with no constraints (a plain open of one lens of a phone's pair, or its
|
|
20
|
+
// teardown, crashes some phones' camera HAL; any facing or size constraint lets the engine pick
|
|
21
|
+
// a lens instead). Where the runtime's 3D camera is the default, the prime opens the pair
|
|
22
|
+
// itself. NOTHING here assumes a list position: on Windows the DisplayXR Browser lists its
|
|
23
|
+
// "3D Camera (DisplayXR)" LAST. A pair is recognised by the track's own `displayxrStereo`
|
|
24
|
+
// description or by its label (stereoLabelHint), and after the prime the list is read again and
|
|
25
|
+
// a named pair is opened by id BEFORE a default that is a pair only by its shape is kept. The
|
|
26
|
+
// primed default is the fallback.
|
|
27
|
+
// - A DisplayXR Browser also lists every shareable running DisplayXR APP as a side-by-side
|
|
28
|
+
// `videoinput` ("DisplayXR: <app> (3D)", id `dxr-app:…`, runtime ADR-061) — 3D screen share.
|
|
29
|
+
// It is a pair, it names itself "3D", and it is NOT a camera: no probe path opens one, whatever
|
|
30
|
+
// `exclude` says (`exclude: false` included). Each is reported in `skipped` as `excluded` with
|
|
31
|
+
// `reason: 'app-share'`. `/call`'s `listShareableApps()` / `openAppShare()` are how a page uses them.
|
|
18
32
|
//
|
|
19
33
|
// RECTIFICATION is not done here (P2): it needs the camera's calibration (intrinsics/extrinsics
|
|
20
34
|
// for the ACTIVE device), which comes from a plug-in or the runtime. P1 sends the raw pair with
|
|
@@ -22,6 +36,7 @@
|
|
|
22
36
|
// `(stream, info) => MediaStream | Promise<MediaStream>`, returning a rectified SBS stream.
|
|
23
37
|
|
|
24
38
|
import { looksSbs } from './geometry.js';
|
|
39
|
+
import { isAppShareDevice } from './app-share.js';
|
|
25
40
|
|
|
26
41
|
const MONO_CONSTRAINTS = { width: { ideal: 1280 }, height: { ideal: 720 }, frameRate: { ideal: 30 } };
|
|
27
42
|
// Ask a candidate stereo device for its widest mode; a 2560x720 or 3840x1080 pair comes back as
|
|
@@ -48,13 +63,57 @@ export function isDeniedError(err) {
|
|
|
48
63
|
* @param {Array<{error: string, busy?: boolean, denied?: boolean}>} skipped
|
|
49
64
|
*/
|
|
50
65
|
export function noCameraCode(skipped) {
|
|
51
|
-
|
|
52
|
-
|
|
66
|
+
// A device left out on purpose (`excluded`) says nothing about why the others failed.
|
|
67
|
+
const tried = skipped.filter((s) => !s.excluded);
|
|
68
|
+
if (tried.length && tried.every((s) => s.busy)) return 'camera-busy';
|
|
69
|
+
if (tried.some((s) => s.denied)) return 'permission-denied';
|
|
53
70
|
return 'no-camera';
|
|
54
71
|
}
|
|
55
72
|
|
|
56
|
-
/**
|
|
73
|
+
/**
|
|
74
|
+
* Labels of video inputs that are sensors, not cameras to be seen through: hand tracking, an eye
|
|
75
|
+
* tracker's or a "tracking camera"'s IR stream, a depth camera. Generic words only. Applied on
|
|
76
|
+
* every probe path unless `exclude: false`.
|
|
77
|
+
*/
|
|
78
|
+
// Generic words, plus the one hand tracker that enumerates under its bare brand name: on Windows
|
|
79
|
+
// an Ultraleap (Leap 2 / Leap Motion Controller) is a UVC "Ultraleap", nothing else in the label —
|
|
80
|
+
// the generic words alone let it through (win box, 2026-11-04: it was the browser's default camera).
|
|
81
|
+
export const CAMERA_EXCLUDE_BUILTIN = /\b(hand[- ]?track|tracking camera|eye[- ]?track|depth camera|ultraleap|leap motion|infrared|ir camera)/i;
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* The exclusion list in force: the built-in pattern plus `exclude` (strings match a label
|
|
85
|
+
* case-insensitively as a substring, or a deviceId exactly; RegExps test the label), or only the
|
|
86
|
+
* app-share rule for `exclude: false`. Returns a predicate `(label, deviceId) => boolean`. An
|
|
87
|
+
* app-share device (camera/app-share.js: a running app's views, not a camera) is ALWAYS excluded —
|
|
88
|
+
* `exclude: false` turns the sensor list off, never that. Pure.
|
|
89
|
+
* @param {false|null|undefined|ReadonlyArray<string|RegExp>|string|RegExp} exclude
|
|
90
|
+
*/
|
|
91
|
+
export function cameraExcluder(exclude) {
|
|
92
|
+
const app = (label, deviceId) => isAppShareDevice({ label: String(label || ''), deviceId: typeof deviceId === 'string' ? deviceId : '' });
|
|
93
|
+
if (exclude === false) return app;
|
|
94
|
+
const list = [CAMERA_EXCLUDE_BUILTIN];
|
|
95
|
+
for (const x of exclude == null ? [] : Array.isArray(exclude) ? exclude : [exclude]) {
|
|
96
|
+
if (x instanceof RegExp || (typeof x === 'string' && x)) list.push(x);
|
|
97
|
+
}
|
|
98
|
+
return (label, deviceId) => {
|
|
99
|
+
if (app(label, deviceId)) return true;
|
|
100
|
+
const l = String(label || '');
|
|
101
|
+
for (const x of list) {
|
|
102
|
+
if (x instanceof RegExp) {
|
|
103
|
+
x.lastIndex = 0; // a /g or /y pattern is stateful
|
|
104
|
+
if (l && x.test(l)) return true;
|
|
105
|
+
} else if ((l && l.toLowerCase().includes(x.toLowerCase())) || (deviceId && deviceId === x)) return true;
|
|
106
|
+
}
|
|
107
|
+
return false;
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* A label that names a stereo camera, tried first when probing. Never an app-share device's
|
|
113
|
+
* ("DisplayXR: <app> (3D)" says "3D" but is a running app, not a camera). Pure.
|
|
114
|
+
*/
|
|
57
115
|
export function stereoLabelHint(label) {
|
|
116
|
+
if (isAppShareDevice(String(label || ''))) return false;
|
|
58
117
|
return /stereo|\b3d\b|\bsbs\b|dual/i.test(label || '');
|
|
59
118
|
}
|
|
60
119
|
|
|
@@ -65,7 +124,9 @@ export function stereoLabelHint(label) {
|
|
|
65
124
|
* @param {'auto'|'stereo'|'mono'|string|MediaStream} want 'auto' | 'stereo' | 'mono' | a
|
|
66
125
|
* deviceId | a page-supplied MediaStream
|
|
67
126
|
* @param {{ format?: 'sbs'|'mono', calibration?: {baselineMm?: number, hfovDeg?: number, rectified?: boolean},
|
|
68
|
-
* mediaDevices?: MediaDevices, log?: Function }} [o]
|
|
127
|
+
* mediaDevices?: MediaDevices, log?: Function, exclude?: false|Array<string|RegExp> }} [o]
|
|
128
|
+
* `exclude`: devices no probe may open (see cameraExcluder); an explicit deviceId is the
|
|
129
|
+
* page's own choice and is opened regardless.
|
|
69
130
|
* @returns {Promise<{ stream: MediaStream, format: 'sbs'|'mono', width: number, height: number,
|
|
70
131
|
* deviceId: string|null, label: string, owned: boolean, skipped: Array<{label: string, error: string}>,
|
|
71
132
|
* calibration: object }>}
|
|
@@ -75,6 +136,7 @@ export async function openCamera(want = 'auto', o = {}) {
|
|
|
75
136
|
const log = o.log || (() => {});
|
|
76
137
|
const skipped = [];
|
|
77
138
|
const cal = o.calibration || {};
|
|
139
|
+
const isExcluded = cameraExcluder(o.exclude);
|
|
78
140
|
|
|
79
141
|
if (want && typeof want === 'object' && typeof want.getVideoTracks === 'function') {
|
|
80
142
|
const t = want.getVideoTracks()[0];
|
|
@@ -98,13 +160,66 @@ export async function openCamera(want = 'auto', o = {}) {
|
|
|
98
160
|
// eye-tracking camera shared without stealing it) describes itself on the opened track:
|
|
99
161
|
// `displayxrStereo: {layout, rectified, baselineMm, horizontalFovDeg}`.
|
|
100
162
|
const stereo = st.displayxrStereo && st.displayxrStereo.layout === 'side-by-side' ? st.displayxrStereo : null;
|
|
101
|
-
|
|
163
|
+
// A running app's views (3D screen share, ADR-061) describe themselves too: `source: 'app'`.
|
|
164
|
+
const appShare = isAppShareDevice({ label: t.label || '', deviceId: st.deviceId || '' }) || !!(st.displayxrStereo && st.displayxrStereo.source === 'app');
|
|
165
|
+
return { stream: s, track: t, width: st.width, height: st.height, deviceId: st.deviceId || null, label: t.label || '', stereo, appShare };
|
|
102
166
|
};
|
|
103
167
|
const skip = (label, err) => {
|
|
104
168
|
skipped.push({ label, error: `${err.name}: ${err.message}`, busy: isBusyError(err), denied: isDeniedError(err) });
|
|
105
169
|
log('camera-skip', { label, error: err.name });
|
|
106
170
|
};
|
|
107
171
|
const stop = (r) => r && r.stream.getTracks().forEach((t) => t.stop());
|
|
172
|
+
const listCameras = async () => {
|
|
173
|
+
try {
|
|
174
|
+
return (await md.enumerateDevices()).filter((d) => d.kind === 'videoinput');
|
|
175
|
+
} catch {
|
|
176
|
+
return []; // no enumeration: the paths below still work
|
|
177
|
+
}
|
|
178
|
+
};
|
|
179
|
+
// Excluded devices: never opened by id; a default that turns out to be one is closed again.
|
|
180
|
+
const excludedIds = new Set();
|
|
181
|
+
// `reason`: 'app-share' for a running app's views (never a camera), else 'excluded'.
|
|
182
|
+
const exclude = (label, deviceId, reason = 'excluded') => {
|
|
183
|
+
if (deviceId && excludedIds.has(deviceId)) return;
|
|
184
|
+
if (deviceId) excludedIds.add(deviceId);
|
|
185
|
+
skipped.push({ label, error: 'excluded', excluded: true, reason });
|
|
186
|
+
log('camera-skip', { label, error: 'excluded', reason });
|
|
187
|
+
};
|
|
188
|
+
const isApp = (d) => isAppShareDevice({ label: d.label || '', deviceId: d.deviceId || '' });
|
|
189
|
+
const listedExcluded = (d) => {
|
|
190
|
+
if (isApp(d)) {
|
|
191
|
+
exclude(d.label || '', d.deviceId, 'app-share');
|
|
192
|
+
return true;
|
|
193
|
+
}
|
|
194
|
+
if (!d.label || !isExcluded(d.label, d.deviceId)) return false;
|
|
195
|
+
exclude(d.label, d.deviceId);
|
|
196
|
+
return true;
|
|
197
|
+
};
|
|
198
|
+
/** An opened device that must not be used: an app share (by its own hint too) or an excluded label. Closes it. */
|
|
199
|
+
const rejectOpened = (r) => {
|
|
200
|
+
if (r.appShare) {
|
|
201
|
+
stop(r);
|
|
202
|
+
exclude(r.label, r.deviceId, 'app-share');
|
|
203
|
+
return true;
|
|
204
|
+
}
|
|
205
|
+
if (isExcluded(r.label, r.deviceId)) {
|
|
206
|
+
stop(r);
|
|
207
|
+
exclude(r.label, r.deviceId);
|
|
208
|
+
return true;
|
|
209
|
+
}
|
|
210
|
+
return false;
|
|
211
|
+
};
|
|
212
|
+
// An open of the DEFAULT (no id): only after it opened is its label known. An excluded default
|
|
213
|
+
// is closed and never opened again in this probe.
|
|
214
|
+
let defaultExcluded = false;
|
|
215
|
+
const openDefault = async (constraints) => {
|
|
216
|
+
const r = await open(constraints);
|
|
217
|
+
if (rejectOpened(r)) {
|
|
218
|
+
defaultExcluded = true;
|
|
219
|
+
return null;
|
|
220
|
+
}
|
|
221
|
+
return r;
|
|
222
|
+
};
|
|
108
223
|
// Ask an open camera for another mode without a second start; keep it as it is if it refuses.
|
|
109
224
|
const withMode = async (r, constraints) => {
|
|
110
225
|
try {
|
|
@@ -135,6 +250,28 @@ export async function openCamera(want = 'auto', o = {}) {
|
|
|
135
250
|
},
|
|
136
251
|
});
|
|
137
252
|
|
|
253
|
+
// `label:<text>`: the first camera whose label contains the text (case-insensitive) — a way to
|
|
254
|
+
// force a device from a URL or a config, where a per-origin deviceId is useless. Labels need
|
|
255
|
+
// permission: with none visible yet, the default camera is primed once and let go. Explicit like
|
|
256
|
+
// a deviceId, so it opens an excluded device too.
|
|
257
|
+
if (typeof want === 'string' && want.startsWith('label:')) {
|
|
258
|
+
const needle = want.slice('label:'.length).trim().toLowerCase();
|
|
259
|
+
let list = await listCameras();
|
|
260
|
+
if (needle && list.length && list.every((d) => !d.label)) {
|
|
261
|
+
try {
|
|
262
|
+
stop(await open(true));
|
|
263
|
+
} catch (err) {
|
|
264
|
+
skip('default', err);
|
|
265
|
+
}
|
|
266
|
+
list = await listCameras();
|
|
267
|
+
}
|
|
268
|
+
// Never a running app's views: `label:3d` means a camera (open an app with openAppShare()).
|
|
269
|
+
const d = needle ? list.find((x) => x.deviceId && !isApp(x) && (x.label || '').toLowerCase().includes(needle)) : null;
|
|
270
|
+
if (!d) throw Object.assign(new Error(`camera unavailable: no camera labelled "${needle}"`), { code: noCameraCode(skipped), skipped });
|
|
271
|
+
log('camera-label', { label: d.label, needle });
|
|
272
|
+
want = d.deviceId;
|
|
273
|
+
}
|
|
274
|
+
|
|
138
275
|
// An explicit device id.
|
|
139
276
|
if (typeof want === 'string' && !['auto', 'stereo', 'mono'].includes(want)) {
|
|
140
277
|
let r;
|
|
@@ -154,17 +291,11 @@ export async function openCamera(want = 'auto', o = {}) {
|
|
|
154
291
|
// camera start can take seconds (a busy capture stack took 7-20 s per start on a field box), and
|
|
155
292
|
// the default-first probe below costs two or three of them before it reaches the pair.
|
|
156
293
|
const tried = new Set();
|
|
157
|
-
const listCameras = async () => {
|
|
158
|
-
try {
|
|
159
|
-
return (await md.enumerateDevices()).filter((d) => d.kind === 'videoinput');
|
|
160
|
-
} catch {
|
|
161
|
-
return []; // no enumeration: the paths below still work
|
|
162
|
-
}
|
|
163
|
-
};
|
|
164
294
|
const tryNamedPairs = async (list) => {
|
|
165
295
|
for (const d of list) {
|
|
166
296
|
if (!d.deviceId || !stereoLabelHint(d.label) || tried.has(d.deviceId)) continue;
|
|
167
297
|
tried.add(d.deviceId);
|
|
298
|
+
if (listedExcluded(d)) continue;
|
|
168
299
|
let r = null;
|
|
169
300
|
try {
|
|
170
301
|
r = await open({ deviceId: { exact: d.deviceId }, ...PROBE_CONSTRAINTS });
|
|
@@ -172,6 +303,7 @@ export async function openCamera(want = 'auto', o = {}) {
|
|
|
172
303
|
skip(d.label, err);
|
|
173
304
|
continue;
|
|
174
305
|
}
|
|
306
|
+
if (rejectOpened(r)) continue; // an app that described itself only on its track
|
|
175
307
|
if (isPair(r)) {
|
|
176
308
|
log('camera-sbs', { label: r.label, width: r.width, height: r.height, direct: true });
|
|
177
309
|
return r;
|
|
@@ -192,33 +324,53 @@ export async function openCamera(want = 'auto', o = {}) {
|
|
|
192
324
|
pair = await tryNamedPairs(listed);
|
|
193
325
|
if (pair) return result(pair, 'sbs');
|
|
194
326
|
}
|
|
195
|
-
// No labels yet (no permission). Prime it on the default camera with NO constraints
|
|
196
|
-
//
|
|
197
|
-
//
|
|
327
|
+
// No labels yet (no permission). Prime it on the default camera with NO constraints (a facing
|
|
328
|
+
// or size constraint would let the engine pick a phone lens instead). Where the runtime's 3D
|
|
329
|
+
// camera is the default this opens the pair itself; elsewhere (Windows lists it LAST) the list
|
|
330
|
+
// is read again and a NAMED pair is opened by id. A default that is a pair only by its SHAPE is
|
|
331
|
+
// kept as the pair only when no named one opens (it could be any wide sensor).
|
|
198
332
|
if (listed.length && listed.every((d) => !d.label)) {
|
|
199
333
|
let primed = null;
|
|
200
334
|
try {
|
|
201
|
-
primed = await
|
|
335
|
+
primed = await openDefault(true);
|
|
202
336
|
} catch (err) {
|
|
203
337
|
skip('default', err);
|
|
204
338
|
}
|
|
205
|
-
if (primed &&
|
|
339
|
+
if (primed && primed.stereo) return result(primed, 'sbs'); // the device says it is the pair
|
|
340
|
+
if (primed && primed.deviceId) tried.add(primed.deviceId);
|
|
206
341
|
const named = await tryNamedPairs(await listCameras());
|
|
207
342
|
if (named) {
|
|
208
343
|
stop(primed);
|
|
209
344
|
return result(named, 'sbs');
|
|
210
345
|
}
|
|
346
|
+
if (primed && isPair(primed)) return result(primed, 'sbs');
|
|
211
347
|
// The primed default is the fallback as is (one camera start); ask it for the usual mode.
|
|
212
348
|
if (primed) mono = await withMode(primed, MONO_CONSTRAINTS);
|
|
213
349
|
}
|
|
214
350
|
}
|
|
215
351
|
|
|
216
352
|
// The default camera: the fallback (and, with labels already visible, also the probe base).
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
353
|
+
// With labels visible and an excluded device listed, the default is not opened blind — it may BE
|
|
354
|
+
// the sensor (an idle Ultraleap is the default camera on some Windows boxes, and an eye tracker's
|
|
355
|
+
// camera should not be started at all) — the first camera that is not excluded is opened by id.
|
|
356
|
+
if (!mono && !defaultExcluded) {
|
|
357
|
+
const listed = await listCameras();
|
|
358
|
+
const anyExcluded = listed.filter(listedExcluded).length > 0; // every excluded device is noted, not just the first
|
|
359
|
+
const first = anyExcluded ? listed.find((d) => d.deviceId && !excludedIds.has(d.deviceId)) || null : null;
|
|
360
|
+
if (first) {
|
|
361
|
+
if (first.deviceId) tried.add(first.deviceId);
|
|
362
|
+
try {
|
|
363
|
+
mono = await open({ deviceId: { exact: first.deviceId }, ...MONO_CONSTRAINTS });
|
|
364
|
+
if (rejectOpened(mono)) mono = null;
|
|
365
|
+
} catch (err) {
|
|
366
|
+
skip(first.label, err);
|
|
367
|
+
}
|
|
368
|
+
} else {
|
|
369
|
+
try {
|
|
370
|
+
mono = await openDefault(MONO_CONSTRAINTS);
|
|
371
|
+
} catch (err) {
|
|
372
|
+
skip('default', err);
|
|
373
|
+
}
|
|
222
374
|
}
|
|
223
375
|
}
|
|
224
376
|
if (want === 'mono' && mono) return result(mono, 'mono');
|
|
@@ -230,11 +382,13 @@ export async function openCamera(want = 'auto', o = {}) {
|
|
|
230
382
|
} catch {
|
|
231
383
|
/* no enumeration: keep the default */
|
|
232
384
|
}
|
|
385
|
+
// A label that names a pair first; never by list position (see the header).
|
|
233
386
|
devices.sort((a, b) => Number(stereoLabelHint(b.label)) - Number(stereoLabelHint(a.label)));
|
|
234
387
|
for (const d of devices) {
|
|
235
388
|
if (!d.deviceId) continue; // no permission yet: an anonymous entry cannot be opened by id
|
|
236
389
|
if (mono && d.deviceId === mono.deviceId) continue;
|
|
237
|
-
if (tried.has(d.deviceId)) continue; //
|
|
390
|
+
if (tried.has(d.deviceId) || excludedIds.has(d.deviceId)) continue; // already opened (not a pair, or busy) or left out
|
|
391
|
+
if (listedExcluded(d)) continue; // the last resort never opens a sensor (a hand-tracking camera)
|
|
238
392
|
let r = null;
|
|
239
393
|
try {
|
|
240
394
|
r = await open({ deviceId: { exact: d.deviceId }, ...PROBE_CONSTRAINTS });
|
|
@@ -243,11 +397,18 @@ export async function openCamera(want = 'auto', o = {}) {
|
|
|
243
397
|
skip(d.label, err);
|
|
244
398
|
continue;
|
|
245
399
|
}
|
|
246
|
-
if (
|
|
400
|
+
if (rejectOpened(r)) continue;
|
|
401
|
+
if (isPair(r) && want !== 'mono') {
|
|
247
402
|
stop(mono);
|
|
248
403
|
log('camera-sbs', { label: r.label, width: r.width, height: r.height });
|
|
249
404
|
return result(r, 'sbs');
|
|
250
405
|
}
|
|
406
|
+
// No default could be used (excluded, busy): the first plain camera found is the fallback.
|
|
407
|
+
if (!mono && !isPair(r)) {
|
|
408
|
+
mono = await withMode(r, MONO_CONSTRAINTS);
|
|
409
|
+
if (want === 'mono') return result(mono, 'mono');
|
|
410
|
+
continue;
|
|
411
|
+
}
|
|
251
412
|
stop(r);
|
|
252
413
|
}
|
|
253
414
|
if (mono) {
|
|
@@ -261,11 +422,14 @@ export async function openCamera(want = 'auto', o = {}) {
|
|
|
261
422
|
);
|
|
262
423
|
}
|
|
263
424
|
|
|
264
|
-
/**
|
|
425
|
+
/**
|
|
426
|
+
* The microphone, with echo cancellation and noise suppression. Null if unavailable/denied.
|
|
427
|
+
* `deviceId`: that input exactly (`handle.setMic`).
|
|
428
|
+
*/
|
|
265
429
|
export async function openMic(o = {}) {
|
|
266
430
|
const md = o.mediaDevices || (globalThis.navigator && navigator.mediaDevices);
|
|
267
431
|
try {
|
|
268
|
-
const s = await md.getUserMedia({ audio: { ...AUDIO_CONSTRAINTS }, video: false });
|
|
432
|
+
const s = await md.getUserMedia({ audio: { ...AUDIO_CONSTRAINTS, ...(o.deviceId ? { deviceId: { exact: o.deviceId } } : {}) }, video: false });
|
|
269
433
|
return s.getAudioTracks()[0] || null;
|
|
270
434
|
} catch (err) {
|
|
271
435
|
(o.log || (() => {}))('mic-skip', { error: err && err.name });
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
// camera/conv-meter.js — the frame grab and the measuring loop behind auto-convergence, shared by
|
|
2
|
+
// a call's SENDER (it measures its own raw, pre-encode frames and sends the result to every peer:
|
|
3
|
+
// docs/call.md "Auto-convergence is measured by the sender") and a receiver's FALLBACK (an older
|
|
4
|
+
// sender that sends nothing; the depth cursor's copy). Internal: importable by path, not part of
|
|
5
|
+
// any entry. The measurement itself is camera/disparity.js.
|
|
6
|
+
|
|
7
|
+
import { lumaFromRgba, createDisparityTrack, createFocusTracker, downsampleLuma } from './disparity.js';
|
|
8
|
+
|
|
9
|
+
/** Sampling interval and the per-eye width of the measured copy: enough for sub-pixel disparity on a face, a few ms per measurement. */
|
|
10
|
+
export const CONV_METER_INTERVAL_MS = 200;
|
|
11
|
+
export const CONV_METER_EYE_WIDTH = 240;
|
|
12
|
+
|
|
13
|
+
/** WebCodecs pixel formats whose plane 0 is luma (Y): read directly, no colour conversion. */
|
|
14
|
+
export const Y_PLANE_FORMATS = new Set(['I420', 'I420A', 'I422', 'I444', 'NV12']);
|
|
15
|
+
|
|
16
|
+
const nowMs = () => (globalThis.performance ? performance.now() : Date.now());
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* A small grayscale copy of `video`'s current frame, per-eye width at most `eyeMax`.
|
|
20
|
+
*
|
|
21
|
+
* Preferred: WebCodecs. `new VideoFrame(video)` + an ASYNC `copyTo()`, reading the luma (Y) plane
|
|
22
|
+
* directly: no synchronous GPU->CPU canvas readback on the paint path (that readback, not the
|
|
23
|
+
* matching, was ~25 ms of a ~30 ms measurement). Otherwise a small canvas (`scratch`, reused) +
|
|
24
|
+
* `getImageData`. Resolves null for a frame that cannot be read (not decoded yet, tainted, a
|
|
25
|
+
* failed copy). Never rejects.
|
|
26
|
+
*
|
|
27
|
+
* `scale` = copy px per source px; `readMs` = main-thread time spent getting the pixels.
|
|
28
|
+
*
|
|
29
|
+
* @param {HTMLVideoElement} video
|
|
30
|
+
* @param {{eyeMax?: number, scratch?: HTMLCanvasElement|null}} [o]
|
|
31
|
+
* @returns {Promise<{img: Uint8Array, w: number, h: number, scale: number, via: 'videoframe'|'canvas', readMs: number}|null>}
|
|
32
|
+
*/
|
|
33
|
+
export async function grabLumaFrame(video, { eyeMax = CONV_METER_EYE_WIDTH, scratch = null } = {}) {
|
|
34
|
+
const W = video ? video.videoWidth : 0;
|
|
35
|
+
const H = video ? video.videoHeight : 0;
|
|
36
|
+
if (!W || !H || (video.readyState || 0) < 2) return null;
|
|
37
|
+
if (typeof VideoFrame === 'function') {
|
|
38
|
+
let frame = null;
|
|
39
|
+
try {
|
|
40
|
+
frame = new VideoFrame(video);
|
|
41
|
+
} catch {
|
|
42
|
+
frame = null;
|
|
43
|
+
}
|
|
44
|
+
if (frame && Y_PLANE_FORMATS.has(frame.format)) {
|
|
45
|
+
try {
|
|
46
|
+
const fw = frame.visibleRect ? frame.visibleRect.width : frame.displayWidth;
|
|
47
|
+
const fh = frame.visibleRect ? frame.visibleRect.height : frame.displayHeight;
|
|
48
|
+
const buf = new Uint8Array(frame.allocationSize());
|
|
49
|
+
const layout = await frame.copyTo(buf);
|
|
50
|
+
const t0 = nowMs();
|
|
51
|
+
const f = Math.max(1, Math.ceil(fw / 2 / eyeMax));
|
|
52
|
+
const ds = downsampleLuma(buf, layout[0].offset, layout[0].stride, fw, fh, f);
|
|
53
|
+
return { img: ds.img, w: ds.w, h: ds.h, scale: ds.w / fw, via: 'videoframe', readMs: nowMs() - t0 };
|
|
54
|
+
} catch {
|
|
55
|
+
return null;
|
|
56
|
+
} finally {
|
|
57
|
+
frame.close();
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
if (frame) frame.close();
|
|
61
|
+
}
|
|
62
|
+
// Fallback: a small canvas + getImageData (synchronous readback).
|
|
63
|
+
const t0 = nowMs();
|
|
64
|
+
const s = Math.min(1, eyeMax / (W / 2));
|
|
65
|
+
const w = Math.max(2, Math.round((W * s) / 2) * 2);
|
|
66
|
+
const h = Math.max(2, Math.round(H * s));
|
|
67
|
+
const c = scratch || (globalThis.document ? document.createElement('canvas') : null);
|
|
68
|
+
if (!c) return null;
|
|
69
|
+
if (c.width !== w || c.height !== h) {
|
|
70
|
+
c.width = w;
|
|
71
|
+
c.height = h;
|
|
72
|
+
}
|
|
73
|
+
try {
|
|
74
|
+
const g = c.getContext('2d', { willReadFrequently: true });
|
|
75
|
+
g.drawImage(video, 0, 0, w, h);
|
|
76
|
+
const img = lumaFromRgba(g.getImageData(0, 0, w, h).data, w * h);
|
|
77
|
+
return { img, w, h, scale: w / W, via: 'canvas', readMs: nowMs() - t0 };
|
|
78
|
+
} catch {
|
|
79
|
+
return null; // a frame that cannot be read (tainted, not decoded yet): skip it
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* The measuring loop: grab a copy at most every `intervalMs`, find the disparity of the point
|
|
85
|
+
* between the subject's eyes (camera/disparity.js focus tracker), median-of-3 it. A failed grab or
|
|
86
|
+
* measurement HOLDS the last value (a blink, a hand over the face). `reset()` forgets everything,
|
|
87
|
+
* including a grab still in flight.
|
|
88
|
+
*
|
|
89
|
+
* `sample(video, at)` resolves null when it did not sample (too soon, still busy, or reset while
|
|
90
|
+
* in flight), else `{ d, frac, luma, ncc, method, via, ms }`: `d` the held disparity in SOURCE px
|
|
91
|
+
* (left x − right x; null until a first lock), `frac = d / (videoWidth / 2)` (resolution-free, what
|
|
92
|
+
* goes on the wire), `luma` the copy (null if the grab failed), `ncc`/`method` of THIS measurement
|
|
93
|
+
* (null when it failed), `ms` = read + match time.
|
|
94
|
+
*
|
|
95
|
+
* @param {{intervalMs?: number, eyeMax?: number, grab?: typeof grabLumaFrame, now?: () => number}} [o]
|
|
96
|
+
*/
|
|
97
|
+
export function createConvergenceMeter({ intervalMs = CONV_METER_INTERVAL_MS, eyeMax = CONV_METER_EYE_WIDTH, grab = grabLumaFrame, now = nowMs } = {}) {
|
|
98
|
+
const focus = createFocusTracker();
|
|
99
|
+
const track = createDisparityTrack();
|
|
100
|
+
let lastAt = -Infinity;
|
|
101
|
+
let busy = false;
|
|
102
|
+
let gen = 0;
|
|
103
|
+
let scratch = null;
|
|
104
|
+
const meter = {
|
|
105
|
+
get busy() {
|
|
106
|
+
return busy;
|
|
107
|
+
},
|
|
108
|
+
/** The held disparity (source px), null until a first lock. */
|
|
109
|
+
get d() {
|
|
110
|
+
return track.value;
|
|
111
|
+
},
|
|
112
|
+
async sample(video, at = now()) {
|
|
113
|
+
if (busy || at - lastAt < intervalMs) return null;
|
|
114
|
+
lastAt = at;
|
|
115
|
+
busy = true;
|
|
116
|
+
const g0 = gen;
|
|
117
|
+
try {
|
|
118
|
+
if (!scratch && globalThis.document && typeof document.createElement === 'function') scratch = document.createElement('canvas');
|
|
119
|
+
let got = null;
|
|
120
|
+
try {
|
|
121
|
+
got = await grab(video, { eyeMax, scratch });
|
|
122
|
+
} catch {
|
|
123
|
+
got = null;
|
|
124
|
+
}
|
|
125
|
+
if (g0 !== gen) return null; // reset() while the grab was in flight: a stale frame
|
|
126
|
+
const t = now();
|
|
127
|
+
const m = got ? focus.measure(got.img, got.w, got.h, at) : null;
|
|
128
|
+
const d = track.push(m ? m.d / got.scale : null);
|
|
129
|
+
const W = video && video.videoWidth;
|
|
130
|
+
return {
|
|
131
|
+
d,
|
|
132
|
+
frac: d !== null && W > 0 ? d / (W / 2) : null,
|
|
133
|
+
luma: got ? { img: got.img, w: got.w, h: got.h, scale: got.scale } : null,
|
|
134
|
+
ncc: m ? m.c : null,
|
|
135
|
+
method: m ? m.method : null,
|
|
136
|
+
via: got ? got.via : null,
|
|
137
|
+
ms: (got ? got.readMs : 0) + (now() - t),
|
|
138
|
+
};
|
|
139
|
+
} finally {
|
|
140
|
+
if (g0 === gen) busy = false;
|
|
141
|
+
}
|
|
142
|
+
},
|
|
143
|
+
reset() {
|
|
144
|
+
gen++;
|
|
145
|
+
busy = false;
|
|
146
|
+
lastAt = -Infinity;
|
|
147
|
+
focus.reset();
|
|
148
|
+
track.reset();
|
|
149
|
+
},
|
|
150
|
+
};
|
|
151
|
+
return meter;
|
|
152
|
+
}
|