@lavitre/apps 1.51.0 → 1.53.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -0
- package/dist/Common/PreviewScaling.d.ts +62 -0
- package/dist/Common/PreviewScaling.js +132 -0
- package/dist/Common/PreviewScaling.js.map +1 -0
- package/dist/Common/index.d.ts +1 -0
- package/dist/Common/index.js +1 -0
- package/dist/Common/index.js.map +1 -1
- package/dist/Frame/Frame.d.ts +11 -0
- package/dist/Frame/Frame.js +45 -3
- package/dist/Frame/Frame.js.map +1 -1
- package/dist/Frame/LocalPreview.d.ts +133 -0
- package/dist/Frame/LocalPreview.js +823 -0
- package/dist/Frame/LocalPreview.js.map +1 -0
- package/dist/Frame/ScaledStreamWorker.d.ts +9 -0
- package/dist/Frame/ScaledStreamWorker.js +34 -0
- package/dist/Frame/ScaledStreamWorker.js.map +1 -0
- package/dist/Frame/index.d.ts +1 -0
- package/dist/Frame/index.js +1 -0
- package/dist/Frame/index.js.map +1 -1
- package/dist/Native/index.d.ts +17 -1
- package/dist/Native/index.js +7 -2
- package/dist/Native/index.js.map +1 -1
- package/dist/Version.d.ts +1 -1
- package/dist/Version.js +1 -1
- package/dist/Version.js.map +1 -1
- package/package.json +2 -1
|
@@ -0,0 +1,823 @@
|
|
|
1
|
+
import { computePreviewLayout, previewAspectRatio, previewMatrix, previewOutputSize, resolvePreviewScaling, } from '../Common/PreviewScaling.js';
|
|
2
|
+
import { SCALED_STREAM_WORKER_SOURCE } from './ScaledStreamWorker.js';
|
|
3
|
+
const MAX_RETRY_DELAY_MS = 30000;
|
|
4
|
+
const ENUMERATE_TIMEOUT_MS = 5000;
|
|
5
|
+
const GET_USER_MEDIA_TIMEOUT_MS = 15000;
|
|
6
|
+
const CAPTURE_TIMEOUT_MS = 5000;
|
|
7
|
+
const WORKER_FRAME_TIMEOUT_MS = 1000;
|
|
8
|
+
/** Consecutive unanswered frames before giving up on the worker: one late answer can be a main-thread pause. */
|
|
9
|
+
const WORKER_MAX_TIMEOUTS = 3;
|
|
10
|
+
/**
|
|
11
|
+
* Local camera preview cropped, zoomed and oriented the way the participants
|
|
12
|
+
* of the space see it — the behavior of the core app's "local preview".
|
|
13
|
+
*
|
|
14
|
+
* ```ts
|
|
15
|
+
* const preview = new LocalPreview(frame)
|
|
16
|
+
* preview.attach(document.getElementById('view')!) // CSS rendering, no per-frame cost
|
|
17
|
+
* // or: video.srcObject = preview.getScaledStream() // already-cropped MediaStream
|
|
18
|
+
* ```
|
|
19
|
+
*
|
|
20
|
+
* It opens the camera the Vitre uses as its main stream (retrying with a
|
|
21
|
+
* growing delay, reopening it when it ends or when the main camera changes)
|
|
22
|
+
* and follows the host's scaling updates through
|
|
23
|
+
* `Frame.addDeviceStateListener`, so the app's own `on…` callbacks stay
|
|
24
|
+
* untouched.
|
|
25
|
+
*/
|
|
26
|
+
export class LocalPreview {
|
|
27
|
+
constructor(frame, options = {}) {
|
|
28
|
+
Object.defineProperty(this, "frame", {
|
|
29
|
+
enumerable: true,
|
|
30
|
+
configurable: true,
|
|
31
|
+
writable: true,
|
|
32
|
+
value: frame
|
|
33
|
+
});
|
|
34
|
+
Object.defineProperty(this, "options", {
|
|
35
|
+
enumerable: true,
|
|
36
|
+
configurable: true,
|
|
37
|
+
writable: true,
|
|
38
|
+
value: void 0
|
|
39
|
+
});
|
|
40
|
+
Object.defineProperty(this, "currentStream", {
|
|
41
|
+
enumerable: true,
|
|
42
|
+
configurable: true,
|
|
43
|
+
writable: true,
|
|
44
|
+
value: null
|
|
45
|
+
});
|
|
46
|
+
Object.defineProperty(this, "cameraDeviceId", {
|
|
47
|
+
enumerable: true,
|
|
48
|
+
configurable: true,
|
|
49
|
+
writable: true,
|
|
50
|
+
value: null
|
|
51
|
+
});
|
|
52
|
+
Object.defineProperty(this, "currentScaling", {
|
|
53
|
+
enumerable: true,
|
|
54
|
+
configurable: true,
|
|
55
|
+
writable: true,
|
|
56
|
+
value: void 0
|
|
57
|
+
});
|
|
58
|
+
Object.defineProperty(this, "currentLayout", {
|
|
59
|
+
enumerable: true,
|
|
60
|
+
configurable: true,
|
|
61
|
+
writable: true,
|
|
62
|
+
value: void 0
|
|
63
|
+
});
|
|
64
|
+
Object.defineProperty(this, "currentError", {
|
|
65
|
+
enumerable: true,
|
|
66
|
+
configurable: true,
|
|
67
|
+
writable: true,
|
|
68
|
+
value: null
|
|
69
|
+
});
|
|
70
|
+
Object.defineProperty(this, "failures", {
|
|
71
|
+
enumerable: true,
|
|
72
|
+
configurable: true,
|
|
73
|
+
writable: true,
|
|
74
|
+
value: 0
|
|
75
|
+
});
|
|
76
|
+
Object.defineProperty(this, "nextAttemptAt", {
|
|
77
|
+
enumerable: true,
|
|
78
|
+
configurable: true,
|
|
79
|
+
writable: true,
|
|
80
|
+
value: 0
|
|
81
|
+
});
|
|
82
|
+
Object.defineProperty(this, "seenMediaDevices", {
|
|
83
|
+
enumerable: true,
|
|
84
|
+
configurable: true,
|
|
85
|
+
writable: true,
|
|
86
|
+
value: void 0
|
|
87
|
+
});
|
|
88
|
+
Object.defineProperty(this, "acquiring", {
|
|
89
|
+
enumerable: true,
|
|
90
|
+
configurable: true,
|
|
91
|
+
writable: true,
|
|
92
|
+
value: false
|
|
93
|
+
});
|
|
94
|
+
Object.defineProperty(this, "checkingCamera", {
|
|
95
|
+
enumerable: true,
|
|
96
|
+
configurable: true,
|
|
97
|
+
writable: true,
|
|
98
|
+
value: false
|
|
99
|
+
});
|
|
100
|
+
Object.defineProperty(this, "recheckCamera", {
|
|
101
|
+
enumerable: true,
|
|
102
|
+
configurable: true,
|
|
103
|
+
writable: true,
|
|
104
|
+
value: false
|
|
105
|
+
});
|
|
106
|
+
Object.defineProperty(this, "disposed", {
|
|
107
|
+
enumerable: true,
|
|
108
|
+
configurable: true,
|
|
109
|
+
writable: true,
|
|
110
|
+
value: false
|
|
111
|
+
});
|
|
112
|
+
Object.defineProperty(this, "listeners", {
|
|
113
|
+
enumerable: true,
|
|
114
|
+
configurable: true,
|
|
115
|
+
writable: true,
|
|
116
|
+
value: new Set()
|
|
117
|
+
});
|
|
118
|
+
Object.defineProperty(this, "cleanups", {
|
|
119
|
+
enumerable: true,
|
|
120
|
+
configurable: true,
|
|
121
|
+
writable: true,
|
|
122
|
+
value: new Set()
|
|
123
|
+
});
|
|
124
|
+
Object.defineProperty(this, "scaledStream", {
|
|
125
|
+
enumerable: true,
|
|
126
|
+
configurable: true,
|
|
127
|
+
writable: true,
|
|
128
|
+
value: null
|
|
129
|
+
});
|
|
130
|
+
Object.defineProperty(this, "stopScaledStream", {
|
|
131
|
+
enumerable: true,
|
|
132
|
+
configurable: true,
|
|
133
|
+
writable: true,
|
|
134
|
+
value: null
|
|
135
|
+
});
|
|
136
|
+
this.options = {
|
|
137
|
+
maxSize: options.maxSize ?? 1920,
|
|
138
|
+
retryDelayMs: options.retryDelayMs ?? 1000,
|
|
139
|
+
fitFrame: options.fitFrame ?? false,
|
|
140
|
+
enumerateDevices: options.enumerateDevices,
|
|
141
|
+
};
|
|
142
|
+
this.currentScaling = resolvePreviewScaling(frame.hostInformation);
|
|
143
|
+
this.currentLayout = computePreviewLayout(this.currentScaling);
|
|
144
|
+
this.seenMediaDevices = this.hostMediaDevices();
|
|
145
|
+
this.cleanups.add(frame.addDeviceStateListener(() => this.onDeviceStateChanged()));
|
|
146
|
+
const watchdog = setInterval(() => this.tick(), this.options.retryDelayMs);
|
|
147
|
+
this.cleanups.add(() => clearInterval(watchdog));
|
|
148
|
+
void this.tick();
|
|
149
|
+
}
|
|
150
|
+
get stream() {
|
|
151
|
+
return this.currentStream;
|
|
152
|
+
}
|
|
153
|
+
get scaling() {
|
|
154
|
+
return this.currentScaling;
|
|
155
|
+
}
|
|
156
|
+
get layout() {
|
|
157
|
+
return this.currentLayout;
|
|
158
|
+
}
|
|
159
|
+
get error() {
|
|
160
|
+
return this.currentError;
|
|
161
|
+
}
|
|
162
|
+
get state() {
|
|
163
|
+
return {
|
|
164
|
+
stream: this.currentStream,
|
|
165
|
+
scaling: this.currentScaling,
|
|
166
|
+
layout: this.currentLayout,
|
|
167
|
+
error: this.currentError,
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
/** Called with the new state whenever the stream, the scaling or the error changes. Returns the unsubscribe function. */
|
|
171
|
+
addListener(listener) {
|
|
172
|
+
this.listeners.add(listener);
|
|
173
|
+
return () => this.listeners.delete(listener);
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Renders the preview inside `container` with CSS transforms on a
|
|
177
|
+
* `<video>`: no per-frame work. Returns the detach function.
|
|
178
|
+
*/
|
|
179
|
+
attach(container, options = {}) {
|
|
180
|
+
const fit = options.fit ?? 'contain';
|
|
181
|
+
const root = document.createElement('div');
|
|
182
|
+
root.className = 'lavitre-local-preview';
|
|
183
|
+
root.style.cssText =
|
|
184
|
+
'position:relative;width:100%;height:100%;overflow:hidden;background:black';
|
|
185
|
+
const box = document.createElement('div');
|
|
186
|
+
box.style.cssText = 'position:absolute;overflow:hidden';
|
|
187
|
+
const video = createVideoElement();
|
|
188
|
+
video.style.cssText = 'position:absolute;left:0;top:0;transform-origin:0 0';
|
|
189
|
+
box.appendChild(video);
|
|
190
|
+
root.appendChild(box);
|
|
191
|
+
container.appendChild(root);
|
|
192
|
+
const update = () => {
|
|
193
|
+
if (video.srcObject !== this.currentStream) {
|
|
194
|
+
video.srcObject = this.currentStream;
|
|
195
|
+
}
|
|
196
|
+
const w = video.videoWidth;
|
|
197
|
+
const h = video.videoHeight;
|
|
198
|
+
box.style.visibility = this.currentStream && w && h ? 'visible' : 'hidden';
|
|
199
|
+
if (!w || !h)
|
|
200
|
+
return;
|
|
201
|
+
const ratio = previewAspectRatio(this.currentLayout, w, h);
|
|
202
|
+
const cw = root.clientWidth;
|
|
203
|
+
const ch = root.clientHeight;
|
|
204
|
+
const boxWidth = fit === 'contain' ? Math.min(cw, ch * ratio) : Math.max(cw, ch * ratio);
|
|
205
|
+
const boxHeight = boxWidth / ratio;
|
|
206
|
+
box.style.left = `${(cw - boxWidth) / 2}px`;
|
|
207
|
+
box.style.top = `${(ch - boxHeight) / 2}px`;
|
|
208
|
+
box.style.width = `${boxWidth}px`;
|
|
209
|
+
box.style.height = `${boxHeight}px`;
|
|
210
|
+
video.style.width = `${w}px`;
|
|
211
|
+
video.style.height = `${h}px`;
|
|
212
|
+
video.style.transform = `matrix(${previewMatrix(this.currentLayout, w, h, boxWidth, boxHeight).join(',')})`;
|
|
213
|
+
};
|
|
214
|
+
const resizeObserver = new ResizeObserver(update);
|
|
215
|
+
resizeObserver.observe(root);
|
|
216
|
+
video.addEventListener('loadedmetadata', update);
|
|
217
|
+
video.addEventListener('resize', update);
|
|
218
|
+
const unsubscribe = this.addListener(update);
|
|
219
|
+
update();
|
|
220
|
+
const detach = () => {
|
|
221
|
+
this.cleanups.delete(detach);
|
|
222
|
+
unsubscribe();
|
|
223
|
+
resizeObserver.disconnect();
|
|
224
|
+
video.srcObject = null;
|
|
225
|
+
root.remove();
|
|
226
|
+
};
|
|
227
|
+
this.cleanups.add(detach);
|
|
228
|
+
return detach;
|
|
229
|
+
}
|
|
230
|
+
/**
|
|
231
|
+
* A `MediaStream` whose frames are already cropped, zoomed, rotated and
|
|
232
|
+
* mirrored: plug it into a `<video>`, WebGL, a recorder… On Chromium each
|
|
233
|
+
* frame goes through insertable streams and is redrawn in a worker — on
|
|
234
|
+
* the main thread while the worker starts, or for good if it cannot run
|
|
235
|
+
* (CSP without `blob:` in `worker-src`). Elsewhere a canvas redraws a
|
|
236
|
+
* hidden `<video>` (`options.frameRate`). It survives camera changes. The
|
|
237
|
+
* same stream is returned until its track ends (stopped by the app, or
|
|
238
|
+
* `dispose()`); the next call then builds a new one.
|
|
239
|
+
*/
|
|
240
|
+
getScaledStream(options = {}) {
|
|
241
|
+
const live = this.scaledStream
|
|
242
|
+
?.getVideoTracks()
|
|
243
|
+
.some((t) => t.readyState === 'live');
|
|
244
|
+
if (this.scaledStream && live)
|
|
245
|
+
return this.scaledStream;
|
|
246
|
+
this.stopScaledStream?.();
|
|
247
|
+
const pipeline = this.startFramePipeline() ??
|
|
248
|
+
this.startCanvasPipeline(options.frameRate ?? 30);
|
|
249
|
+
const stop = () => {
|
|
250
|
+
this.cleanups.delete(stop);
|
|
251
|
+
pipeline.stop();
|
|
252
|
+
if (this.scaledStream === pipeline.stream) {
|
|
253
|
+
this.scaledStream = null;
|
|
254
|
+
this.stopScaledStream = null;
|
|
255
|
+
}
|
|
256
|
+
};
|
|
257
|
+
this.scaledStream = pipeline.stream;
|
|
258
|
+
this.stopScaledStream = stop;
|
|
259
|
+
this.cleanups.add(stop);
|
|
260
|
+
return pipeline.stream;
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* Still image of the preview, as the participants see it. Rejects when the
|
|
264
|
+
* camera is not open or yields no image within 5 s.
|
|
265
|
+
*/
|
|
266
|
+
async capture(type = 'image/jpeg', quality) {
|
|
267
|
+
const stream = this.currentStream;
|
|
268
|
+
if (!stream)
|
|
269
|
+
throw new Error('camera not open');
|
|
270
|
+
const video = createVideoElement();
|
|
271
|
+
video.srcObject = stream;
|
|
272
|
+
try {
|
|
273
|
+
await waitForVideoData(video, CAPTURE_TIMEOUT_MS);
|
|
274
|
+
const sw = video.videoWidth;
|
|
275
|
+
const sh = video.videoHeight;
|
|
276
|
+
const { width, height } = previewOutputSize(this.currentLayout, sw, sh, this.options.maxSize);
|
|
277
|
+
const canvas = document.createElement('canvas');
|
|
278
|
+
canvas.width = width;
|
|
279
|
+
canvas.height = height;
|
|
280
|
+
paintPreview(canvas.getContext('2d'), width, height, previewMatrix(this.currentLayout, sw, sh, width, height), video, sw, sh);
|
|
281
|
+
return await new Promise((resolve, reject) => canvas.toBlob((blob) => (blob ? resolve(blob) : reject(new Error('toBlob failed'))), type, quality));
|
|
282
|
+
}
|
|
283
|
+
finally {
|
|
284
|
+
video.srcObject = null;
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
/** Stops the camera, the scaled stream and every attached rendering. */
|
|
288
|
+
dispose() {
|
|
289
|
+
if (this.disposed)
|
|
290
|
+
return;
|
|
291
|
+
this.disposed = true;
|
|
292
|
+
for (const cleanup of [...this.cleanups].reverse())
|
|
293
|
+
cleanup();
|
|
294
|
+
this.cleanups.clear();
|
|
295
|
+
this.setStream(null);
|
|
296
|
+
this.listeners.clear();
|
|
297
|
+
}
|
|
298
|
+
notify() {
|
|
299
|
+
const state = this.state;
|
|
300
|
+
for (const listener of this.listeners) {
|
|
301
|
+
try {
|
|
302
|
+
listener(state);
|
|
303
|
+
}
|
|
304
|
+
catch (error) {
|
|
305
|
+
console.error('[LocalPreview] listener failed', error);
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
setStream(stream) {
|
|
310
|
+
if (stream === this.currentStream)
|
|
311
|
+
return;
|
|
312
|
+
for (const t of this.currentStream?.getTracks() ?? [])
|
|
313
|
+
t.stop();
|
|
314
|
+
this.currentStream = stream;
|
|
315
|
+
if (!stream)
|
|
316
|
+
this.cameraDeviceId = null;
|
|
317
|
+
this.notify();
|
|
318
|
+
}
|
|
319
|
+
setError(error) {
|
|
320
|
+
const previous = this.currentError;
|
|
321
|
+
if (previous?.name === error?.name &&
|
|
322
|
+
previous?.message === error?.message) {
|
|
323
|
+
return;
|
|
324
|
+
}
|
|
325
|
+
if (error && error.name !== previous?.name) {
|
|
326
|
+
console.warn(`[LocalPreview] camera not open: ${error.name}: ${error.message}`);
|
|
327
|
+
}
|
|
328
|
+
this.currentError = error;
|
|
329
|
+
this.notify();
|
|
330
|
+
}
|
|
331
|
+
/**
|
|
332
|
+
* The host's cameras as a comparable key: `vitreSettingsUpdated` replaces
|
|
333
|
+
* the whole list on any settings change, so its reference says nothing.
|
|
334
|
+
*/
|
|
335
|
+
hostMediaDevices() {
|
|
336
|
+
const devices = this.frame.hostInformation.vitreSettings?.availableMediaDevices ?? [];
|
|
337
|
+
return JSON.stringify(devices
|
|
338
|
+
.filter((d) => d.kind === 'videoinput')
|
|
339
|
+
.map((d) => [d.deviceId, d.inUseBy]));
|
|
340
|
+
}
|
|
341
|
+
onDeviceStateChanged() {
|
|
342
|
+
const scaling = resolvePreviewScaling(this.frame.hostInformation);
|
|
343
|
+
if (JSON.stringify(scaling) !== JSON.stringify(this.currentScaling)) {
|
|
344
|
+
this.currentScaling = scaling;
|
|
345
|
+
this.currentLayout = computePreviewLayout(scaling);
|
|
346
|
+
this.notify();
|
|
347
|
+
}
|
|
348
|
+
// The list of cameras changed: the main one may have been switched, or
|
|
349
|
+
// the missing one plugged in — retry now rather than after the backoff.
|
|
350
|
+
const mediaDevices = this.hostMediaDevices();
|
|
351
|
+
if (mediaDevices !== this.seenMediaDevices) {
|
|
352
|
+
this.seenMediaDevices = mediaDevices;
|
|
353
|
+
this.failures = 0;
|
|
354
|
+
this.nextAttemptAt = 0;
|
|
355
|
+
if (this.currentStream)
|
|
356
|
+
void this.checkMainCamera();
|
|
357
|
+
else
|
|
358
|
+
void this.tick();
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
enumerate() {
|
|
362
|
+
const enumerate = this.options.enumerateDevices ??
|
|
363
|
+
(() => {
|
|
364
|
+
const connector = this.frame.getMainConnector();
|
|
365
|
+
if (!connector) {
|
|
366
|
+
throw namedError('NotReadyError', 'main connector not available');
|
|
367
|
+
}
|
|
368
|
+
return connector.requestEnumerateDevices();
|
|
369
|
+
});
|
|
370
|
+
return withTimeout(Promise.resolve().then(enumerate), ENUMERATE_TIMEOUT_MS, 'listing the cameras');
|
|
371
|
+
}
|
|
372
|
+
/** Reopens the camera when the main one is no longer the one open. */
|
|
373
|
+
async checkMainCamera() {
|
|
374
|
+
if (this.checkingCamera) {
|
|
375
|
+
this.recheckCamera = true;
|
|
376
|
+
return;
|
|
377
|
+
}
|
|
378
|
+
this.checkingCamera = true;
|
|
379
|
+
try {
|
|
380
|
+
const camera = findMainCamera(await this.enumerate());
|
|
381
|
+
if (this.currentStream &&
|
|
382
|
+
camera &&
|
|
383
|
+
camera.deviceId !== this.cameraDeviceId) {
|
|
384
|
+
this.setStream(null);
|
|
385
|
+
void this.tick();
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
catch (_error) {
|
|
389
|
+
// Transient; the next device change checks again.
|
|
390
|
+
}
|
|
391
|
+
finally {
|
|
392
|
+
this.checkingCamera = false;
|
|
393
|
+
if (this.recheckCamera) {
|
|
394
|
+
this.recheckCamera = false;
|
|
395
|
+
void this.checkMainCamera();
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
async tick() {
|
|
400
|
+
if (this.disposed)
|
|
401
|
+
return;
|
|
402
|
+
const track = this.currentStream?.getVideoTracks()[0];
|
|
403
|
+
if (this.currentStream && (!track || track.readyState === 'ended')) {
|
|
404
|
+
this.setStream(null);
|
|
405
|
+
}
|
|
406
|
+
if (this.currentStream || this.acquiring)
|
|
407
|
+
return;
|
|
408
|
+
if (Date.now() < this.nextAttemptAt)
|
|
409
|
+
return;
|
|
410
|
+
this.acquiring = true;
|
|
411
|
+
try {
|
|
412
|
+
await this.openCamera();
|
|
413
|
+
this.failures = 0;
|
|
414
|
+
this.nextAttemptAt = 0;
|
|
415
|
+
this.setError(null);
|
|
416
|
+
}
|
|
417
|
+
catch (error) {
|
|
418
|
+
this.failures += 1;
|
|
419
|
+
this.nextAttemptAt =
|
|
420
|
+
Date.now() +
|
|
421
|
+
Math.min(this.options.retryDelayMs * 2 ** (this.failures - 1), MAX_RETRY_DELAY_MS);
|
|
422
|
+
if (!this.disposed)
|
|
423
|
+
this.setError(toPreviewError(error));
|
|
424
|
+
}
|
|
425
|
+
finally {
|
|
426
|
+
this.acquiring = false;
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
async openCamera() {
|
|
430
|
+
const devicesBefore = this.hostMediaDevices();
|
|
431
|
+
const camera = findMainCamera(await this.enumerate());
|
|
432
|
+
if (!camera) {
|
|
433
|
+
throw namedError('NotFoundError', 'no camera used as main stream');
|
|
434
|
+
}
|
|
435
|
+
const sideways = camera.orientation === 'left' || camera.orientation === 'right';
|
|
436
|
+
const video = {
|
|
437
|
+
deviceId: { exact: camera.deviceId },
|
|
438
|
+
};
|
|
439
|
+
if (camera.orientationBySystem === true && sideways) {
|
|
440
|
+
video.height = { max: this.options.maxSize };
|
|
441
|
+
}
|
|
442
|
+
else {
|
|
443
|
+
video.width = { max: this.options.maxSize };
|
|
444
|
+
}
|
|
445
|
+
const stream = await withTimeout(navigator.mediaDevices.getUserMedia({ video }), GET_USER_MEDIA_TIMEOUT_MS, 'opening the camera', (late) => {
|
|
446
|
+
for (const t of late.getTracks())
|
|
447
|
+
t.stop();
|
|
448
|
+
});
|
|
449
|
+
if (this.disposed) {
|
|
450
|
+
for (const t of stream.getTracks())
|
|
451
|
+
t.stop();
|
|
452
|
+
return;
|
|
453
|
+
}
|
|
454
|
+
const track = stream.getVideoTracks()[0];
|
|
455
|
+
track?.addEventListener('ended', () => {
|
|
456
|
+
if (this.currentStream === stream)
|
|
457
|
+
this.setStream(null);
|
|
458
|
+
});
|
|
459
|
+
this.cameraDeviceId = camera.deviceId;
|
|
460
|
+
this.setStream(stream);
|
|
461
|
+
if (this.options.fitFrame && track)
|
|
462
|
+
this.fitFrame(track);
|
|
463
|
+
// The main camera may have changed while it was opening.
|
|
464
|
+
if (this.hostMediaDevices() !== devicesBefore)
|
|
465
|
+
void this.checkMainCamera();
|
|
466
|
+
}
|
|
467
|
+
fitFrame(track) {
|
|
468
|
+
const { width, height } = track.getSettings();
|
|
469
|
+
if (!width || !height)
|
|
470
|
+
return;
|
|
471
|
+
const portrait = previewAspectRatio(this.currentLayout, width, height) < 1;
|
|
472
|
+
const { width: fw, height: fh } = this.frame.position;
|
|
473
|
+
const short = Math.min(fw, fh);
|
|
474
|
+
const long = Math.max(fw, fh);
|
|
475
|
+
this.frame.setPosition(portrait
|
|
476
|
+
? { width: `${short}px`, height: `${long}px` }
|
|
477
|
+
: { width: `${long}px`, height: `${short}px` });
|
|
478
|
+
}
|
|
479
|
+
/** Chromium: camera frames → transform (worker, else main thread) → generator track. */
|
|
480
|
+
startFramePipeline() {
|
|
481
|
+
if (typeof MediaStreamTrackProcessor === 'undefined' ||
|
|
482
|
+
typeof MediaStreamTrackGenerator === 'undefined' ||
|
|
483
|
+
typeof VideoFrame === 'undefined' ||
|
|
484
|
+
typeof OffscreenCanvas === 'undefined') {
|
|
485
|
+
return null;
|
|
486
|
+
}
|
|
487
|
+
const generator = new MediaStreamTrackGenerator({ kind: 'video' });
|
|
488
|
+
const writer = generator.writable.getWriter();
|
|
489
|
+
const transformer = new FrameTransformer();
|
|
490
|
+
let source = null;
|
|
491
|
+
let reader = null;
|
|
492
|
+
let stopped = false;
|
|
493
|
+
const pump = async (current) => {
|
|
494
|
+
for (;;) {
|
|
495
|
+
let result;
|
|
496
|
+
try {
|
|
497
|
+
result = await current.read();
|
|
498
|
+
}
|
|
499
|
+
catch (_error) {
|
|
500
|
+
return;
|
|
501
|
+
}
|
|
502
|
+
if (result.done)
|
|
503
|
+
return;
|
|
504
|
+
const frame = result.value;
|
|
505
|
+
if (current !== reader) {
|
|
506
|
+
frame.close();
|
|
507
|
+
return;
|
|
508
|
+
}
|
|
509
|
+
const sw = frame.displayWidth;
|
|
510
|
+
const sh = frame.displayHeight;
|
|
511
|
+
const { width, height } = previewOutputSize(this.currentLayout, sw, sh, this.options.maxSize);
|
|
512
|
+
const matrix = previewMatrix(this.currentLayout, sw, sh, width, height);
|
|
513
|
+
let out;
|
|
514
|
+
try {
|
|
515
|
+
out = await transformer.transform(frame, width, height, matrix);
|
|
516
|
+
}
|
|
517
|
+
catch (error) {
|
|
518
|
+
// Drop this frame, keep the stream going.
|
|
519
|
+
frame.close();
|
|
520
|
+
console.warn('[LocalPreview] frame dropped', error);
|
|
521
|
+
continue;
|
|
522
|
+
}
|
|
523
|
+
if (!out)
|
|
524
|
+
continue;
|
|
525
|
+
if (current !== reader) {
|
|
526
|
+
out.close();
|
|
527
|
+
return;
|
|
528
|
+
}
|
|
529
|
+
try {
|
|
530
|
+
await writer.write(out);
|
|
531
|
+
}
|
|
532
|
+
catch (_error) {
|
|
533
|
+
// The app stopped the track.
|
|
534
|
+
out.close();
|
|
535
|
+
stop();
|
|
536
|
+
return;
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
};
|
|
540
|
+
const sync = () => {
|
|
541
|
+
const track = this.currentStream?.getVideoTracks()[0] ?? null;
|
|
542
|
+
if (track === source)
|
|
543
|
+
return;
|
|
544
|
+
source = track;
|
|
545
|
+
reader?.cancel().catch(() => { });
|
|
546
|
+
reader = track
|
|
547
|
+
? new MediaStreamTrackProcessor({
|
|
548
|
+
track,
|
|
549
|
+
}).readable.getReader()
|
|
550
|
+
: null;
|
|
551
|
+
if (reader)
|
|
552
|
+
void pump(reader);
|
|
553
|
+
};
|
|
554
|
+
const unsubscribe = this.addListener(sync);
|
|
555
|
+
const stop = () => {
|
|
556
|
+
if (stopped)
|
|
557
|
+
return;
|
|
558
|
+
stopped = true;
|
|
559
|
+
unsubscribe();
|
|
560
|
+
reader?.cancel().catch(() => { });
|
|
561
|
+
reader = null;
|
|
562
|
+
transformer.dispose();
|
|
563
|
+
generator.stop();
|
|
564
|
+
};
|
|
565
|
+
sync();
|
|
566
|
+
return { stream: new MediaStream([generator]), stop };
|
|
567
|
+
}
|
|
568
|
+
/** Fallback: hidden `<video>` drawn into a canvas on every new camera frame. */
|
|
569
|
+
startCanvasPipeline(frameRate) {
|
|
570
|
+
const video = createVideoElement();
|
|
571
|
+
const canvas = document.createElement('canvas');
|
|
572
|
+
const ctx = canvas.getContext('2d');
|
|
573
|
+
const stream = canvas.captureStream(frameRate);
|
|
574
|
+
// One frame-callback chain per camera stream; a stale chain stops at its
|
|
575
|
+
// next callback, and none runs while there is no camera.
|
|
576
|
+
let chain = 0;
|
|
577
|
+
const draw = (id) => {
|
|
578
|
+
if (id !== chain || !video.srcObject)
|
|
579
|
+
return;
|
|
580
|
+
const sw = video.videoWidth;
|
|
581
|
+
const sh = video.videoHeight;
|
|
582
|
+
if (sw && sh) {
|
|
583
|
+
const { width, height } = previewOutputSize(this.currentLayout, sw, sh, this.options.maxSize);
|
|
584
|
+
if (canvas.width !== width)
|
|
585
|
+
canvas.width = width;
|
|
586
|
+
if (canvas.height !== height)
|
|
587
|
+
canvas.height = height;
|
|
588
|
+
paintPreview(ctx, width, height, previewMatrix(this.currentLayout, sw, sh, width, height), video, sw, sh);
|
|
589
|
+
}
|
|
590
|
+
if ('requestVideoFrameCallback' in video) {
|
|
591
|
+
;
|
|
592
|
+
video.requestVideoFrameCallback(() => draw(id));
|
|
593
|
+
}
|
|
594
|
+
else {
|
|
595
|
+
setTimeout(() => draw(id), 1000 / frameRate);
|
|
596
|
+
}
|
|
597
|
+
};
|
|
598
|
+
const sync = () => {
|
|
599
|
+
if (video.srcObject === this.currentStream)
|
|
600
|
+
return;
|
|
601
|
+
video.srcObject = this.currentStream;
|
|
602
|
+
draw(++chain);
|
|
603
|
+
};
|
|
604
|
+
const unsubscribe = this.addListener(sync);
|
|
605
|
+
sync();
|
|
606
|
+
return {
|
|
607
|
+
stream,
|
|
608
|
+
stop: () => {
|
|
609
|
+
chain = -1;
|
|
610
|
+
unsubscribe();
|
|
611
|
+
video.srcObject = null;
|
|
612
|
+
for (const t of stream.getTracks())
|
|
613
|
+
t.stop();
|
|
614
|
+
},
|
|
615
|
+
};
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
/**
|
|
619
|
+
* Redraws camera frames, in a worker when one can run. Until the worker says
|
|
620
|
+
* it is ready, and for good once it fails (load error, CSP, three frames in a
|
|
621
|
+
* row without an answer within a second), frames are drawn on the main
|
|
622
|
+
* thread instead.
|
|
623
|
+
*/
|
|
624
|
+
class FrameTransformer {
|
|
625
|
+
constructor() {
|
|
626
|
+
Object.defineProperty(this, "worker", {
|
|
627
|
+
enumerable: true,
|
|
628
|
+
configurable: true,
|
|
629
|
+
writable: true,
|
|
630
|
+
value: null
|
|
631
|
+
});
|
|
632
|
+
Object.defineProperty(this, "ready", {
|
|
633
|
+
enumerable: true,
|
|
634
|
+
configurable: true,
|
|
635
|
+
writable: true,
|
|
636
|
+
value: false
|
|
637
|
+
});
|
|
638
|
+
Object.defineProperty(this, "nextId", {
|
|
639
|
+
enumerable: true,
|
|
640
|
+
configurable: true,
|
|
641
|
+
writable: true,
|
|
642
|
+
value: 0
|
|
643
|
+
});
|
|
644
|
+
Object.defineProperty(this, "timeouts", {
|
|
645
|
+
enumerable: true,
|
|
646
|
+
configurable: true,
|
|
647
|
+
writable: true,
|
|
648
|
+
value: 0
|
|
649
|
+
});
|
|
650
|
+
Object.defineProperty(this, "pending", {
|
|
651
|
+
enumerable: true,
|
|
652
|
+
configurable: true,
|
|
653
|
+
writable: true,
|
|
654
|
+
value: new Map()
|
|
655
|
+
});
|
|
656
|
+
Object.defineProperty(this, "canvas", {
|
|
657
|
+
enumerable: true,
|
|
658
|
+
configurable: true,
|
|
659
|
+
writable: true,
|
|
660
|
+
value: null
|
|
661
|
+
});
|
|
662
|
+
Object.defineProperty(this, "ctx", {
|
|
663
|
+
enumerable: true,
|
|
664
|
+
configurable: true,
|
|
665
|
+
writable: true,
|
|
666
|
+
value: null
|
|
667
|
+
});
|
|
668
|
+
if (typeof Worker === 'undefined')
|
|
669
|
+
return;
|
|
670
|
+
const url = URL.createObjectURL(new Blob([SCALED_STREAM_WORKER_SOURCE], { type: 'text/javascript' }));
|
|
671
|
+
try {
|
|
672
|
+
const worker = new Worker(url);
|
|
673
|
+
worker.onmessage = (event) => {
|
|
674
|
+
const msg = event.data;
|
|
675
|
+
if (msg?.type === 'ready') {
|
|
676
|
+
this.ready = true;
|
|
677
|
+
return;
|
|
678
|
+
}
|
|
679
|
+
const resolve = this.pending.get(msg?.id);
|
|
680
|
+
if (!resolve) {
|
|
681
|
+
msg?.frame?.close();
|
|
682
|
+
return;
|
|
683
|
+
}
|
|
684
|
+
this.pending.delete(msg.id);
|
|
685
|
+
this.timeouts = 0;
|
|
686
|
+
resolve(msg.frame ?? null);
|
|
687
|
+
if (msg.error)
|
|
688
|
+
this.fail(msg.error);
|
|
689
|
+
};
|
|
690
|
+
worker.onerror = (event) => {
|
|
691
|
+
event.preventDefault();
|
|
692
|
+
this.fail(event.message || 'worker error');
|
|
693
|
+
};
|
|
694
|
+
worker.onmessageerror = () => this.fail('worker message error');
|
|
695
|
+
this.worker = worker;
|
|
696
|
+
}
|
|
697
|
+
catch (error) {
|
|
698
|
+
console.warn(`[LocalPreview] scaled-stream worker unavailable, drawing on the main thread: ${error}`);
|
|
699
|
+
}
|
|
700
|
+
finally {
|
|
701
|
+
// The worker has fetched its script by the time it posts 'ready'; a
|
|
702
|
+
// worker revoked too early fails with an error event and is replaced
|
|
703
|
+
// by the main thread like any other failure.
|
|
704
|
+
setTimeout(() => URL.revokeObjectURL(url), 10000);
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
/** Takes ownership of `frame`. Null when the frame was lost to a failing worker. */
|
|
708
|
+
async transform(frame, width, height, matrix) {
|
|
709
|
+
const worker = this.worker;
|
|
710
|
+
if (worker && this.ready) {
|
|
711
|
+
const id = this.nextId++;
|
|
712
|
+
return await new Promise((resolve) => {
|
|
713
|
+
const timer = setTimeout(() => {
|
|
714
|
+
if (this.pending.delete(id)) {
|
|
715
|
+
resolve(null);
|
|
716
|
+
this.timeouts += 1;
|
|
717
|
+
if (this.timeouts >= WORKER_MAX_TIMEOUTS) {
|
|
718
|
+
this.fail('no answer from the worker');
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
}, WORKER_FRAME_TIMEOUT_MS);
|
|
722
|
+
this.pending.set(id, (out) => {
|
|
723
|
+
clearTimeout(timer);
|
|
724
|
+
resolve(out);
|
|
725
|
+
});
|
|
726
|
+
worker.postMessage({ id, frame, width, height, matrix }, [frame]);
|
|
727
|
+
});
|
|
728
|
+
}
|
|
729
|
+
try {
|
|
730
|
+
if (!this.canvas ||
|
|
731
|
+
this.canvas.width !== width ||
|
|
732
|
+
this.canvas.height !== height) {
|
|
733
|
+
this.canvas = new OffscreenCanvas(width, height);
|
|
734
|
+
this.ctx = this.canvas.getContext('2d');
|
|
735
|
+
}
|
|
736
|
+
paintPreview(this.ctx, width, height, matrix, frame, frame.displayWidth, frame.displayHeight);
|
|
737
|
+
return new VideoFrame(this.canvas, { timestamp: frame.timestamp });
|
|
738
|
+
}
|
|
739
|
+
finally {
|
|
740
|
+
frame.close();
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
dispose() {
|
|
744
|
+
this.worker?.terminate();
|
|
745
|
+
this.worker = null;
|
|
746
|
+
this.ready = false;
|
|
747
|
+
for (const resolve of this.pending.values())
|
|
748
|
+
resolve(null);
|
|
749
|
+
this.pending.clear();
|
|
750
|
+
}
|
|
751
|
+
fail(reason) {
|
|
752
|
+
if (!this.worker)
|
|
753
|
+
return;
|
|
754
|
+
console.warn(`[LocalPreview] scaled-stream worker disabled, drawing on the main thread: ${reason}`);
|
|
755
|
+
this.dispose();
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
const findMainCamera = (devices) => devices.find((d) => d.kind === 'videoinput' && d.inUseBy?.includes('mainStream'));
|
|
759
|
+
const createVideoElement = () => {
|
|
760
|
+
const video = document.createElement('video');
|
|
761
|
+
video.muted = true;
|
|
762
|
+
video.autoplay = true;
|
|
763
|
+
video.playsInline = true;
|
|
764
|
+
return video;
|
|
765
|
+
};
|
|
766
|
+
/** Black background, then the source through the preview matrix. Same drawing as the worker. */
|
|
767
|
+
const paintPreview = (ctx, width, height, matrix, source, sourceWidth, sourceHeight) => {
|
|
768
|
+
ctx.setTransform(1, 0, 0, 1, 0, 0);
|
|
769
|
+
ctx.fillStyle = 'black';
|
|
770
|
+
ctx.fillRect(0, 0, width, height);
|
|
771
|
+
ctx.setTransform(...matrix);
|
|
772
|
+
ctx.drawImage(source, 0, 0, sourceWidth, sourceHeight);
|
|
773
|
+
};
|
|
774
|
+
const namedError = (name, message) => {
|
|
775
|
+
const error = new Error(message);
|
|
776
|
+
error.name = name;
|
|
777
|
+
return error;
|
|
778
|
+
};
|
|
779
|
+
const toPreviewError = (error) => error instanceof Error || error instanceof DOMException
|
|
780
|
+
? { name: error.name, message: error.message }
|
|
781
|
+
: { name: 'Error', message: String(error) };
|
|
782
|
+
/** Rejects with a `TimeoutError` after `ms`; a value arriving later goes to `onLate` (to release it). */
|
|
783
|
+
const withTimeout = (promise, ms, what, onLate) => new Promise((resolve, reject) => {
|
|
784
|
+
let timedOut = false;
|
|
785
|
+
const timer = setTimeout(() => {
|
|
786
|
+
timedOut = true;
|
|
787
|
+
reject(namedError('TimeoutError', `${what} took more than ${ms} ms`));
|
|
788
|
+
}, ms);
|
|
789
|
+
promise.then((value) => {
|
|
790
|
+
clearTimeout(timer);
|
|
791
|
+
if (timedOut)
|
|
792
|
+
onLate?.(value);
|
|
793
|
+
else
|
|
794
|
+
resolve(value);
|
|
795
|
+
}, (error) => {
|
|
796
|
+
clearTimeout(timer);
|
|
797
|
+
if (!timedOut)
|
|
798
|
+
reject(error);
|
|
799
|
+
});
|
|
800
|
+
});
|
|
801
|
+
const waitForVideoData = (video, ms) => new Promise((resolve, reject) => {
|
|
802
|
+
if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {
|
|
803
|
+
resolve();
|
|
804
|
+
return;
|
|
805
|
+
}
|
|
806
|
+
const done = (error) => {
|
|
807
|
+
clearTimeout(timer);
|
|
808
|
+
video.removeEventListener('loadeddata', onData);
|
|
809
|
+
video.removeEventListener('error', onError);
|
|
810
|
+
video.removeEventListener('emptied', onError);
|
|
811
|
+
if (error)
|
|
812
|
+
reject(error);
|
|
813
|
+
else
|
|
814
|
+
resolve();
|
|
815
|
+
};
|
|
816
|
+
const onData = () => done();
|
|
817
|
+
const onError = () => done(new Error('camera stream ended'));
|
|
818
|
+
const timer = setTimeout(() => done(namedError('TimeoutError', `no image within ${ms} ms`)), ms);
|
|
819
|
+
video.addEventListener('loadeddata', onData);
|
|
820
|
+
video.addEventListener('error', onError);
|
|
821
|
+
video.addEventListener('emptied', onError);
|
|
822
|
+
});
|
|
823
|
+
//# sourceMappingURL=LocalPreview.js.map
|