@clockworkdog/cogs-client 3.0.0-alpha.4 → 3.0.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/AudioPlayer.js +5 -4
- package/dist/CogsConnection.js +37 -20
- package/dist/DataStore.js +11 -15
- package/dist/VideoPlayer.js +8 -4
- package/dist/browser/index.mjs +1874 -1592
- package/dist/browser/index.umd.js +13 -13
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/state-based/AudioManager.d.ts +15 -0
- package/dist/state-based/AudioManager.js +112 -0
- package/dist/state-based/ClipManager.d.ts +22 -0
- package/dist/state-based/ClipManager.js +55 -0
- package/dist/state-based/ImageManager.d.ts +9 -0
- package/dist/state-based/ImageManager.js +53 -0
- package/dist/state-based/SurfaceManager.d.ts +16 -0
- package/dist/state-based/SurfaceManager.js +80 -0
- package/dist/state-based/VideoManager.d.ts +17 -0
- package/dist/state-based/VideoManager.js +212 -0
- package/dist/utils/getStateAtTime.js +3 -3
- package/package.json +12 -4
package/dist/index.d.ts
CHANGED
|
@@ -7,6 +7,7 @@ export type { default as MediaObjectFit } from './types/MediaObjectFit';
|
|
|
7
7
|
export * as MediaSchema from './types/MediaSchema';
|
|
8
8
|
export { default as CogsAudioPlayer } from './AudioPlayer';
|
|
9
9
|
export { default as CogsVideoPlayer } from './VideoPlayer';
|
|
10
|
+
export { SurfaceManager } from './state-based/SurfaceManager';
|
|
10
11
|
export * from './types/AudioState';
|
|
11
12
|
export { assetUrl, preloadUrl } from './utils/urls';
|
|
12
13
|
export { getStateAtTime } from './utils/getStateAtTime';
|
package/dist/index.js
CHANGED
|
@@ -3,6 +3,7 @@ export * from './CogsConnection';
|
|
|
3
3
|
export * as MediaSchema from './types/MediaSchema';
|
|
4
4
|
export { default as CogsAudioPlayer } from './AudioPlayer';
|
|
5
5
|
export { default as CogsVideoPlayer } from './VideoPlayer';
|
|
6
|
+
export { SurfaceManager } from './state-based/SurfaceManager';
|
|
6
7
|
export * from './types/AudioState';
|
|
7
8
|
export { assetUrl, preloadUrl } from './utils/urls';
|
|
8
9
|
export { getStateAtTime } from './utils/getStateAtTime';
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { AudioState } from '../types/MediaSchema';
|
|
2
|
+
import { ClipManager } from './ClipManager';
|
|
3
|
+
export declare class AudioManager extends ClipManager<AudioState> {
|
|
4
|
+
private audioElement?;
|
|
5
|
+
private isSeeking;
|
|
6
|
+
constructor(surfaceElement: HTMLElement, clipElement: HTMLElement, state: AudioState);
|
|
7
|
+
private updateAudioElement;
|
|
8
|
+
/**
|
|
9
|
+
* Helper function to seek to a specified time.
|
|
10
|
+
* Works with the update loop to poll until seeked event has fired.
|
|
11
|
+
*/
|
|
12
|
+
private seekTo;
|
|
13
|
+
protected update(): void;
|
|
14
|
+
destroy(): void;
|
|
15
|
+
}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import { defaultAudioOptions } from '../types/MediaSchema';
|
|
2
|
+
import { getStateAtTime } from '../utils/getStateAtTime';
|
|
3
|
+
import { ClipManager } from './ClipManager';
|
|
4
|
+
const DEFAULT_AUDIO_POLLING = 1_000;
|
|
5
|
+
const TARGET_SYNC_THRESHOLD_MS = 10; // If we're closer than this we're good enough
|
|
6
|
+
const MAX_SYNC_THRESHOLD_MS = 1_000; // If we're further away than this, we'll seek instead
|
|
7
|
+
const SEEK_LOOKAHEAD_MS = 200; // We won't seek ahead instantly, so lets seek ahead
|
|
8
|
+
const MAX_PLAYBACK_RATE_ADJUSTMENT = 0.2;
|
|
9
|
+
// We smoothly ramp playbackRate up and down
|
|
10
|
+
const PLAYBACK_ADJUSTMENT_SMOOTHING = 0.5;
|
|
11
|
+
function playbackSmoothing(deltaTime) {
|
|
12
|
+
return Math.sign(deltaTime) * Math.pow(Math.abs(deltaTime) / MAX_SYNC_THRESHOLD_MS, PLAYBACK_ADJUSTMENT_SMOOTHING) * MAX_PLAYBACK_RATE_ADJUSTMENT;
|
|
13
|
+
}
|
|
14
|
+
export class AudioManager extends ClipManager {
|
|
15
|
+
audioElement;
|
|
16
|
+
isSeeking = false;
|
|
17
|
+
constructor(surfaceElement, clipElement, state) {
|
|
18
|
+
super(surfaceElement, clipElement, state);
|
|
19
|
+
this.clipElement = clipElement;
|
|
20
|
+
}
|
|
21
|
+
updateAudioElement() {
|
|
22
|
+
this.destroy();
|
|
23
|
+
this.audioElement = document.createElement('audio');
|
|
24
|
+
this.clipElement.replaceChildren(this.audioElement);
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Helper function to seek to a specified time.
|
|
28
|
+
* Works with the update loop to poll until seeked event has fired.
|
|
29
|
+
*/
|
|
30
|
+
seekTo(time) {
|
|
31
|
+
if (!this.audioElement)
|
|
32
|
+
return;
|
|
33
|
+
this.audioElement.addEventListener('seeked', () => {
|
|
34
|
+
this.isSeeking = false;
|
|
35
|
+
}, { once: true, passive: true });
|
|
36
|
+
this.audioElement.currentTime = time / 1_000;
|
|
37
|
+
}
|
|
38
|
+
update() {
|
|
39
|
+
// Update loop used to poll until seek finished
|
|
40
|
+
if (this.isSeeking)
|
|
41
|
+
return;
|
|
42
|
+
this.delay = DEFAULT_AUDIO_POLLING;
|
|
43
|
+
// Does the <audio /> element need adding/removing?
|
|
44
|
+
const currentState = getStateAtTime(this._state, Date.now());
|
|
45
|
+
if (currentState) {
|
|
46
|
+
if (!this.audioElement || !this.isConnected(this.audioElement)) {
|
|
47
|
+
this.updateAudioElement();
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
else {
|
|
51
|
+
this.destroy();
|
|
52
|
+
}
|
|
53
|
+
if (!currentState || !this.audioElement)
|
|
54
|
+
return;
|
|
55
|
+
const { t, rate, volume } = { ...defaultAudioOptions, ...currentState };
|
|
56
|
+
// this.audioElement.src will be a fully qualified URL
|
|
57
|
+
if (!this.audioElement.src.endsWith(this._state.file)) {
|
|
58
|
+
this.audioElement.src = this._state.file;
|
|
59
|
+
}
|
|
60
|
+
if (this.audioElement.volume !== volume) {
|
|
61
|
+
this.audioElement.volume = volume;
|
|
62
|
+
}
|
|
63
|
+
// Should the element be playing?
|
|
64
|
+
if (this.audioElement.paused && rate > 0) {
|
|
65
|
+
this.audioElement.play().catch(() => {
|
|
66
|
+
// Do nothing - this will be retried in the next loop
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
const currentTime = this.audioElement.currentTime * 1000;
|
|
70
|
+
const deltaTime = currentTime - t;
|
|
71
|
+
const deltaTimeAbs = Math.abs(deltaTime);
|
|
72
|
+
this.delay = 100;
|
|
73
|
+
switch (true) {
|
|
74
|
+
case deltaTimeAbs <= TARGET_SYNC_THRESHOLD_MS:
|
|
75
|
+
// We are on course:
|
|
76
|
+
// - The audio is within accepted latency of the server time
|
|
77
|
+
// - The playback rate is aligned with the server rate
|
|
78
|
+
if (this.audioElement.playbackRate !== rate) {
|
|
79
|
+
this.audioElement.playbackRate = rate;
|
|
80
|
+
}
|
|
81
|
+
break;
|
|
82
|
+
case rate > 0 && deltaTimeAbs > TARGET_SYNC_THRESHOLD_MS && deltaTimeAbs <= MAX_SYNC_THRESHOLD_MS: {
|
|
83
|
+
// We are close, we can smoothly adjust with playbackRate:
|
|
84
|
+
// - The audio must be playing
|
|
85
|
+
// - We must be close in time to the server time
|
|
86
|
+
const playbackRateAdjustment = playbackSmoothing(deltaTime);
|
|
87
|
+
const adjustedPlaybackRate = Math.max(0, rate - playbackRateAdjustment);
|
|
88
|
+
if (this.audioElement.playbackRate !== adjustedPlaybackRate) {
|
|
89
|
+
this.audioElement.playbackRate = adjustedPlaybackRate;
|
|
90
|
+
}
|
|
91
|
+
break;
|
|
92
|
+
}
|
|
93
|
+
default: {
|
|
94
|
+
// We cannot smoothly recover:
|
|
95
|
+
// - We seek just ahead of server time
|
|
96
|
+
if (this.audioElement.playbackRate !== rate) {
|
|
97
|
+
this.audioElement.playbackRate = rate;
|
|
98
|
+
}
|
|
99
|
+
// delay to poll until seeked
|
|
100
|
+
this.delay = 10;
|
|
101
|
+
this.seekTo(t + rate * (SEEK_LOOKAHEAD_MS / 1000));
|
|
102
|
+
break;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
destroy() {
|
|
107
|
+
if (this.audioElement) {
|
|
108
|
+
this.audioElement.src = '';
|
|
109
|
+
this.audioElement.remove();
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { MediaClipState } from '../types/MediaSchema';
|
|
2
|
+
/**
|
|
3
|
+
* Each instance of a ClipManager is responsible for displaying
|
|
4
|
+
* an image/audio/video clip in the correct state.
|
|
5
|
+
*/
|
|
6
|
+
export declare abstract class ClipManager<T extends MediaClipState> {
|
|
7
|
+
private surfaceElement;
|
|
8
|
+
protected clipElement: HTMLElement;
|
|
9
|
+
constructor(surfaceElement: HTMLElement, clipElement: HTMLElement, state: T);
|
|
10
|
+
/**
|
|
11
|
+
* This is the delay to be used in the update loop.
|
|
12
|
+
* It is intended to be dynamic for each loop.
|
|
13
|
+
*/
|
|
14
|
+
protected delay: number;
|
|
15
|
+
protected abstract update(): void;
|
|
16
|
+
abstract destroy(): void;
|
|
17
|
+
isConnected(element?: HTMLElement): boolean;
|
|
18
|
+
protected _state: T;
|
|
19
|
+
setState(newState: T): void;
|
|
20
|
+
private timeout;
|
|
21
|
+
private loop;
|
|
22
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
const DEFAULT_DELAY = 1_000;
|
|
2
|
+
/**
|
|
3
|
+
* Each instance of a ClipManager is responsible for displaying
|
|
4
|
+
* an image/audio/video clip in the correct state.
|
|
5
|
+
*/
|
|
6
|
+
export class ClipManager {
|
|
7
|
+
surfaceElement;
|
|
8
|
+
clipElement;
|
|
9
|
+
constructor(surfaceElement, clipElement, state) {
|
|
10
|
+
this.surfaceElement = surfaceElement;
|
|
11
|
+
this.clipElement = clipElement;
|
|
12
|
+
this._state = state;
|
|
13
|
+
// Allow the class to be constructed, then call the loop
|
|
14
|
+
setTimeout(this.loop);
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* This is the delay to be used in the update loop.
|
|
18
|
+
* It is intended to be dynamic for each loop.
|
|
19
|
+
*/
|
|
20
|
+
delay = DEFAULT_DELAY;
|
|
21
|
+
isConnected(element) {
|
|
22
|
+
if (!this.surfaceElement) {
|
|
23
|
+
return false;
|
|
24
|
+
}
|
|
25
|
+
if (!this.clipElement) {
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
if (!this.surfaceElement.contains(this.clipElement)) {
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
31
|
+
if (element) {
|
|
32
|
+
if (!this.clipElement.contains(element))
|
|
33
|
+
return false;
|
|
34
|
+
}
|
|
35
|
+
return true;
|
|
36
|
+
}
|
|
37
|
+
_state;
|
|
38
|
+
setState(newState) {
|
|
39
|
+
this._state = newState;
|
|
40
|
+
clearTimeout(this.timeout);
|
|
41
|
+
this.loop();
|
|
42
|
+
}
|
|
43
|
+
timeout;
|
|
44
|
+
loop = async () => {
|
|
45
|
+
if (this.isConnected()) {
|
|
46
|
+
this.update();
|
|
47
|
+
if (isFinite(this.delay)) {
|
|
48
|
+
this.timeout = setTimeout(this.loop, this.delay);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
else {
|
|
52
|
+
this.destroy();
|
|
53
|
+
}
|
|
54
|
+
};
|
|
55
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { ImageState } from '../types/MediaSchema';
|
|
2
|
+
import { ClipManager } from './ClipManager';
|
|
3
|
+
export declare class ImageManager extends ClipManager<ImageState> {
|
|
4
|
+
private imageElement?;
|
|
5
|
+
constructor(surfaceElement: HTMLElement, clipElement: HTMLElement, state: ImageState);
|
|
6
|
+
private updateImageElement;
|
|
7
|
+
protected update(): void;
|
|
8
|
+
destroy(): void;
|
|
9
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { defaultImageOptions } from '../types/MediaSchema';
|
|
2
|
+
import { getStateAtTime } from '../utils/getStateAtTime';
|
|
3
|
+
import { ClipManager } from './ClipManager';
|
|
4
|
+
export class ImageManager extends ClipManager {
|
|
5
|
+
imageElement;
|
|
6
|
+
constructor(surfaceElement, clipElement, state) {
|
|
7
|
+
super(surfaceElement, clipElement, state);
|
|
8
|
+
this.clipElement = clipElement;
|
|
9
|
+
}
|
|
10
|
+
updateImageElement() {
|
|
11
|
+
this.imageElement = document.createElement('img');
|
|
12
|
+
this.clipElement.replaceChildren(this.imageElement);
|
|
13
|
+
this.imageElement.style.position = 'absolute';
|
|
14
|
+
this.imageElement.style.height = '100%';
|
|
15
|
+
this.imageElement.style.widows = '100%';
|
|
16
|
+
}
|
|
17
|
+
update() {
|
|
18
|
+
const currentState = getStateAtTime(this._state, Date.now());
|
|
19
|
+
// Does the <img /> element need adding/removing?
|
|
20
|
+
if (currentState) {
|
|
21
|
+
if (!this.imageElement || !this.isConnected(this.imageElement)) {
|
|
22
|
+
this.updateImageElement();
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
else {
|
|
26
|
+
this.imageElement?.remove();
|
|
27
|
+
this.imageElement = undefined;
|
|
28
|
+
}
|
|
29
|
+
if (!this.imageElement || !currentState)
|
|
30
|
+
return;
|
|
31
|
+
// this.imageElement.src will be a fully qualified URL
|
|
32
|
+
if (!this.imageElement.src.startsWith(this._state.file)) {
|
|
33
|
+
this.imageElement.src = this._state.file;
|
|
34
|
+
}
|
|
35
|
+
if (this.imageElement.style.objectFit !== this._state.fit) {
|
|
36
|
+
this.imageElement.style.objectFit = this._state.fit;
|
|
37
|
+
}
|
|
38
|
+
if (parseFloat(this.imageElement.style.opacity) !== currentState.opacity) {
|
|
39
|
+
this.imageElement.style.opacity = String(currentState.opacity ?? defaultImageOptions.opacity);
|
|
40
|
+
}
|
|
41
|
+
const z = Math.round(currentState.zIndex ?? defaultImageOptions.zIndex);
|
|
42
|
+
if (parseInt(this.imageElement.style.zIndex) !== z) {
|
|
43
|
+
this.imageElement.style.zIndex = String(z);
|
|
44
|
+
}
|
|
45
|
+
const { opacity } = currentState;
|
|
46
|
+
if (typeof opacity === 'string' && opacity !== this.imageElement.style.opacity) {
|
|
47
|
+
this.imageElement.style.opacity = opacity;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
destroy() {
|
|
51
|
+
this.imageElement?.remove();
|
|
52
|
+
}
|
|
53
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { MediaSurfaceState } from '../types/MediaSchema';
|
|
2
|
+
export declare const DATA_CLIP_ID = "data-clip-id";
|
|
3
|
+
/**
|
|
4
|
+
* The SurfaceManager will receive state updates and:
|
|
5
|
+
* - Ensure that each clip has a parent element
|
|
6
|
+
* - Instantiate a ClipManager attached to each respective element
|
|
7
|
+
*/
|
|
8
|
+
export declare class SurfaceManager {
|
|
9
|
+
private _state;
|
|
10
|
+
setState(newState: MediaSurfaceState): void;
|
|
11
|
+
private _element;
|
|
12
|
+
get element(): HTMLDivElement;
|
|
13
|
+
private resources;
|
|
14
|
+
constructor(testState?: MediaSurfaceState);
|
|
15
|
+
update(): void;
|
|
16
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { ImageManager } from './ImageManager';
|
|
2
|
+
import { VideoManager } from './VideoManager';
|
|
3
|
+
import { AudioManager } from './AudioManager';
|
|
4
|
+
export const DATA_CLIP_ID = 'data-clip-id';
|
|
5
|
+
/**
|
|
6
|
+
* The SurfaceManager will receive state updates and:
|
|
7
|
+
* - Ensure that each clip has a parent element
|
|
8
|
+
* - Instantiate a ClipManager attached to each respective element
|
|
9
|
+
*/
|
|
10
|
+
export class SurfaceManager {
|
|
11
|
+
_state = {};
|
|
12
|
+
setState(newState) {
|
|
13
|
+
this._state = newState;
|
|
14
|
+
this.update();
|
|
15
|
+
}
|
|
16
|
+
_element;
|
|
17
|
+
get element() {
|
|
18
|
+
return this._element;
|
|
19
|
+
}
|
|
20
|
+
resources = {};
|
|
21
|
+
constructor(testState) {
|
|
22
|
+
this._element = document.createElement('div');
|
|
23
|
+
this._element.style.width = '100%';
|
|
24
|
+
this._element.style.height = '100%';
|
|
25
|
+
this._state = testState || {};
|
|
26
|
+
this.update();
|
|
27
|
+
}
|
|
28
|
+
update() {
|
|
29
|
+
// Destroy stale managers
|
|
30
|
+
Object.entries(this.resources).forEach(([clipId, { element, manager }]) => {
|
|
31
|
+
if (!(clipId in this._state)) {
|
|
32
|
+
delete this.resources[clipId];
|
|
33
|
+
element.remove();
|
|
34
|
+
manager?.destroy();
|
|
35
|
+
}
|
|
36
|
+
});
|
|
37
|
+
// Create and attach new wrapper elements
|
|
38
|
+
const elements = Object.keys(this._state)
|
|
39
|
+
.toSorted()
|
|
40
|
+
.map((clipId) => {
|
|
41
|
+
const resource = this.resources[clipId];
|
|
42
|
+
if (resource) {
|
|
43
|
+
return resource.element;
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
const element = document.createElement('div');
|
|
47
|
+
element.setAttribute(DATA_CLIP_ID, clipId);
|
|
48
|
+
this.resources[clipId] = { element };
|
|
49
|
+
return element;
|
|
50
|
+
}
|
|
51
|
+
});
|
|
52
|
+
this._element.replaceChildren(...elements);
|
|
53
|
+
// Create new managers
|
|
54
|
+
Object.keys(this._state)
|
|
55
|
+
.toSorted()
|
|
56
|
+
.forEach((clipId) => {
|
|
57
|
+
const clip = this._state[clipId];
|
|
58
|
+
const resource = this.resources[clipId];
|
|
59
|
+
if (!resource) {
|
|
60
|
+
throw new Error('Failed to create resource');
|
|
61
|
+
}
|
|
62
|
+
if (!resource.manager) {
|
|
63
|
+
switch (clip.type) {
|
|
64
|
+
case 'image':
|
|
65
|
+
resource.manager = new ImageManager(this._element, resource.element, clip);
|
|
66
|
+
break;
|
|
67
|
+
case 'audio':
|
|
68
|
+
resource.manager = new AudioManager(this._element, resource.element, clip);
|
|
69
|
+
break;
|
|
70
|
+
case 'video':
|
|
71
|
+
resource.manager = new VideoManager(this._element, resource.element, clip);
|
|
72
|
+
break;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
resource.manager.setState(clip);
|
|
77
|
+
}
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { VideoState } from '../types/MediaSchema';
|
|
2
|
+
import { ClipManager } from './ClipManager';
|
|
3
|
+
export declare class VideoManager extends ClipManager<VideoState> {
|
|
4
|
+
private videoElement?;
|
|
5
|
+
private isSeeking;
|
|
6
|
+
private timeToIntercept;
|
|
7
|
+
constructor(surfaceElement: HTMLElement, clipElement: HTMLElement, state: VideoState);
|
|
8
|
+
private updateVideoElement;
|
|
9
|
+
private get videoDuration();
|
|
10
|
+
/**
|
|
11
|
+
* Helper function to seek to a specified time.
|
|
12
|
+
* Works with the update loop to poll until seeked event has fired.
|
|
13
|
+
*/
|
|
14
|
+
private seekTo;
|
|
15
|
+
protected update(): void;
|
|
16
|
+
destroy(): void;
|
|
17
|
+
}
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
import { defaultVideoOptions } from '../types/MediaSchema';
|
|
2
|
+
import { getStateAtTime } from '../utils/getStateAtTime';
|
|
3
|
+
import { ClipManager } from './ClipManager';
|
|
4
|
+
const DEFAULT_VIDEO_POLLING_MS = 1_000;
|
|
5
|
+
const TARGET_SYNC_THRESHOLD_MS = 10; // If we're closer than this we're good enough
|
|
6
|
+
const MAX_SYNC_THRESHOLD_MS = 1_000; // If we're further away than this, we'll seek instead
|
|
7
|
+
const SEEK_LOOKAHEAD_MS = 200; // We won't seek ahead instantly, so lets seek ahead
|
|
8
|
+
const MAX_PLAYBACK_RATE_ADJUSTMENT = 0.15; // Don't speed up or slow down the video more than this
|
|
9
|
+
const INTERCEPTION_EARLY_CHECK_IN = 0.7; // When on course for interception of server time, how early to check in beforehand.
|
|
10
|
+
// We smoothly ramp playbackRate up and down
|
|
11
|
+
const PLAYBACK_ADJUSTMENT_SMOOTHING = 0.3;
|
|
12
|
+
function playbackSmoothing(deltaTime) {
|
|
13
|
+
return -Math.sign(deltaTime) * Math.pow(Math.abs(deltaTime) / MAX_SYNC_THRESHOLD_MS, PLAYBACK_ADJUSTMENT_SMOOTHING) * MAX_PLAYBACK_RATE_ADJUSTMENT;
|
|
14
|
+
}
|
|
15
|
+
// If we notice that at the end of the current playback, we set t=0 we should loop
|
|
16
|
+
const LOOPING_EPSILON_MS = 5;
|
|
17
|
+
function isLooping(state, time, duration) {
|
|
18
|
+
const currentState = getStateAtTime(state, time);
|
|
19
|
+
if (!currentState)
|
|
20
|
+
return false;
|
|
21
|
+
const { t, rate } = currentState;
|
|
22
|
+
if (t === undefined || rate === undefined)
|
|
23
|
+
return false;
|
|
24
|
+
const nextTemporalKeyframe = state.keyframes.filter(([t, kf]) => t > time && (kf?.set?.t !== undefined || kf?.set?.rate !== undefined))[0];
|
|
25
|
+
if (nextTemporalKeyframe?.[1]?.set?.t !== 0)
|
|
26
|
+
return false;
|
|
27
|
+
const timeRemaining = (duration - t) / rate;
|
|
28
|
+
const timeUntilKeyframe = nextTemporalKeyframe[0] - time;
|
|
29
|
+
return Math.abs(timeRemaining - timeUntilKeyframe) <= LOOPING_EPSILON_MS;
|
|
30
|
+
}
|
|
31
|
+
export class VideoManager extends ClipManager {
|
|
32
|
+
videoElement;
|
|
33
|
+
// We seek to another part of the video and do nothing until we get there
|
|
34
|
+
isSeeking = false;
|
|
35
|
+
// We change playbackRate to intercept the server time of the video and don't change course until we intercept
|
|
36
|
+
timeToIntercept = undefined;
|
|
37
|
+
constructor(surfaceElement, clipElement, state) {
|
|
38
|
+
super(surfaceElement, clipElement, state);
|
|
39
|
+
this.clipElement = clipElement;
|
|
40
|
+
}
|
|
41
|
+
updateVideoElement() {
|
|
42
|
+
this.destroy();
|
|
43
|
+
this.videoElement = document.createElement('video');
|
|
44
|
+
this.clipElement.replaceChildren(this.videoElement);
|
|
45
|
+
this.videoElement.style.position = 'absolute';
|
|
46
|
+
this.videoElement.style.width = '100%';
|
|
47
|
+
this.videoElement.style.height = '100%';
|
|
48
|
+
}
|
|
49
|
+
get videoDuration() {
|
|
50
|
+
if (!this.videoElement)
|
|
51
|
+
return undefined;
|
|
52
|
+
if (this.videoElement.readyState < HTMLMediaElement.HAVE_METADATA)
|
|
53
|
+
return undefined;
|
|
54
|
+
return this.videoElement.duration * 1000;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Helper function to seek to a specified time.
|
|
58
|
+
* Works with the update loop to poll until seeked event has fired.
|
|
59
|
+
*/
|
|
60
|
+
seekTo(time) {
|
|
61
|
+
if (!this.videoElement)
|
|
62
|
+
return;
|
|
63
|
+
this.videoElement.addEventListener('seeked', () => {
|
|
64
|
+
console.debug('seeked');
|
|
65
|
+
this.isSeeking = false;
|
|
66
|
+
}, { once: true, passive: true });
|
|
67
|
+
this.videoElement.currentTime = time / 1_000;
|
|
68
|
+
}
|
|
69
|
+
update() {
|
|
70
|
+
// Update loop used to poll until seek finished
|
|
71
|
+
if (this.isSeeking)
|
|
72
|
+
return;
|
|
73
|
+
// Does the <video /> element need adding/removing?
|
|
74
|
+
const now = Date.now();
|
|
75
|
+
const currentState = getStateAtTime(this._state, now);
|
|
76
|
+
if (currentState) {
|
|
77
|
+
if (!this.videoElement || !this.isConnected(this.videoElement)) {
|
|
78
|
+
this.updateVideoElement();
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
this.videoElement?.remove();
|
|
83
|
+
this.videoElement = undefined;
|
|
84
|
+
}
|
|
85
|
+
if (!currentState || !this.videoElement)
|
|
86
|
+
return;
|
|
87
|
+
const { t, rate, volume } = { ...defaultVideoOptions, ...currentState };
|
|
88
|
+
// this.videoElement.src will be a fully qualified URL
|
|
89
|
+
if (!this.videoElement.src.endsWith(this._state.file)) {
|
|
90
|
+
this.videoElement.src = this._state.file;
|
|
91
|
+
}
|
|
92
|
+
if (this.videoElement.style.objectFit !== this._state.fit) {
|
|
93
|
+
this.videoElement.style.objectFit = this._state.fit;
|
|
94
|
+
}
|
|
95
|
+
if (parseFloat(this.videoElement.style.opacity) !== currentState.opacity) {
|
|
96
|
+
this.videoElement.style.opacity = String(currentState.opacity ?? defaultVideoOptions.opacity);
|
|
97
|
+
}
|
|
98
|
+
const z = Math.round(currentState.zIndex ?? defaultVideoOptions.zIndex);
|
|
99
|
+
if (parseInt(this.videoElement.style.zIndex) !== z) {
|
|
100
|
+
this.videoElement.style.zIndex = String(z);
|
|
101
|
+
}
|
|
102
|
+
if (this.videoElement.volume !== volume) {
|
|
103
|
+
this.videoElement.volume = volume;
|
|
104
|
+
}
|
|
105
|
+
const duration = this.videoDuration;
|
|
106
|
+
if (duration !== undefined) {
|
|
107
|
+
// Is the video looping?
|
|
108
|
+
if (isLooping(this._state, now, duration)) {
|
|
109
|
+
if (!this.videoElement.loop) {
|
|
110
|
+
console.debug('starting loop');
|
|
111
|
+
this.videoElement.loop = true;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
else {
|
|
115
|
+
if (this.videoElement.loop) {
|
|
116
|
+
console.debug('stopping loop');
|
|
117
|
+
this.videoElement.loop = false;
|
|
118
|
+
}
|
|
119
|
+
// Has the video finished
|
|
120
|
+
if (t > duration) {
|
|
121
|
+
console.debug('ended');
|
|
122
|
+
this.delay = Infinity;
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
// Should the video be playing
|
|
128
|
+
if (this.videoElement.paused && rate > 0) {
|
|
129
|
+
if (duration === undefined || duration > t) {
|
|
130
|
+
this.videoElement.play().catch(() => {
|
|
131
|
+
// Do nothing - this will be retried in the next loop
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
const currentTime = this.videoElement.currentTime * 1000;
|
|
136
|
+
const deltaTime = currentTime - t;
|
|
137
|
+
const deltaTimeAbs = Math.abs(deltaTime);
|
|
138
|
+
// Handle current playbackRateAdjustment
|
|
139
|
+
if (this.timeToIntercept !== undefined) {
|
|
140
|
+
if (deltaTimeAbs <= TARGET_SYNC_THRESHOLD_MS) {
|
|
141
|
+
// We've successfully got back on track
|
|
142
|
+
console.log('intercepted', `${deltaTime.toFixed(0)}ms`);
|
|
143
|
+
this.timeToIntercept = undefined;
|
|
144
|
+
}
|
|
145
|
+
else {
|
|
146
|
+
const newTimeToIntercept = deltaTime / (rate - this.videoElement.playbackRate);
|
|
147
|
+
if (newTimeToIntercept < this.timeToIntercept && newTimeToIntercept > 0) {
|
|
148
|
+
// We're getting there, let's stay on course
|
|
149
|
+
console.debug(`intercepting ${newTimeToIntercept.toFixed(0)}ms`, `${deltaTime.toFixed(0)}ms`);
|
|
150
|
+
this.timeToIntercept = newTimeToIntercept;
|
|
151
|
+
}
|
|
152
|
+
else {
|
|
153
|
+
// We've gone too far
|
|
154
|
+
console.debug('missed intercept', deltaTime, this.timeToIntercept, newTimeToIntercept);
|
|
155
|
+
this.timeToIntercept = undefined;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
switch (true) {
|
|
160
|
+
case deltaTimeAbs <= TARGET_SYNC_THRESHOLD_MS: {
|
|
161
|
+
// We are on course:
|
|
162
|
+
// - The video is within accepted latency of the server time
|
|
163
|
+
// - The playback rate is aligned with the server rate
|
|
164
|
+
console.debug(`${rate}x`, deltaTime.toFixed(0));
|
|
165
|
+
this.timeToIntercept = undefined;
|
|
166
|
+
if (this.videoElement.playbackRate !== rate) {
|
|
167
|
+
this.videoElement.playbackRate = rate;
|
|
168
|
+
}
|
|
169
|
+
this.delay = DEFAULT_VIDEO_POLLING_MS;
|
|
170
|
+
break;
|
|
171
|
+
}
|
|
172
|
+
case this.timeToIntercept !== undefined:
|
|
173
|
+
// We are currently on course to intercept
|
|
174
|
+
// - We don't want to adjust the playbackRate excessively to pop audio
|
|
175
|
+
// - We are on track to get back on time. So we can wait.
|
|
176
|
+
this.delay = this.timeToIntercept * INTERCEPTION_EARLY_CHECK_IN;
|
|
177
|
+
break;
|
|
178
|
+
case rate > 0 && deltaTimeAbs > TARGET_SYNC_THRESHOLD_MS && deltaTimeAbs <= MAX_SYNC_THRESHOLD_MS && this.timeToIntercept === undefined: {
|
|
179
|
+
// We are close, we can smoothly adjust with playbackRate:
|
|
180
|
+
// - The video must be playing
|
|
181
|
+
// - We must be close in time to the server time
|
|
182
|
+
const playbackRateAdjustment = playbackSmoothing(deltaTime);
|
|
183
|
+
const adjustedPlaybackRate = Math.max(0, rate + playbackRateAdjustment);
|
|
184
|
+
this.timeToIntercept = deltaTime / (rate - adjustedPlaybackRate);
|
|
185
|
+
console.debug(`${adjustedPlaybackRate.toFixed(2)}x`, `${deltaTime.toFixed(0)}ms`);
|
|
186
|
+
if (this.videoElement.playbackRate !== adjustedPlaybackRate) {
|
|
187
|
+
this.videoElement.playbackRate = adjustedPlaybackRate;
|
|
188
|
+
}
|
|
189
|
+
this.delay = this.timeToIntercept * INTERCEPTION_EARLY_CHECK_IN;
|
|
190
|
+
break;
|
|
191
|
+
}
|
|
192
|
+
default: {
|
|
193
|
+
// We cannot smoothly recover:
|
|
194
|
+
// - We seek just ahead of server time
|
|
195
|
+
if (this.videoElement.playbackRate !== rate) {
|
|
196
|
+
this.videoElement.playbackRate = rate;
|
|
197
|
+
}
|
|
198
|
+
// delay to poll until seeked
|
|
199
|
+
console.debug('seeking');
|
|
200
|
+
this.delay = 10;
|
|
201
|
+
this.seekTo(t + rate * SEEK_LOOKAHEAD_MS);
|
|
202
|
+
break;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
destroy() {
|
|
207
|
+
if (this.videoElement) {
|
|
208
|
+
this.videoElement.src = '';
|
|
209
|
+
this.videoElement.remove();
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
}
|
|
@@ -47,11 +47,11 @@ export function getPropertiesAtTime(keyframes, time) {
|
|
|
47
47
|
// If lerp and set are both present, we assume we lerp up until the timestamp,
|
|
48
48
|
// then set to a new value
|
|
49
49
|
Object.entries(properties.lerp ?? {}).forEach(([property, value]) => {
|
|
50
|
-
propertyKeyframes[property]
|
|
50
|
+
propertyKeyframes[property] ??= {};
|
|
51
51
|
propertyKeyframes[property].before = [timestamp, value];
|
|
52
52
|
});
|
|
53
53
|
Object.entries(properties.set ?? {}).forEach(([property, value]) => {
|
|
54
|
-
propertyKeyframes[property]
|
|
54
|
+
propertyKeyframes[property] ??= {};
|
|
55
55
|
propertyKeyframes[property].before = [timestamp, value];
|
|
56
56
|
});
|
|
57
57
|
}
|
|
@@ -59,7 +59,7 @@ export function getPropertiesAtTime(keyframes, time) {
|
|
|
59
59
|
// We're trying to find the closest timestamp afterwards for lerping
|
|
60
60
|
// So only set if not yet set
|
|
61
61
|
Object.entries(properties.lerp ?? {}).forEach(([property, value]) => {
|
|
62
|
-
propertyKeyframes[property]
|
|
62
|
+
propertyKeyframes[property] ??= {};
|
|
63
63
|
if (propertyKeyframes[property].after === undefined) {
|
|
64
64
|
propertyKeyframes[property].after = [timestamp, value];
|
|
65
65
|
}
|