@clockworkdog/cogs-client 3.0.0-alpha.5 → 3.0.0-alpha.7

package/dist/index.d.ts

@@ -5,6 +5,7 @@ export type { default as MediaClipStateMessage } from './types/MediaClipStateMes
 export type { default as ShowPhase } from './types/ShowPhase';
 export type { default as MediaObjectFit } from './types/MediaObjectFit';
 export * as MediaSchema from './types/MediaSchema';
+export * from './state-based/SurfaceManager';
 export { default as CogsAudioPlayer } from './AudioPlayer';
 export { default as CogsVideoPlayer } from './VideoPlayer';
 export { SurfaceManager } from './state-based/SurfaceManager';

package/dist/index.js

@@ -1,6 +1,7 @@
 export { default as CogsConnection } from './CogsConnection';
 export * from './CogsConnection';
 export * as MediaSchema from './types/MediaSchema';
+export * from './state-based/SurfaceManager';
 export { default as CogsAudioPlayer } from './AudioPlayer';
 export { default as CogsVideoPlayer } from './VideoPlayer';
 export { SurfaceManager } from './state-based/SurfaceManager';

package/dist/state-based/AudioManager.d.ts

@@ -3,7 +3,7 @@ import { ClipManager } from './ClipManager';
 export declare class AudioManager extends ClipManager<AudioState> {
     private audioElement?;
     private isSeeking;
-    constructor(surfaceElement: HTMLElement, clipElement: HTMLElement, state: AudioState);
+    constructor(surfaceElement: HTMLElement, clipElement: HTMLElement, state: AudioState, constructAssetURL: (file: string) => string);
     private updateAudioElement;
     /**
      * Helper function to seek to a specified time.

package/dist/state-based/AudioManager.js

@@ -14,8 +14,8 @@ function playbackSmoothing(deltaTime) {
 export class AudioManager extends ClipManager {
     audioElement;
     isSeeking = false;
-    constructor(surfaceElement, clipElement, state) {
-        super(surfaceElement, clipElement, state);
+    constructor(surfaceElement, clipElement, state, constructAssetURL) {
+        super(surfaceElement, clipElement, state, constructAssetURL);
         this.clipElement = clipElement;
     }
     updateAudioElement() {
@@ -53,9 +53,10 @@ export class AudioManager extends ClipManager {
         if (!currentState || !this.audioElement)
             return;
         const { t, rate, volume } = { ...defaultAudioOptions, ...currentState };
-        // this.audioElement.src will be a fully qualified URL
-        if (!this.audioElement.src.endsWith(this._state.file)) {
-            this.audioElement.src = this._state.file;
+        // this.videoElement.src will be a fully qualified URL
+        const assetURL = this.constructAssetURL(this._state.file);
+        if (!this.audioElement.src.includes(assetURL)) {
+            this.audioElement.src = assetURL;
         }
         if (this.audioElement.volume !== volume) {
             this.audioElement.volume = volume;

package/dist/state-based/ClipManager.d.ts

@@ -6,7 +6,8 @@ import { MediaClipState } from '../types/MediaSchema';
 export declare abstract class ClipManager<T extends MediaClipState> {
     private surfaceElement;
     protected clipElement: HTMLElement;
-    constructor(surfaceElement: HTMLElement, clipElement: HTMLElement, state: T);
+    protected constructAssetURL: (file: string) => string;
+    constructor(surfaceElement: HTMLElement, clipElement: HTMLElement, state: T, constructAssetURL: (file: string) => string);
     /**
      * This is the delay to be used in the update loop.
      * It is intended to be dynamic for each loop.

package/dist/state-based/ClipManager.js

@@ -6,9 +6,11 @@ const DEFAULT_DELAY = 1_000;
 export class ClipManager {
     surfaceElement;
     clipElement;
-    constructor(surfaceElement, clipElement, state) {
+    constructAssetURL;
+    constructor(surfaceElement, clipElement, state, constructAssetURL) {
         this.surfaceElement = surfaceElement;
         this.clipElement = clipElement;
+        this.constructAssetURL = constructAssetURL;
         this._state = state;
         // Allow the class to be constructed, then call the loop
         setTimeout(this.loop);
@@ -44,7 +46,9 @@ export class ClipManager {
     loop = async () => {
         if (this.isConnected()) {
             this.update();
-            this.timeout = setTimeout(this.loop, this.delay);
+            if (isFinite(this.delay)) {
+                this.timeout = setTimeout(this.loop, this.delay);
+            }
         }
         else {
             this.destroy();

package/dist/state-based/ImageManager.d.ts

@@ -2,7 +2,7 @@ import { ImageState } from '../types/MediaSchema';
 import { ClipManager } from './ClipManager';
 export declare class ImageManager extends ClipManager<ImageState> {
     private imageElement?;
-    constructor(surfaceElement: HTMLElement, clipElement: HTMLElement, state: ImageState);
+    constructor(surfaceElement: HTMLElement, clipElement: HTMLElement, state: ImageState, constructAssetURL: (file: string) => string);
     private updateImageElement;
     protected update(): void;
     destroy(): void;

package/dist/state-based/ImageManager.js

@@ -3,8 +3,8 @@ import { getStateAtTime } from '../utils/getStateAtTime';
 import { ClipManager } from './ClipManager';
 export class ImageManager extends ClipManager {
     imageElement;
-    constructor(surfaceElement, clipElement, state) {
-        super(surfaceElement, clipElement, state);
+    constructor(surfaceElement, clipElement, state, constructAssetURL) {
+        super(surfaceElement, clipElement, state, constructAssetURL);
         this.clipElement = clipElement;
     }
     updateImageElement() {
@@ -28,9 +28,10 @@ export class ImageManager extends ClipManager {
         }
         if (!this.imageElement || !currentState)
             return;
-        // this.imageElement.src will be a fully qualified URL
-        if (!this.imageElement.src.startsWith(this._state.file)) {
-            this.imageElement.src = this._state.file;
+        // this.videoElement.src will be a fully qualified URL
+        const assetURL = this.constructAssetURL(this._state.file);
+        if (!this.imageElement.src.includes(assetURL)) {
+            this.imageElement.src = assetURL;
         }
         if (this.imageElement.style.objectFit !== this._state.fit) {
             this.imageElement.style.objectFit = this._state.fit;

package/dist/state-based/SurfaceManager.d.ts

@@ -6,11 +6,12 @@ export declare const DATA_CLIP_ID = "data-clip-id";
  * - Instantiate a ClipManager attached to each respective element
  */
 export declare class SurfaceManager {
+    private constructAssetUrl;
     private _state;
     setState(newState: MediaSurfaceState): void;
     private _element;
     get element(): HTMLDivElement;
     private resources;
-    constructor(testState?: MediaSurfaceState);
+    constructor(constructAssetUrl: (file: string) => string, testState?: MediaSurfaceState);
     update(): void;
 }

package/dist/state-based/SurfaceManager.js

@@ -8,6 +8,7 @@ export const DATA_CLIP_ID = 'data-clip-id';
  * - Instantiate a ClipManager attached to each respective element
  */
 export class SurfaceManager {
+    constructAssetUrl;
     _state = {};
     setState(newState) {
         this._state = newState;
@@ -18,8 +19,10 @@ export class SurfaceManager {
         return this._element;
     }
     resources = {};
-    constructor(testState) {
+    constructor(constructAssetUrl, testState) {
+        this.constructAssetUrl = constructAssetUrl;
         this._element = document.createElement('div');
+        this._element.className = 'surface-manager';
         this._element.style.width = '100%';
         this._element.style.height = '100%';
         this._state = testState || {};
@@ -62,13 +65,13 @@ export class SurfaceManager {
             if (!resource.manager) {
                 switch (clip.type) {
                     case 'image':
-                        resource.manager = new ImageManager(this._element, resource.element, clip);
+                        resource.manager = new ImageManager(this._element, resource.element, clip, this.constructAssetUrl);
                         break;
                     case 'audio':
-                        resource.manager = new AudioManager(this._element, resource.element, clip);
+                        resource.manager = new AudioManager(this._element, resource.element, clip, this.constructAssetUrl);
                         break;
                     case 'video':
-                        resource.manager = new VideoManager(this._element, resource.element, clip);
+                        resource.manager = new VideoManager(this._element, resource.element, clip, this.constructAssetUrl);
                         break;
                 }
             }

package/dist/state-based/VideoManager.d.ts

@@ -3,8 +3,10 @@ import { ClipManager } from './ClipManager';
 export declare class VideoManager extends ClipManager<VideoState> {
     private videoElement?;
     private isSeeking;
-    constructor(surfaceElement: HTMLElement, clipElement: HTMLElement, state: VideoState);
+    private timeToIntercept;
+    constructor(surfaceElement: HTMLElement, clipElement: HTMLElement, state: VideoState, constructAssetURL: (file: string) => string);
     private updateVideoElement;
+    private get videoDuration();
     /**
      * Helper function to seek to a specified time.
      * Works with the update loop to poll until seeked event has fired.

package/dist/state-based/VideoManager.js

@@ -1,21 +1,41 @@
 import { defaultVideoOptions } from '../types/MediaSchema';
 import { getStateAtTime } from '../utils/getStateAtTime';
 import { ClipManager } from './ClipManager';
-const DEFAULT_VIDEO_POLLING = 1_000;
+const DEFAULT_VIDEO_POLLING_MS = 1_000;
 const TARGET_SYNC_THRESHOLD_MS = 10; // If we're closer than this we're good enough
 const MAX_SYNC_THRESHOLD_MS = 1_000; // If we're further away than this, we'll seek instead
 const SEEK_LOOKAHEAD_MS = 200; // We won't seek ahead instantly, so lets seek ahead
-const MAX_PLAYBACK_RATE_ADJUSTMENT = 0.5;
+const MAX_PLAYBACK_RATE_ADJUSTMENT = 0.15; // Don't speed up or slow down the video more than this
+const INTERCEPTION_EARLY_CHECK_IN = 0.7; // When on course for interception of server time, how early to check in beforehand.
 // We smoothly ramp playbackRate up and down
 const PLAYBACK_ADJUSTMENT_SMOOTHING = 0.3;
 function playbackSmoothing(deltaTime) {
-    return Math.sign(deltaTime) * Math.pow(Math.abs(deltaTime) / MAX_SYNC_THRESHOLD_MS, PLAYBACK_ADJUSTMENT_SMOOTHING) * MAX_PLAYBACK_RATE_ADJUSTMENT;
+    return -Math.sign(deltaTime) * Math.pow(Math.abs(deltaTime) / MAX_SYNC_THRESHOLD_MS, PLAYBACK_ADJUSTMENT_SMOOTHING) * MAX_PLAYBACK_RATE_ADJUSTMENT;
+}
+// If we notice that at the end of the current playback, we set t=0 we should loop
+const LOOPING_EPSILON_MS = 5;
+function isLooping(state, time, duration) {
+    const currentState = getStateAtTime(state, time);
+    if (!currentState)
+        return false;
+    const { t, rate } = currentState;
+    if (t === undefined || rate === undefined)
+        return false;
+    const nextTemporalKeyframe = state.keyframes.filter(([t, kf]) => t > time && (kf?.set?.t !== undefined || kf?.set?.rate !== undefined))[0];
+    if (nextTemporalKeyframe?.[1]?.set?.t !== 0)
+        return false;
+    const timeRemaining = (duration - t) / rate;
+    const timeUntilKeyframe = nextTemporalKeyframe[0] - time;
+    return Math.abs(timeRemaining - timeUntilKeyframe) <= LOOPING_EPSILON_MS;
 }
 export class VideoManager extends ClipManager {
     videoElement;
+    // We seek to another part of the video and do nothing until we get there
     isSeeking = false;
-    constructor(surfaceElement, clipElement, state) {
-        super(surfaceElement, clipElement, state);
+    // We change playbackRate to intercept the server time of the video and don't change course until we intercept
+    timeToIntercept = undefined;
+    constructor(surfaceElement, clipElement, state, constructAssetURL) {
+        super(surfaceElement, clipElement, state, constructAssetURL);
         this.clipElement = clipElement;
     }
     updateVideoElement() {
@@ -26,6 +46,13 @@ export class VideoManager extends ClipManager {
         this.videoElement.style.width = '100%';
         this.videoElement.style.height = '100%';
     }
+    get videoDuration() {
+        if (!this.videoElement)
+            return undefined;
+        if (this.videoElement.readyState < HTMLMediaElement.HAVE_METADATA)
+            return undefined;
+        return this.videoElement.duration * 1000;
+    }
     /**
      * Helper function to seek to a specified time.
      * Works with the update loop to poll until seeked event has fired.
@@ -34,6 +61,7 @@ export class VideoManager extends ClipManager {
         if (!this.videoElement)
             return;
         this.videoElement.addEventListener('seeked', () => {
+            console.debug('seeked');
             this.isSeeking = false;
         }, { once: true, passive: true });
         this.videoElement.currentTime = time / 1_000;
@@ -42,9 +70,9 @@ export class VideoManager extends ClipManager {
         // Update loop used to poll until seek finished
         if (this.isSeeking)
             return;
-        this.delay = DEFAULT_VIDEO_POLLING;
         // Does the <video /> element need adding/removing?
-        const currentState = getStateAtTime(this._state, Date.now());
+        const now = Date.now();
+        const currentState = getStateAtTime(this._state, now);
         if (currentState) {
             if (!this.videoElement || !this.isConnected(this.videoElement)) {
                 this.updateVideoElement();
@@ -58,8 +86,9 @@ export class VideoManager extends ClipManager {
             return;
         const { t, rate, volume } = { ...defaultVideoOptions, ...currentState };
         // this.videoElement.src will be a fully qualified URL
-        if (!this.videoElement.src.endsWith(this._state.file)) {
-            this.videoElement.src = this._state.file;
+        const assetURL = this.constructAssetURL(this._state.file);
+        if (!this.videoElement.src.includes(assetURL)) {
+            this.videoElement.src = assetURL;
         }
         if (this.videoElement.style.objectFit !== this._state.fit) {
             this.videoElement.style.objectFit = this._state.fit;
@@ -74,34 +103,91 @@ export class VideoManager extends ClipManager {
         if (this.videoElement.volume !== volume) {
             this.videoElement.volume = volume;
         }
-        // Should the element be playing?
+        const duration = this.videoDuration;
+        if (duration !== undefined) {
+            // Is the video looping?
+            if (isLooping(this._state, now, duration)) {
+                if (!this.videoElement.loop) {
+                    console.debug('starting loop');
+                    this.videoElement.loop = true;
+                }
+            }
+            else {
+                if (this.videoElement.loop) {
+                    console.debug('stopping loop');
+                    this.videoElement.loop = false;
+                }
+                // Has the video finished
+                if (t > duration) {
+                    console.debug('ended');
+                    this.delay = Infinity;
+                    return;
+                }
+            }
+        }
+        // Should the video be playing
         if (this.videoElement.paused && rate > 0) {
-            this.videoElement.play().catch(() => {
-                // Do nothing - this will be retried in the next loop
-            });
+            if (duration === undefined || duration > t) {
+                this.videoElement.play().catch(() => {
+                    // Do nothing - this will be retried in the next loop
+                });
+            }
         }
         const currentTime = this.videoElement.currentTime * 1000;
         const deltaTime = currentTime - t;
         const deltaTimeAbs = Math.abs(deltaTime);
-        this.delay = 100;
+        // Handle current playbackRateAdjustment
+        if (this.timeToIntercept !== undefined) {
+            if (deltaTimeAbs <= TARGET_SYNC_THRESHOLD_MS) {
+                // We've successfully got back on track
+                console.log('intercepted', `${deltaTime.toFixed(0)}ms`);
+                this.timeToIntercept = undefined;
+            }
+            else {
+                const newTimeToIntercept = deltaTime / (rate - this.videoElement.playbackRate);
+                if (newTimeToIntercept < this.timeToIntercept && newTimeToIntercept > 0) {
+                    // We're getting there, let's stay on course
+                    console.debug(`intercepting ${newTimeToIntercept.toFixed(0)}ms`, `${deltaTime.toFixed(0)}ms`);
+                    this.timeToIntercept = newTimeToIntercept;
+                }
+                else {
+                    // We've gone too far
+                    console.debug('missed intercept', deltaTime, this.timeToIntercept, newTimeToIntercept);
+                    this.timeToIntercept = undefined;
+                }
+            }
+        }
         switch (true) {
-            case deltaTimeAbs <= TARGET_SYNC_THRESHOLD_MS:
+            case deltaTimeAbs <= TARGET_SYNC_THRESHOLD_MS: {
                 // We are on course:
                 //   - The video is within accepted latency of the server time
                 //   - The playback rate is aligned with the server rate
+                console.debug(`${rate}x`, deltaTime.toFixed(0));
+                this.timeToIntercept = undefined;
                 if (this.videoElement.playbackRate !== rate) {
                     this.videoElement.playbackRate = rate;
                 }
+                this.delay = DEFAULT_VIDEO_POLLING_MS;
+                break;
+            }
+            case this.timeToIntercept !== undefined:
+                // We are currently on course to intercept
+                //   - We don't want to adjust the playbackRate excessively to pop audio
+                //   - We are on track to get back on time.  So we can wait.
+                this.delay = this.timeToIntercept * INTERCEPTION_EARLY_CHECK_IN;
                 break;
-            case rate > 0 && deltaTimeAbs > TARGET_SYNC_THRESHOLD_MS && deltaTimeAbs <= MAX_SYNC_THRESHOLD_MS: {
+            case rate > 0 && deltaTimeAbs > TARGET_SYNC_THRESHOLD_MS && deltaTimeAbs <= MAX_SYNC_THRESHOLD_MS && this.timeToIntercept === undefined: {
                 // We are close, we can smoothly adjust with playbackRate:
                 //  - The video must be playing
                 //  - We must be close in time to the server time
                 const playbackRateAdjustment = playbackSmoothing(deltaTime);
-                const adjustedPlaybackRate = Math.max(0, rate - playbackRateAdjustment);
+                const adjustedPlaybackRate = Math.max(0, rate + playbackRateAdjustment);
+                this.timeToIntercept = deltaTime / (rate - adjustedPlaybackRate);
+                console.debug(`${adjustedPlaybackRate.toFixed(2)}x`, `${deltaTime.toFixed(0)}ms`);
                 if (this.videoElement.playbackRate !== adjustedPlaybackRate) {
                     this.videoElement.playbackRate = adjustedPlaybackRate;
                 }
+                this.delay = this.timeToIntercept * INTERCEPTION_EARLY_CHECK_IN;
                 break;
             }
             default: {
@@ -111,8 +197,9 @@ export class VideoManager extends ClipManager {
                     this.videoElement.playbackRate = rate;
                 }
                 // delay to poll until seeked
+                console.debug('seeking');
                 this.delay = 10;
-                this.seekTo(t + rate * (SEEK_LOOKAHEAD_MS / 1000));
+                this.seekTo(t + rate * SEEK_LOOKAHEAD_MS);
                 break;
             }
         }

package/package.json

@@ -3,7 +3,7 @@
   "description": "Connect to COGS to build a custom Media Master",
   "author": "Clockwork Dog <info@clockwork.dog>",
   "homepage": "https://github.com/clockwork-dog/cogs-sdk/tree/main/packages/javascript",
-  "version": "3.0.0-alpha.5",
+  "version": "3.0.0-alpha.7",
   "keywords": [],
   "license": "MIT",
   "repository": {
@@ -37,7 +37,7 @@
     "cy:generate": "cypress run --e2e"
   },
   "dependencies": {
-    "@clockworkdog/timesync": "^3.0.0-alpha.5",
+    "@clockworkdog/timesync": "^3.0.0-alpha.7",
     "howler": "clockwork-dog/howler.js#fix-looping-clips",
     "reconnecting-websocket": "^4.4.0",
     "zod": "^4.1.13"