@clockworkdog/cogs-client 3.0.0-alpha.11 → 3.0.0-alpha.14

package/dist/index.d.ts

@@ -8,6 +8,7 @@ export * as MediaSchema from './types/MediaSchema';
 export { default as CogsAudioPlayer } from './AudioPlayer';
 export { default as CogsVideoPlayer } from './VideoPlayer';
 export { SurfaceManager } from './state-based/SurfaceManager';
+export { MediaPreloader } from './state-based/MediaPreloader';
 export * from './types/AudioState';
 export { assetUrl, preloadUrl } from './utils/urls';
 export { getStateAtTime } from './utils/getStateAtTime';

package/dist/index.js

@@ -4,6 +4,7 @@ export * as MediaSchema from './types/MediaSchema';
 export { default as CogsAudioPlayer } from './AudioPlayer';
 export { default as CogsVideoPlayer } from './VideoPlayer';
 export { SurfaceManager } from './state-based/SurfaceManager';
+export { MediaPreloader } from './state-based/MediaPreloader';
 export * from './types/AudioState';
 export { assetUrl, preloadUrl } from './utils/urls';
 export { getStateAtTime } from './utils/getStateAtTime';

package/dist/state-based/MediaClipManager.d.ts

@@ -8,23 +8,43 @@ export declare abstract class MediaClipManager<T extends MediaClipState> {
     private surfaceElement;
     protected clipElement: HTMLElement;
     protected constructAssetURL: (file: string) => string;
+    protected getAudioOutput: (outputLabel: string) => string;
     protected mediaPreloader: MediaPreloader;
-    constructor(surfaceElement: HTMLElement, clipElement: HTMLElement, state: T, constructAssetURL: (file: string) => string, mediaPreloader: MediaPreloader);
+    constructor(surfaceElement: HTMLElement, clipElement: HTMLElement, state: T, constructAssetURL: (file: string) => string, getAudioOutput: (outputLabel: string) => string, mediaPreloader: MediaPreloader);
     protected abstract update(): void;
     abstract destroy(): void;
     isConnected(element?: HTMLElement): boolean;
     protected _state: T;
     setState(newState: T): void;
     private timeout;
-    private loop;
+    loop: () => Promise<void>;
 }
-export declare function assertElement(mediaElement: HTMLElement | undefined, parentElement: HTMLElement, clip: MediaClipState, constructAssetURL: (file: string) => string, preloader: MediaPreloader): HTMLElement;
+/**
+ * Makes sure that the child media element exists and is of the correct type
+ * - If it isn't or doesn't exist we'll get a new one
+ * - If it is audio or video we'll try and get a preloaded media element
+ * - Otherwise we'll directly create and set the src
+ */
+export declare function assertElement(mediaElement: HTMLMediaElement | HTMLImageElement | undefined, parentElement: HTMLElement, clip: MediaClipState, constructAssetURL: (file: string) => string, preloader: MediaPreloader): HTMLElement;
+/**
+ * Makes sure that the element looks correct.
+ * - If the opacity, zIndex or fit are incorrect, we'll set again
+ */
 export declare function assertVisualProperties(mediaElement: HTMLMediaElement | HTMLImageElement, properties: VisualProperties, objectFit: ImageMetadata['fit']): void;
-export declare function assertAudialProperties(mediaElement: HTMLMediaElement, properties: AudialProperties, sinkId: string): void;
+/**
+ * Makes sure that the element sounds correct.
+ * - It should have the right volume, and play out the correct speaker.
+ */
+export declare function assertAudialProperties(mediaElement: HTMLMediaElement, properties: AudialProperties, sinkId: string, surfaceVolume: number): void;
 interface TemporalSyncState {
     state: 'idle' | 'seeking' | 'intercepting';
 }
-export declare function assertTemporalProperties(mediaElement: HTMLMediaElement, properties: TemporalProperties, keyframes: VideoState['keyframes'], syncState: TemporalSyncState): TemporalSyncState;
+/**
+ * Makes sure the media is at the correct time and speed.
+ * - If we fall slightly behind, we will lightly adjust the speed to catch up.
+ * - If we are too far away to smoothly realign, we will seek to the correct time.
+ */
+export declare function assertTemporalProperties(mediaElement: HTMLMediaElement, properties: TemporalProperties, keyframes: VideoState['keyframes'], syncState: TemporalSyncState, disablePlaybackRateAdjustment?: boolean): TemporalSyncState;
 export declare class ImageManager extends MediaClipManager<ImageState> {
     private imageElement;
     protected update(): void;
@@ -33,12 +53,14 @@ export declare class ImageManager extends MediaClipManager<ImageState> {
 export declare class AudioManager extends MediaClipManager<AudioState> {
     private syncState;
     private audioElement;
+    volume: number;
     protected update(): void;
     destroy(): void;
 }
 export declare class VideoManager extends MediaClipManager<VideoState> {
     private syncState;
     private videoElement?;
+    volume: number;
     protected update(): void;
     destroy(): void;
 }

package/dist/state-based/MediaClipManager.js

@@ -1,4 +1,14 @@
 import { getStateAtTime } from '../utils/getStateAtTime';
+const getPath = (url) => {
+    try {
+        const { pathname } = new URL(url, window.location.href);
+        return pathname;
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    }
+    catch (_) {
+        return undefined;
+    }
+};
 /**
  * Each instance of a MediaClipManager is responsible for displaying
  * an image/audio/video clip in the correct state.
@@ -7,15 +17,15 @@ export class MediaClipManager {
     surfaceElement;
     clipElement;
     constructAssetURL;
+    getAudioOutput;
     mediaPreloader;
-    constructor(surfaceElement, clipElement, state, constructAssetURL, mediaPreloader) {
+    constructor(surfaceElement, clipElement, state, constructAssetURL, getAudioOutput, mediaPreloader) {
         this.surfaceElement = surfaceElement;
         this.clipElement = clipElement;
         this.constructAssetURL = constructAssetURL;
+        this.getAudioOutput = getAudioOutput;
         this.mediaPreloader = mediaPreloader;
         this._state = state;
-        // Allow the class to be constructed, then call the loop
-        setTimeout(this.loop);
     }
     isConnected(element) {
         if (!this.surfaceElement) {
@@ -39,50 +49,62 @@ export class MediaClipManager {
     }
     timeout;
     loop = async () => {
+        clearTimeout(this.timeout);
         if (this.isConnected()) {
             this.update();
-            this.timeout = setTimeout(this.loop, 0);
+            this.timeout = setTimeout(this.loop, INNER_TARGET_SYNC_THRESHOLD_MS);
         }
         else {
             this.destroy();
         }
     };
 }
+/**
+ * Makes sure that the child media element exists and is of the correct type
+ * - If it isn't or doesn't exist we'll get a new one
+ * - If it is audio or video we'll try and get a preloaded media element
+ * - Otherwise we'll directly create and set the src
+ */
 export function assertElement(mediaElement, parentElement, clip, constructAssetURL, preloader) {
-    let element;
+    let element = undefined;
     const assetURL = constructAssetURL(clip.file);
+    const assetPath = getPath(assetURL);
     switch (clip.type) {
         case 'image':
             {
                 element = mediaElement instanceof HTMLImageElement ? mediaElement : document.createElement('img');
-                if (!element.src.includes(assetURL)) {
+                const elementPath = getPath(element.src);
+                if (elementPath !== assetPath) {
                     element.src = assetURL;
                 }
             }
             break;
         case 'audio':
-            if (mediaElement instanceof HTMLAudioElement && mediaElement.src.includes(assetURL)) {
-                element = mediaElement;
-            }
-            else {
-                element = preloader.getElement(clip.file, clip.type);
-            }
-            break;
-        case 'video':
-            if (mediaElement instanceof HTMLVideoElement && mediaElement.src.includes(assetURL)) {
-                element = mediaElement;
+        case 'video': {
+            if (mediaElement !== undefined) {
+                const path = getPath(mediaElement.src);
+                if (mediaElement.tagName.toLowerCase() === clip.type && path !== undefined && path === assetPath) {
+                    element = mediaElement;
+                }
             }
-            else {
+            if (!element) {
                 element = preloader.getElement(clip.file, clip.type);
             }
             break;
+        }
+    }
+    if (parentElement.children.length !== 1 || parentElement.childNodes[0] !== element) {
+        parentElement.replaceChildren(element);
     }
-    parentElement.replaceChildren(element);
     element.style.position = 'absolute';
     element.style.width = '100%';
     element.style.height = '100%';
     return element;
 }
+/**
+ * Makes sure that the element looks correct.
+ * - If the opacity, zIndex or fit are incorrect, we'll set again
+ */
 export function assertVisualProperties(mediaElement, properties, objectFit) {
     const opacityString = String(properties.opacity);
     if (mediaElement.style.opacity !== opacityString) {
@@ -96,9 +118,14 @@ export function assertVisualProperties(mediaElement, properties, objectFit) {
         mediaElement.style.objectFit = objectFit;
     }
 }
-export function assertAudialProperties(mediaElement, properties, sinkId) {
-    if (mediaElement.volume !== properties.volume) {
-        mediaElement.volume = properties.volume;
+/**
+ * Makes sure that the element sounds correct.
+ * - It should have the right volume, and play out the correct speaker.
+ */
+export function assertAudialProperties(mediaElement, properties, sinkId, surfaceVolume) {
+    const clipVolume = properties.volume * surfaceVolume;
+    if (mediaElement.volume !== clipVolume) {
+        mediaElement.volume = clipVolume;
     }
     if (mediaElement.sinkId !== sinkId) {
         try {
@@ -113,6 +140,7 @@ export function assertAudialProperties(mediaElement, properties, sinkId) {
     }
 }
 const OUTER_TARGET_SYNC_THRESHOLD_MS = 50; // When outside of this range we attempt to sync playback
+const OUTER_TARGET_SYNC_NO_PLAYBACK_RATE_ADJUSTMENT_THRESHOLD_MS = 500; // When outside of this range we attempt to sync playback (when playback rate adjustment not allowed)
 const INNER_TARGET_SYNC_THRESHOLD_MS = 5; // When attempting to sync playback, we aim for this accuracy
 const MAX_SYNC_THRESHOLD_MS = 1_000; // If we are further than this, we will seek instead
 const SEEK_LOOKAHEAD_MS = 5; // If it takes time to seek, we should seek ahead a little
@@ -122,7 +150,12 @@ const MAX_PLAYBACK_RATE_ADJUSTMENT = 0.1;
 function playbackSmoothing(deltaTime) {
     return Math.sign(deltaTime) * Math.pow(Math.abs(deltaTime) / MAX_SYNC_THRESHOLD_MS, PLAYBACK_ADJUSTMENT_SMOOTHING) * MAX_PLAYBACK_RATE_ADJUSTMENT;
 }
-export function assertTemporalProperties(mediaElement, properties, keyframes, syncState) {
+/**
+ * Makes sure the media is at the correct time and speed.
+ * - If we fall slightly behind, we will lightly adjust the speed to catch up.
+ * - If we are too far away to smoothly realign, we will seek to the correct time.
+ */
+export function assertTemporalProperties(mediaElement, properties, keyframes, syncState, disablePlaybackRateAdjustment) {
     if (mediaElement.paused && properties.rate > 0) {
         mediaElement.play().catch(() => {
             /* Do nothing, will be tried in next loop */
@@ -154,6 +187,19 @@ export function assertTemporalProperties(mediaElement, properties, keyframes, sy
             }
             return { state: 'idle' };
         case syncState.state === 'idle' &&
+            properties.rate > 0 &&
+            disablePlaybackRateAdjustment === true &&
+            deltaTimeAbs <= OUTER_TARGET_SYNC_NO_PLAYBACK_RATE_ADJUSTMENT_THRESHOLD_MS:
+            // If we aren't able to adjust playback rate, we are more forgiving
+            // in our "synced" check to avoid the clip being seeked forward
+            // in normal playback where we expect to be a little out of sync
+            // due to network and startup latency
+            if (mediaElement.playbackRate !== properties.rate) {
+                mediaElement.playbackRate = properties.rate;
+            }
+            return { state: 'idle' };
+        case syncState.state === 'idle' &&
+            disablePlaybackRateAdjustment !== true && // Never adjust playback rate if disabled for this clip
             properties.rate > 0 &&
             deltaTimeAbs > OUTER_TARGET_SYNC_THRESHOLD_MS &&
             deltaTimeAbs <= MAX_SYNC_THRESHOLD_MS:
@@ -177,7 +223,6 @@ export function assertTemporalProperties(mediaElement, properties, keyframes, sy
         }
         case syncState.state === 'intercepting' && Math.sign(deltaTime) === Math.sign(mediaElement.playbackRate - properties.rate): {
             // We have missed our interception.  Go back to idle to try again.
-            console.warn(deltaTime, 'missed intercept');
             return { state: 'idle' };
         }
         case syncState.state === 'intercepting':
@@ -221,6 +266,7 @@ export class ImageManager extends MediaClipManager {
 export class AudioManager extends MediaClipManager {
     syncState = { state: 'idle' };
     audioElement;
+    volume = 1;
     update() {
         const currentState = getStateAtTime(this._state, Date.now());
         if (currentState) {
@@ -231,8 +277,9 @@ export class AudioManager extends MediaClipManager {
         }
         if (!currentState || !this.audioElement)
             return;
-        assertAudialProperties(this.audioElement, currentState, this._state.audioOutput);
-        const nextSyncState = assertTemporalProperties(this.audioElement, currentState, this._state.keyframes, this.syncState);
+        const sinkId = this.getAudioOutput(this._state.audioOutput);
+        assertAudialProperties(this.audioElement, currentState, sinkId, this.volume);
+        const nextSyncState = assertTemporalProperties(this.audioElement, currentState, this._state.keyframes, this.syncState, this._state.disablePlaybackRateAdjustment);
         if (this.syncState.state !== 'seeking' && nextSyncState.state === 'seeking') {
             this.audioElement.addEventListener('seeked', () => {
                 this.syncState = { state: 'idle' };
@@ -241,13 +288,20 @@ export class AudioManager extends MediaClipManager {
         this.syncState = nextSyncState;
     }
     destroy() {
-        this.audioElement?.remove();
+        if (this.audioElement) {
+            this.audioElement.pause();
+            this.audioElement.remove();
+            this.audioElement.volume = 0;
+            this.audioElement.currentTime = 0;
+            this.mediaPreloader.releaseElement(this.audioElement);
+        }
         this.audioElement = undefined;
     }
 }
 export class VideoManager extends MediaClipManager {
     syncState = { state: 'idle' };
     videoElement;
+    volume = 1;
     update() {
         const currentState = getStateAtTime(this._state, Date.now());
         if (currentState) {
@@ -258,9 +312,10 @@ export class VideoManager extends MediaClipManager {
         }
         if (!currentState || !this.videoElement)
             return;
+        const sinkId = this.getAudioOutput(this._state.audioOutput);
         assertVisualProperties(this.videoElement, currentState, this._state.fit);
-        assertAudialProperties(this.videoElement, currentState, this._state.audioOutput);
-        const nextSyncState = assertTemporalProperties(this.videoElement, currentState, this._state.keyframes, this.syncState);
+        assertAudialProperties(this.videoElement, currentState, sinkId, this.volume);
+        const nextSyncState = assertTemporalProperties(this.videoElement, currentState, this._state.keyframes, this.syncState, this._state.disablePlaybackRateAdjustment);
         if (this.syncState.state !== 'seeking' && nextSyncState.state === 'seeking') {
             this.videoElement.addEventListener('seeked', () => {
                 this.syncState = { state: 'idle' };
@@ -269,7 +324,13 @@ export class VideoManager extends MediaClipManager {
         this.syncState = nextSyncState;
     }
     destroy() {
-        this.videoElement?.remove();
+        if (this.videoElement) {
+            this.videoElement.pause();
+            this.videoElement.remove();
+            this.videoElement.volume = 0;
+            this.videoElement.currentTime = 0;
+            this.mediaPreloader.releaseElement(this.videoElement);
+        }
         this.videoElement = undefined;
     }
 }

package/dist/state-based/MediaPreloader.d.ts

@@ -10,5 +10,5 @@ export declare class MediaPreloader {
     setState(newState: MediaClientConfig['files']): void;
     private update;
     getElement(file: string, type: 'audio' | 'video'): HTMLAudioElement;
-    releaseElement(resource: string | HTMLMediaElement): void;
+    releaseElement(resource: string | HTMLElement): void;
 }

package/dist/state-based/MediaPreloader.js

@@ -83,9 +83,9 @@ export class MediaPreloader {
             }
         }
         else {
-            Object.entries(this._elements).forEach(([file, media]) => {
+            Object.values(this._elements).forEach((media) => {
                 if (media.element === resource) {
-                    delete this._elements[file];
+                    media.inUse = false;
                 }
             });
         }

package/dist/state-based/SurfaceManager.d.ts

@@ -12,6 +12,9 @@ export declare class SurfaceManager {
     private mediaPreloader;
     private _state;
     setState(newState: MediaSurfaceState): void;
+    private _volume;
+    get volume(): number;
+    set volume(newVolume: number);
     private _element;
     get element(): HTMLDivElement;
     private resources;

package/dist/state-based/SurfaceManager.js

@@ -15,6 +15,18 @@ export class SurfaceManager {
         this._state = newState;
         this.update();
     }
+    _volume = 1;
+    get volume() {
+        return this._volume;
+    }
+    set volume(newVolume) {
+        this._volume = newVolume;
+        Object.values(this.resources).forEach(({ manager }) => {
+            if (manager instanceof AudioManager || manager instanceof VideoManager) {
+                manager.volume = newVolume;
+            }
+        });
+    }
     _element;
     get element() {
         return this._element;
@@ -68,14 +80,23 @@ export class SurfaceManager {
             if (!resource.manager) {
                 switch (clip.type) {
                     case 'image':
-                        resource.manager = new ImageManager(this._element, resource.element, clip, this.constructAssetUrl, this.mediaPreloader);
+                        resource.manager = new ImageManager(this._element, resource.element, clip, this.constructAssetUrl, this.getAudioOutput, this.mediaPreloader);
+                        resource.manager.loop();
                         break;
-                    case 'audio':
-                        resource.manager = new AudioManager(this._element, resource.element, clip, this.constructAssetUrl, this.mediaPreloader);
+                    case 'audio': {
+                        const audioManager = new AudioManager(this._element, resource.element, clip, this.constructAssetUrl, this.getAudioOutput, this.mediaPreloader);
+                        resource.manager = audioManager;
+                        audioManager.volume = this._volume;
+                        audioManager.loop();
                         break;
-                    case 'video':
-                        resource.manager = new VideoManager(this._element, resource.element, clip, this.constructAssetUrl, this.mediaPreloader);
+                    }
+                    case 'video': {
+                        const videoManager = new VideoManager(this._element, resource.element, clip, this.constructAssetUrl, this.getAudioOutput, this.mediaPreloader);
+                        resource.manager = videoManager;
+                        videoManager.volume = this._volume;
+                        videoManager.loop();
                         break;
+                    }
                 }
             }
             else {

package/dist/types/MediaSchema.d.ts

@@ -194,6 +194,7 @@ export type AudioState = {
     type: 'audio';
     file: string;
     audioOutput: string;
+    disablePlaybackRateAdjustment?: boolean;
     keyframes: [InitialAudioKeyframe, ...Array<AudioKeyframe | NullKeyframe>];
 };
 export type VideoState = {
@@ -201,6 +202,7 @@ export type VideoState = {
     file: string;
     fit: 'cover' | 'contain' | 'none';
     audioOutput: string;
+    disablePlaybackRateAdjustment?: boolean;
     keyframes: [InitialVideoKeyframe, ...Array<VideoKeyframe | NullKeyframe>];
 };
 export type MediaClipState = ImageState | AudioState | VideoState;

package/package.json

@@ -3,7 +3,7 @@
   "description": "Connect to COGS to build a custom Media Master",
   "author": "Clockwork Dog <info@clockwork.dog>",
   "homepage": "https://github.com/clockwork-dog/cogs-sdk/tree/main/packages/javascript",
-  "version": "3.0.0-alpha.11",
+  "version": "3.0.0-alpha.14",
   "keywords": [],
   "license": "MIT",
   "repository": {
@@ -37,7 +37,7 @@
     "cy:generate": "cypress run --e2e"
   },
   "dependencies": {
-    "@clockworkdog/timesync": "^3.0.0-alpha.11",
+    "@clockworkdog/timesync": "^3.0.0-alpha.14",
     "howler": "clockwork-dog/howler.js#fix-looping-clips",
     "reconnecting-websocket": "^4.4.0",
     "zod": "^4.1.13"