@clockworkdog/cogs-client 3.0.0-alpha.4 → 3.0.0-alpha.5

package/dist/index.d.ts

@@ -7,6 +7,7 @@ export type { default as MediaObjectFit } from './types/MediaObjectFit';
 export * as MediaSchema from './types/MediaSchema';
 export { default as CogsAudioPlayer } from './AudioPlayer';
 export { default as CogsVideoPlayer } from './VideoPlayer';
+export { SurfaceManager } from './state-based/SurfaceManager';
 export * from './types/AudioState';
 export { assetUrl, preloadUrl } from './utils/urls';
 export { getStateAtTime } from './utils/getStateAtTime';

package/dist/index.js

@@ -3,6 +3,7 @@ export * from './CogsConnection';
 export * as MediaSchema from './types/MediaSchema';
 export { default as CogsAudioPlayer } from './AudioPlayer';
 export { default as CogsVideoPlayer } from './VideoPlayer';
+export { SurfaceManager } from './state-based/SurfaceManager';
 export * from './types/AudioState';
 export { assetUrl, preloadUrl } from './utils/urls';
 export { getStateAtTime } from './utils/getStateAtTime';

package/dist/state-based/AudioManager.d.ts

@@ -0,0 +1,15 @@
+import { AudioState } from '../types/MediaSchema';
+import { ClipManager } from './ClipManager';
+export declare class AudioManager extends ClipManager<AudioState> {
+    private audioElement?;
+    private isSeeking;
+    constructor(surfaceElement: HTMLElement, clipElement: HTMLElement, state: AudioState);
+    private updateAudioElement;
+    /**
+     * Helper function to seek to a specified time.
+     * Works with the update loop to poll until seeked event has fired.
+     */
+    private seekTo;
+    protected update(): void;
+    destroy(): void;
+}

package/dist/state-based/AudioManager.js

@@ -0,0 +1,112 @@
+import { defaultAudioOptions } from '../types/MediaSchema';
+import { getStateAtTime } from '../utils/getStateAtTime';
+import { ClipManager } from './ClipManager';
+const DEFAULT_AUDIO_POLLING = 1_000;
+const TARGET_SYNC_THRESHOLD_MS = 10; // If we're closer than this we're good enough
+const MAX_SYNC_THRESHOLD_MS = 1_000; // If we're further away than this, we'll seek instead
+const SEEK_LOOKAHEAD_MS = 200; // We won't seek ahead instantly, so lets seek ahead
+const MAX_PLAYBACK_RATE_ADJUSTMENT = 0.2;
+// We smoothly ramp playbackRate up and down
+const PLAYBACK_ADJUSTMENT_SMOOTHING = 0.5;
+function playbackSmoothing(deltaTime) {
+    return Math.sign(deltaTime) * Math.pow(Math.abs(deltaTime) / MAX_SYNC_THRESHOLD_MS, PLAYBACK_ADJUSTMENT_SMOOTHING) * MAX_PLAYBACK_RATE_ADJUSTMENT;
+}
+export class AudioManager extends ClipManager {
+    audioElement;
+    isSeeking = false;
+    constructor(surfaceElement, clipElement, state) {
+        super(surfaceElement, clipElement, state);
+        this.clipElement = clipElement;
+    }
+    updateAudioElement() {
+        this.destroy();
+        this.audioElement = document.createElement('audio');
+        this.clipElement.replaceChildren(this.audioElement);
+    }
+    /**
+     * Helper function to seek to a specified time.
+     * Works with the update loop to poll until seeked event has fired.
+     */
+    seekTo(time) {
+        if (!this.audioElement)
+            return;
+        this.audioElement.addEventListener('seeked', () => {
+            this.isSeeking = false;
+        }, { once: true, passive: true });
+        this.audioElement.currentTime = time / 1_000;
+    }
+    update() {
+        // Update loop used to poll until seek finished
+        if (this.isSeeking)
+            return;
+        this.delay = DEFAULT_AUDIO_POLLING;
+        // Does the <audio /> element need adding/removing?
+        const currentState = getStateAtTime(this._state, Date.now());
+        if (currentState) {
+            if (!this.audioElement || !this.isConnected(this.audioElement)) {
+                this.updateAudioElement();
+            }
+        }
+        else {
+            this.destroy();
+        }
+        if (!currentState || !this.audioElement)
+            return;
+        const { t, rate, volume } = { ...defaultAudioOptions, ...currentState };
+        // this.audioElement.src will be a fully qualified URL
+        if (!this.audioElement.src.endsWith(this._state.file)) {
+            this.audioElement.src = this._state.file;
+        }
+        if (this.audioElement.volume !== volume) {
+            this.audioElement.volume = volume;
+        }
+        // Should the element be playing?
+        if (this.audioElement.paused && rate > 0) {
+            this.audioElement.play().catch(() => {
+                // Do nothing - this will be retried in the next loop
+            });
+        }
+        const currentTime = this.audioElement.currentTime * 1000;
+        const deltaTime = currentTime - t;
+        const deltaTimeAbs = Math.abs(deltaTime);
+        this.delay = 100;
+        switch (true) {
+            case deltaTimeAbs <= TARGET_SYNC_THRESHOLD_MS:
+                // We are on course:
+                //   - The audio is within accepted latency of the server time
+                //   - The playback rate is aligned with the server rate
+                if (this.audioElement.playbackRate !== rate) {
+                    this.audioElement.playbackRate = rate;
+                }
+                break;
+            case rate > 0 && deltaTimeAbs > TARGET_SYNC_THRESHOLD_MS && deltaTimeAbs <= MAX_SYNC_THRESHOLD_MS: {
+                // We are close, we can smoothly adjust with playbackRate:
+                //  - The audio must be playing
+                //  - We must be close in time to the server time
+                const playbackRateAdjustment = playbackSmoothing(deltaTime);
+                const adjustedPlaybackRate = Math.max(0, rate - playbackRateAdjustment);
+                if (this.audioElement.playbackRate !== adjustedPlaybackRate) {
+                    this.audioElement.playbackRate = adjustedPlaybackRate;
+                }
+                break;
+            }
+            default: {
+                // We cannot smoothly recover:
+                //  - We seek just ahead of server time
+                if (this.audioElement.playbackRate !== rate) {
+                    this.audioElement.playbackRate = rate;
+                }
+                // delay to poll until seeked
+                this.delay = 10;
+                this.seekTo(t + rate * (SEEK_LOOKAHEAD_MS / 1000));
+                break;
+            }
+        }
+    }
+    destroy() {
+        if (this.audioElement) {
+            this.audioElement.src = '';
+            this.audioElement.remove();
+        }
+    }
+}

package/dist/state-based/ClipManager.d.ts

@@ -0,0 +1,22 @@
+import { MediaClipState } from '../types/MediaSchema';
+/**
+ * Each instance of a ClipManager is responsible for displaying
+ * an image/audio/video clip in the correct state.
+ */
+export declare abstract class ClipManager<T extends MediaClipState> {
+    private surfaceElement;
+    protected clipElement: HTMLElement;
+    constructor(surfaceElement: HTMLElement, clipElement: HTMLElement, state: T);
+    /**
+     * This is the delay to be used in the update loop.
+     * It is intended to be dynamic for each loop.
+     */
+    protected delay: number;
+    protected abstract update(): void;
+    abstract destroy(): void;
+    isConnected(element?: HTMLElement): boolean;
+    protected _state: T;
+    setState(newState: T): void;
+    private timeout;
+    private loop;
+}

package/dist/state-based/ClipManager.js

@@ -0,0 +1,53 @@
+const DEFAULT_DELAY = 1_000;
+/**
+ * Each instance of a ClipManager is responsible for displaying
+ * an image/audio/video clip in the correct state.
+ */
+export class ClipManager {
+    surfaceElement;
+    clipElement;
+    constructor(surfaceElement, clipElement, state) {
+        this.surfaceElement = surfaceElement;
+        this.clipElement = clipElement;
+        this._state = state;
+        // Allow the class to be constructed, then call the loop
+        setTimeout(this.loop);
+    }
+    /**
+     * This is the delay to be used in the update loop.
+     * It is intended to be dynamic for each loop.
+     */
+    delay = DEFAULT_DELAY;
+    isConnected(element) {
+        if (!this.surfaceElement) {
+            return false;
+        }
+        if (!this.clipElement) {
+            return false;
+        }
+        if (!this.surfaceElement.contains(this.clipElement)) {
+            return false;
+        }
+        if (element) {
+            if (!this.clipElement.contains(element))
+                return false;
+        }
+        return true;
+    }
+    _state;
+    setState(newState) {
+        this._state = newState;
+        clearTimeout(this.timeout);
+        this.loop();
+    }
+    timeout;
+    loop = async () => {
+        if (this.isConnected()) {
+            this.update();
+            this.timeout = setTimeout(this.loop, this.delay);
+        }
+        else {
+            this.destroy();
+        }
+    };
+}

package/dist/state-based/ImageManager.d.ts

@@ -0,0 +1,9 @@
+import { ImageState } from '../types/MediaSchema';
+import { ClipManager } from './ClipManager';
+export declare class ImageManager extends ClipManager<ImageState> {
+    private imageElement?;
+    constructor(surfaceElement: HTMLElement, clipElement: HTMLElement, state: ImageState);
+    private updateImageElement;
+    protected update(): void;
+    destroy(): void;
+}

package/dist/state-based/ImageManager.js

@@ -0,0 +1,53 @@
+import { defaultImageOptions } from '../types/MediaSchema';
+import { getStateAtTime } from '../utils/getStateAtTime';
+import { ClipManager } from './ClipManager';
+export class ImageManager extends ClipManager {
+    imageElement;
+    constructor(surfaceElement, clipElement, state) {
+        super(surfaceElement, clipElement, state);
+        this.clipElement = clipElement;
+    }
+    updateImageElement() {
+        this.imageElement = document.createElement('img');
+        this.clipElement.replaceChildren(this.imageElement);
+        this.imageElement.style.position = 'absolute';
+        this.imageElement.style.height = '100%';
+        this.imageElement.style.widows = '100%';
+    }
+    update() {
+        const currentState = getStateAtTime(this._state, Date.now());
+        // Does the <img /> element need adding/removing?
+        if (currentState) {
+            if (!this.imageElement || !this.isConnected(this.imageElement)) {
+                this.updateImageElement();
+            }
+        }
+        else {
+            this.imageElement?.remove();
+            this.imageElement = undefined;
+        }
+        if (!this.imageElement || !currentState)
+            return;
+        // this.imageElement.src will be a fully qualified URL
+        if (!this.imageElement.src.startsWith(this._state.file)) {
+            this.imageElement.src = this._state.file;
+        }
+        if (this.imageElement.style.objectFit !== this._state.fit) {
+            this.imageElement.style.objectFit = this._state.fit;
+        }
+        if (parseFloat(this.imageElement.style.opacity) !== currentState.opacity) {
+            this.imageElement.style.opacity = String(currentState.opacity ?? defaultImageOptions.opacity);
+        }
+        const z = Math.round(currentState.zIndex ?? defaultImageOptions.zIndex);
+        if (parseInt(this.imageElement.style.zIndex) !== z) {
+            this.imageElement.style.zIndex = String(z);
+        }
+        const { opacity } = currentState;
+        if (typeof opacity === 'string' && opacity !== this.imageElement.style.opacity) {
+            this.imageElement.style.opacity = opacity;
+        }
+    }
+    destroy() {
+        this.imageElement?.remove();
+    }
+}

package/dist/state-based/SurfaceManager.d.ts

@@ -0,0 +1,16 @@
+import { MediaSurfaceState } from '../types/MediaSchema';
+export declare const DATA_CLIP_ID = "data-clip-id";
+/**
+ * The SurfaceManager will receive state updates and:
+ * - Ensure that each clip has a parent element
+ * - Instantiate a ClipManager attached to each respective element
+ */
+export declare class SurfaceManager {
+    private _state;
+    setState(newState: MediaSurfaceState): void;
+    private _element;
+    get element(): HTMLDivElement;
+    private resources;
+    constructor(testState?: MediaSurfaceState);
+    update(): void;
+}

package/dist/state-based/SurfaceManager.js

@@ -0,0 +1,80 @@
+import { ImageManager } from './ImageManager';
+import { VideoManager } from './VideoManager';
+import { AudioManager } from './AudioManager';
+export const DATA_CLIP_ID = 'data-clip-id';
+/**
+ * The SurfaceManager will receive state updates and:
+ * - Ensure that each clip has a parent element
+ * - Instantiate a ClipManager attached to each respective element
+ */
+export class SurfaceManager {
+    _state = {};
+    setState(newState) {
+        this._state = newState;
+        this.update();
+    }
+    _element;
+    get element() {
+        return this._element;
+    }
+    resources = {};
+    constructor(testState) {
+        this._element = document.createElement('div');
+        this._element.style.width = '100%';
+        this._element.style.height = '100%';
+        this._state = testState || {};
+        this.update();
+    }
+    update() {
+        // Destroy stale managers
+        Object.entries(this.resources).forEach(([clipId, { element, manager }]) => {
+            if (!(clipId in this._state)) {
+                delete this.resources[clipId];
+                element.remove();
+                manager?.destroy();
+            }
+        });
+        // Create and attach new wrapper elements
+        const elements = Object.keys(this._state)
+            .toSorted()
+            .map((clipId) => {
+            const resource = this.resources[clipId];
+            if (resource) {
+                return resource.element;
+            }
+            else {
+                const element = document.createElement('div');
+                element.setAttribute(DATA_CLIP_ID, clipId);
+                this.resources[clipId] = { element };
+                return element;
+            }
+        });
+        this._element.replaceChildren(...elements);
+        // Create new managers
+        Object.keys(this._state)
+            .toSorted()
+            .forEach((clipId) => {
+            const clip = this._state[clipId];
+            const resource = this.resources[clipId];
+            if (!resource) {
+                throw new Error('Failed to create resource');
+            }
+            if (!resource.manager) {
+                switch (clip.type) {
+                    case 'image':
+                        resource.manager = new ImageManager(this._element, resource.element, clip);
+                        break;
+                    case 'audio':
+                        resource.manager = new AudioManager(this._element, resource.element, clip);
+                        break;
+                    case 'video':
+                        resource.manager = new VideoManager(this._element, resource.element, clip);
+                        break;
+                }
+            }
+            else {
+                resource.manager.setState(clip);
+            }
+        });
+    }
+}

package/dist/state-based/VideoManager.d.ts

@@ -0,0 +1,15 @@
+import { VideoState } from '../types/MediaSchema';
+import { ClipManager } from './ClipManager';
+export declare class VideoManager extends ClipManager<VideoState> {
+    private videoElement?;
+    private isSeeking;
+    constructor(surfaceElement: HTMLElement, clipElement: HTMLElement, state: VideoState);
+    private updateVideoElement;
+    /**
+     * Helper function to seek to a specified time.
+     * Works with the update loop to poll until seeked event has fired.
+     */
+    private seekTo;
+    protected update(): void;
+    destroy(): void;
+}

package/dist/state-based/VideoManager.js

@@ -0,0 +1,126 @@
+import { defaultVideoOptions } from '../types/MediaSchema';
+import { getStateAtTime } from '../utils/getStateAtTime';
+import { ClipManager } from './ClipManager';
+const DEFAULT_VIDEO_POLLING = 1_000;
+const TARGET_SYNC_THRESHOLD_MS = 10; // If we're closer than this we're good enough
+const MAX_SYNC_THRESHOLD_MS = 1_000; // If we're further away than this, we'll seek instead
+const SEEK_LOOKAHEAD_MS = 200; // We won't seek ahead instantly, so lets seek ahead
+const MAX_PLAYBACK_RATE_ADJUSTMENT = 0.5;
+// We smoothly ramp playbackRate up and down
+const PLAYBACK_ADJUSTMENT_SMOOTHING = 0.3;
+function playbackSmoothing(deltaTime) {
+    return Math.sign(deltaTime) * Math.pow(Math.abs(deltaTime) / MAX_SYNC_THRESHOLD_MS, PLAYBACK_ADJUSTMENT_SMOOTHING) * MAX_PLAYBACK_RATE_ADJUSTMENT;
+}
+export class VideoManager extends ClipManager {
+    videoElement;
+    isSeeking = false;
+    constructor(surfaceElement, clipElement, state) {
+        super(surfaceElement, clipElement, state);
+        this.clipElement = clipElement;
+    }
+    updateVideoElement() {
+        this.destroy();
+        this.videoElement = document.createElement('video');
+        this.clipElement.replaceChildren(this.videoElement);
+        this.videoElement.style.position = 'absolute';
+        this.videoElement.style.width = '100%';
+        this.videoElement.style.height = '100%';
+    }
+    /**
+     * Helper function to seek to a specified time.
+     * Works with the update loop to poll until seeked event has fired.
+     */
+    seekTo(time) {
+        if (!this.videoElement)
+            return;
+        this.videoElement.addEventListener('seeked', () => {
+            this.isSeeking = false;
+        }, { once: true, passive: true });
+        this.videoElement.currentTime = time / 1_000;
+    }
+    update() {
+        // Update loop used to poll until seek finished
+        if (this.isSeeking)
+            return;
+        this.delay = DEFAULT_VIDEO_POLLING;
+        // Does the <video /> element need adding/removing?
+        const currentState = getStateAtTime(this._state, Date.now());
+        if (currentState) {
+            if (!this.videoElement || !this.isConnected(this.videoElement)) {
+                this.updateVideoElement();
+            }
+        }
+        else {
+            this.videoElement?.remove();
+            this.videoElement = undefined;
+        }
+        if (!currentState || !this.videoElement)
+            return;
+        const { t, rate, volume } = { ...defaultVideoOptions, ...currentState };
+        // this.videoElement.src will be a fully qualified URL
+        if (!this.videoElement.src.endsWith(this._state.file)) {
+            this.videoElement.src = this._state.file;
+        }
+        if (this.videoElement.style.objectFit !== this._state.fit) {
+            this.videoElement.style.objectFit = this._state.fit;
+        }
+        if (parseFloat(this.videoElement.style.opacity) !== currentState.opacity) {
+            this.videoElement.style.opacity = String(currentState.opacity ?? defaultVideoOptions.opacity);
+        }
+        const z = Math.round(currentState.zIndex ?? defaultVideoOptions.zIndex);
+        if (parseInt(this.videoElement.style.zIndex) !== z) {
+            this.videoElement.style.zIndex = String(z);
+        }
+        if (this.videoElement.volume !== volume) {
+            this.videoElement.volume = volume;
+        }
+        // Should the element be playing?
+        if (this.videoElement.paused && rate > 0) {
+            this.videoElement.play().catch(() => {
+                // Do nothing - this will be retried in the next loop
+            });
+        }
+        const currentTime = this.videoElement.currentTime * 1000;
+        const deltaTime = currentTime - t;
+        const deltaTimeAbs = Math.abs(deltaTime);
+        this.delay = 100;
+        switch (true) {
+            case deltaTimeAbs <= TARGET_SYNC_THRESHOLD_MS:
+                // We are on course:
+                //   - The video is within accepted latency of the server time
+                //   - The playback rate is aligned with the server rate
+                if (this.videoElement.playbackRate !== rate) {
+                    this.videoElement.playbackRate = rate;
+                }
+                break;
+            case rate > 0 && deltaTimeAbs > TARGET_SYNC_THRESHOLD_MS && deltaTimeAbs <= MAX_SYNC_THRESHOLD_MS: {
+                // We are close, we can smoothly adjust with playbackRate:
+                //  - The video must be playing
+                //  - We must be close in time to the server time
+                const playbackRateAdjustment = playbackSmoothing(deltaTime);
+                const adjustedPlaybackRate = Math.max(0, rate - playbackRateAdjustment);
+                if (this.videoElement.playbackRate !== adjustedPlaybackRate) {
+                    this.videoElement.playbackRate = adjustedPlaybackRate;
+                }
+                break;
+            }
+            default: {
+                // We cannot smoothly recover:
+                //  - We seek just ahead of server time
+                if (this.videoElement.playbackRate !== rate) {
+                    this.videoElement.playbackRate = rate;
+                }
+                // delay to poll until seeked
+                this.delay = 10;
+                this.seekTo(t + rate * (SEEK_LOOKAHEAD_MS / 1000));
+                break;
+            }
+        }
+    }
+    destroy() {
+        if (this.videoElement) {
+            this.videoElement.src = '';
+            this.videoElement.remove();
+        }
+    }
+}

package/dist/utils/getStateAtTime.js

@@ -47,11 +47,11 @@ export function getPropertiesAtTime(keyframes, time) {
             // If lerp and set are both present, we assume we lerp up until the timestamp,
             // then set to a new value
             Object.entries(properties.lerp ?? {}).forEach(([property, value]) => {
-                propertyKeyframes[property] ?? (propertyKeyframes[property] = {});
+                propertyKeyframes[property] ??= {};
                 propertyKeyframes[property].before = [timestamp, value];
             });
             Object.entries(properties.set ?? {}).forEach(([property, value]) => {
-                propertyKeyframes[property] ?? (propertyKeyframes[property] = {});
+                propertyKeyframes[property] ??= {};
                 propertyKeyframes[property].before = [timestamp, value];
             });
         }
@@ -59,7 +59,7 @@ export function getPropertiesAtTime(keyframes, time) {
             // We're trying to find the closest timestamp afterwards for lerping
             // So only set if not yet set
             Object.entries(properties.lerp ?? {}).forEach(([property, value]) => {
-                propertyKeyframes[property] ?? (propertyKeyframes[property] = {});
+                propertyKeyframes[property] ??= {};
                 if (propertyKeyframes[property].after === undefined) {
                     propertyKeyframes[property].after = [timestamp, value];
                 }

package/package.json

@@ -3,7 +3,7 @@
   "description": "Connect to COGS to build a custom Media Master",
   "author": "Clockwork Dog <info@clockwork.dog>",
   "homepage": "https://github.com/clockwork-dog/cogs-sdk/tree/main/packages/javascript",
-  "version": "3.0.0-alpha.4",
+  "version": "3.0.0-alpha.5",
   "keywords": [],
   "license": "MIT",
   "repository": {
@@ -22,7 +22,7 @@
     "default": "./dist/index.js"
   },
   "scripts": {
-    "test": "yarn types && yarn lint && vitest",
+    "test": "yarn types && yarn lint && vitest && yarn cy:run",
     "types": "tsc --noEmit",
     "lint": "eslint .",
     "build": "yarn build:ts && yarn build:browser",
@@ -31,24 +31,32 @@
     "watch-build": "tsc -w",
     "build-docs": "typedoc --out ../../docs/javascript --name @clockworkdog/cogs-client src/index.ts",
     "release": "yarn npm publish --access public",
-    "prerelease": "yarn npm publish --access public --tag=next"
+    "prerelease": "yarn npm publish --access public --tag=next",
+    "cy:open": "cypress open",
+    "cy:run": "cypress run --component",
+    "cy:generate": "cypress run --e2e"
   },
   "dependencies": {
-    "@clockworkdog/timesync": "^3.0.0-alpha.4",
+    "@clockworkdog/timesync": "^3.0.0-alpha.5",
     "howler": "clockwork-dog/howler.js#fix-looping-clips",
     "reconnecting-websocket": "^4.4.0",
     "zod": "^4.1.13"
   },
   "devDependencies": {
+    "@cypress/mount-utils": "^4.1.2",
     "@eslint/js": "^9.17.0",
     "@types/howler": "2.2.12",
     "@types/jsdom": "^27",
     "@types/node": "^22.10.2",
+    "cypress": "^14.5.4",
     "eslint": "^9.17.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-prettier": "^5.2.1",
     "jsdom": "^27.1.0",
     "prettier": "^3.4.2",
+    "react": "^19.1.1",
+    "react-dom": "^19.1.1",
+    "sharp": "^0.34.5",
     "typedoc": "^0.27.5",
     "typescript": "~5.7.2",
     "typescript-eslint": "^8.18.1",