@elah/core 0.1.0

package/dist/media/audio/AudioPlaybackController.js

@@ -0,0 +1,125 @@
+import { resolveTimeline } from '../../resolver/resolveTimeline';
+export class AudioPlaybackController {
+    constructor(playback, getProject, options = {}) {
+        this._ctx = null;
+        this._buffers = new Map();
+        this._node = null;
+        this._gain = null;
+        this._activeClipId = null;
+        this._lastEpoch = -1;
+        this._scheduleToken = 0;
+        this._unsub = null;
+        this._playback = playback;
+        this._getProject = getProject;
+        this._audioContextFactory =
+            options.audioContextFactory ?? (() => new AudioContext());
+    }
+    start() {
+        if (this._unsub)
+            return;
+        if (!this._ctx) {
+            this._ctx = this._audioContextFactory();
+        }
+        this._unsub = this._playback.subscribe((snap) => this._onSnapshot(snap));
+        this._onSnapshot({
+            currentFrame: this._playback.currentFrame,
+            isPlaying: this._playback.isPlaying,
+            playbackRate: this._playback.playbackRate,
+            loop: this._playback.loop,
+            epoch: -1,
+        });
+    }
+    destroy() {
+        this._unsub?.();
+        this._unsub = null;
+        this._stop();
+        this._activeClipId = null;
+        if (this._ctx) {
+            void this._ctx.close();
+            this._ctx = null;
+        }
+    }
+    _onSnapshot(snap) {
+        const transportChanged = snap.epoch !== this._lastEpoch;
+        this._lastEpoch = snap.epoch;
+        const scene = resolveTimeline(snap.currentFrame, this._getProject());
+        const clip = scene.audios.length > 0 ? scene.audios[scene.audios.length - 1] : null;
+        if (!clip) {
+            this._stop();
+            this._activeClipId = null;
+            return;
+        }
+        if (this._gain) {
+            this._gain.gain.value = clip.volume;
+        }
+        if (!snap.isPlaying) {
+            this._stop();
+            return;
+        }
+        const needRestart = transportChanged || clip.id !== this._activeClipId || this._node === null;
+        if (!needRestart)
+            return;
+        const offsetSec = clip.sourceFrame / scene.fps;
+        void this._schedule(clip.id, clip.src, offsetSec, clip.volume);
+    }
+    async _schedule(clipId, src, offsetSec, volume) {
+        const token = ++this._scheduleToken;
+        let buffer;
+        try {
+            buffer = await this._ensureBuffer(src);
+        }
+        catch {
+            return;
+        }
+        if (token !== this._scheduleToken || !this._ctx)
+            return;
+        this._startNode(buffer, offsetSec, volume);
+        this._activeClipId = clipId;
+    }
+    _startNode(buffer, offsetSec, volume) {
+        const ctx = this._ctx;
+        if (!ctx)
+            return;
+        this._stop();
+        if (offsetSec >= buffer.duration)
+            return;
+        const offset = Math.max(0, offsetSec);
+        if (ctx.state === 'suspended') {
+            void ctx.resume();
+        }
+        const node = ctx.createBufferSource();
+        node.buffer = buffer;
+        const gain = ctx.createGain();
+        gain.gain.value = volume;
+        node.connect(gain).connect(ctx.destination);
+        node.start(ctx.currentTime, offset);
+        this._node = node;
+        this._gain = gain;
+    }
+    _stop() {
+        if (this._node) {
+            try {
+                this._node.stop();
+                this._node.disconnect();
+            }
+            catch {
+            }
+        }
+        this._node = null;
+        this._gain = null;
+        this._scheduleToken++;
+    }
+    _ensureBuffer(src) {
+        const cached = this._buffers.get(src);
+        if (cached)
+            return cached;
+        const ctx = this._ctx;
+        if (!ctx)
+            return Promise.reject(new Error('AudioPlaybackController: no AudioContext'));
+        const promise = fetch(src)
+            .then((res) => res.arrayBuffer())
+            .then((buf) => ctx.decodeAudioData(buf));
+        this._buffers.set(src, promise);
+        return promise;
+    }
+}

package/dist/media/video/DecoderBackedVideoFrameProvider.d.ts

@@ -0,0 +1,51 @@
+import { type FrameCacheHooks } from './FrameCache';
+import type { VideoFrameProvider } from './VideoFrameProvider';
+import type { DemuxerFactory } from './demuxer/MediabunnyDemuxer';
+import type { VideoDecoderFactory } from './VideoDecoderManager';
+export interface DecoderBackedVideoFrameProviderOptions {
+    src: string;
+    fps?: number;
+    maxOutstanding?: number;
+    maxFrames?: number;
+    demuxerFactory: DemuxerFactory;
+    decoderFactory?: VideoDecoderFactory;
+    cacheHooks?: FrameCacheHooks;
+    strictNoOutput?: boolean;
+    decodeTimeoutMs?: number;
+}
+type ProviderState = 'active' | 'idle' | 'disposed';
+export declare class DecoderBackedVideoFrameProvider implements VideoFrameProvider {
+    private readonly _src;
+    private readonly _fps;
+    private readonly _maxOutstanding;
+    private readonly _cache;
+    private readonly _manager;
+    private readonly _pending;
+    private _lastRequested;
+    private _state;
+    private _idleTimer;
+    private _idleCallback;
+    private _openPromise;
+    private _openError;
+    private _reopening;
+    constructor(opts: DecoderBackedVideoFrameProviderOptions);
+    getCurrent(sourceFrame: number): VideoFrame | null;
+    requestFrame(_sourceFrame: number): void;
+    prefetch(_fromSourceFrame: number, _count: number): void;
+    setPlayhead(_sourceFrame: number, _opts?: {
+        lookaheadFrames?: number;
+    }): void;
+    markIdle(): void;
+    markActive(): void;
+    dispose(): void;
+    get state(): ProviderState;
+    get pendingCount(): number;
+    get cacheSize(): number;
+    get decoderState(): string;
+    get openError(): Error | null;
+    get openPromise(): Promise<void> | null;
+    setIdleCallback(cb: (() => void) | null): void;
+    private _clearIdleTimer;
+    private _enqueueRequestFrame;
+}
+export {};

package/dist/media/video/DecoderBackedVideoFrameProvider.js

@@ -0,0 +1,125 @@
+import { FrameCache } from './FrameCache';
+import { GpuDebugCounters } from '../../renderer/gpu/debug/GpuDebugCounters';
+import { VideoDecoderManager } from './VideoDecoderManager';
+const DEFAULT_MAX_OUTSTANDING = 4;
+const DEFAULT_MAX_FRAMES = 30;
+const DEFAULT_FPS = 30;
+const DEFAULT_IDLE_TIMEOUT_MS = 5000;
+export class DecoderBackedVideoFrameProvider {
+    constructor(opts) {
+        this._pending = new Set();
+        this._lastRequested = null;
+        this._state = 'active';
+        this._idleTimer = null;
+        this._idleCallback = null;
+        this._openPromise = null;
+        this._openError = null;
+        this._reopening = false;
+        this._src = opts.src;
+        this._fps = opts.fps ?? DEFAULT_FPS;
+        this._maxOutstanding = opts.maxOutstanding ?? DEFAULT_MAX_OUTSTANDING;
+        this._cache = new FrameCache({
+            maxFrames: opts.maxFrames ?? DEFAULT_MAX_FRAMES,
+            hooks: opts.cacheHooks,
+        });
+        this._manager = new VideoDecoderManager({
+            fps: this._fps,
+            demuxerFactory: opts.demuxerFactory,
+            decoderFactory: opts.decoderFactory,
+            onDroppedFrame: () => {
+                GpuDebugCounters.incDropped();
+            },
+            onError: (_err) => {
+                if (this._state === 'disposed' || this._reopening)
+                    return;
+                this._reopening = true;
+                this._openPromise = this._manager
+                    .reopen(this._src)
+                    .catch((reopenErr) => {
+                    this._openError = reopenErr;
+                })
+                    .finally(() => {
+                    this._reopening = false;
+                });
+            },
+        });
+        this._openPromise = this._manager.open(this._src).catch((err) => {
+            this._openError = err;
+        });
+    }
+    getCurrent(sourceFrame) {
+        if (this._state === 'disposed')
+            return null;
+        this._cache.setPivot(sourceFrame);
+        const frame = this._cache.get(sourceFrame);
+        if (frame !== null) {
+            GpuDebugCounters.cacheHits++;
+        }
+        else {
+            GpuDebugCounters.cacheMisses++;
+        }
+        GpuDebugCounters.cacheSize = this._cache.size;
+        return frame;
+    }
+    requestFrame(_sourceFrame) {
+    }
+    prefetch(_fromSourceFrame, _count) {
+    }
+    setPlayhead(_sourceFrame, _opts) {
+    }
+    markIdle() {
+        if (this._state === 'disposed')
+            return;
+        this._state = 'idle';
+        this._clearIdleTimer();
+        this._idleTimer = setTimeout(() => {
+            this._idleCallback?.();
+        }, DEFAULT_IDLE_TIMEOUT_MS);
+    }
+    markActive() {
+        if (this._state === 'disposed')
+            return;
+        this._clearIdleTimer();
+        this._state = 'active';
+    }
+    dispose() {
+        if (this._state === 'disposed')
+            return;
+        this._state = 'disposed';
+        this._clearIdleTimer();
+        this._manager.dispose();
+        this._cache.dispose();
+        GpuDebugCounters.cacheSize = 0;
+        this._pending.clear();
+        this._lastRequested = null;
+    }
+    get state() {
+        return this._state;
+    }
+    get pendingCount() {
+        return this._pending.size;
+    }
+    get cacheSize() {
+        return this._cache.size;
+    }
+    get decoderState() {
+        return this._manager.state;
+    }
+    get openError() {
+        return this._openError;
+    }
+    get openPromise() {
+        return this._openPromise;
+    }
+    setIdleCallback(cb) {
+        this._idleCallback = cb;
+    }
+    _clearIdleTimer() {
+        if (this._idleTimer !== null) {
+            clearTimeout(this._idleTimer);
+            this._idleTimer = null;
+        }
+    }
+    _enqueueRequestFrame(_sourceFrame) {
+    }
+}

package/dist/media/video/FrameCache.d.ts

@@ -0,0 +1,28 @@
+export interface Closeable {
+    close(): void;
+}
+export interface FrameCacheHooks {
+    onPut?: (sourceFrame: number) => void;
+    onEvict?: (sourceFrame: number) => void;
+    onClear?: () => void;
+}
+export interface FrameCacheOptions {
+    maxFrames?: number;
+    hooks?: FrameCacheHooks;
+}
+export declare class FrameCache<T extends Closeable = VideoFrame> {
+    private readonly _maxFrames;
+    private readonly _hooks;
+    private readonly _frames;
+    private _pivot;
+    constructor(maxFramesOrOptions?: number | FrameCacheOptions);
+    get size(): number;
+    setPivot(sourceFrame: number): void;
+    get(sourceFrame: number, maxLookback?: number): T | null;
+    put(sourceFrame: number, frame: T): void;
+    has(sourceFrame: number): boolean;
+    evictBefore(sourceFrame: number): void;
+    clear(): void;
+    dispose(): void;
+    private _evictFurthest;
+}

package/dist/media/video/FrameCache.js

@@ -0,0 +1,88 @@
+const DEFAULT_MAX_FRAMES = 30;
+export class FrameCache {
+    constructor(maxFramesOrOptions) {
+        this._frames = new Map();
+        this._pivot = 0;
+        if (typeof maxFramesOrOptions === 'number' || maxFramesOrOptions === undefined) {
+            this._maxFrames = maxFramesOrOptions ?? DEFAULT_MAX_FRAMES;
+            this._hooks = {};
+        }
+        else {
+            this._maxFrames = maxFramesOrOptions.maxFrames ?? DEFAULT_MAX_FRAMES;
+            this._hooks = maxFramesOrOptions.hooks ?? {};
+        }
+    }
+    get size() {
+        return this._frames.size;
+    }
+    setPivot(sourceFrame) {
+        this._pivot = sourceFrame;
+    }
+    get(sourceFrame, maxLookback = 0) {
+        const exact = this._frames.get(sourceFrame);
+        if (exact !== undefined)
+            return exact;
+        if (maxLookback <= 0)
+            return null;
+        let bestKey = sourceFrame - maxLookback - 1;
+        let best = null;
+        for (const [key, frame] of this._frames) {
+            if (key <= sourceFrame && key > bestKey) {
+                bestKey = key;
+                best = frame;
+            }
+        }
+        return best;
+    }
+    put(sourceFrame, frame) {
+        const existing = this._frames.get(sourceFrame);
+        if (existing) {
+            existing.close();
+            this._frames.delete(sourceFrame);
+            this._hooks.onEvict?.(sourceFrame);
+        }
+        else if (this._frames.size >= this._maxFrames) {
+            this._evictFurthest();
+        }
+        this._frames.set(sourceFrame, frame);
+        this._hooks.onPut?.(sourceFrame);
+    }
+    has(sourceFrame) {
+        return this._frames.has(sourceFrame);
+    }
+    evictBefore(sourceFrame) {
+        for (const key of [...this._frames.keys()]) {
+            if (key < sourceFrame) {
+                this._frames.get(key).close();
+                this._frames.delete(key);
+                this._hooks.onEvict?.(key);
+            }
+        }
+    }
+    clear() {
+        for (const frame of this._frames.values()) {
+            frame.close();
+        }
+        this._frames.clear();
+        this._hooks.onClear?.();
+    }
+    dispose() {
+        this.clear();
+    }
+    _evictFurthest() {
+        let victimKey = null;
+        let maxDist = -1;
+        for (const key of this._frames.keys()) {
+            const dist = Math.abs(key - this._pivot);
+            if (dist > maxDist || (dist === maxDist && victimKey !== null && key < victimKey)) {
+                maxDist = dist;
+                victimKey = key;
+            }
+        }
+        if (victimKey !== null) {
+            this._frames.get(victimKey).close();
+            this._frames.delete(victimKey);
+            this._hooks.onEvict?.(victimKey);
+        }
+    }
+}

package/dist/media/video/StreamingFrameProducer.d.ts

@@ -0,0 +1,57 @@
+import type { ProvidedFrame, VideoFrameProvider } from './VideoFrameProvider';
+import { type FrameCacheHooks } from './FrameCache';
+import type { DemuxerFactory } from './demuxer/MediabunnyDemuxer';
+import type { VideoDecoderFactory } from './VideoDecoderManager';
+type ProviderState = 'active' | 'idle' | 'disposed';
+export interface StreamingFrameProducerOptions {
+    src: string;
+    fps?: number;
+    lookaheadFrames?: number;
+    maxFrames?: number;
+    demuxerFactory: DemuxerFactory;
+    decoderFactory?: VideoDecoderFactory;
+    cacheHooks?: FrameCacheHooks;
+    frameConverter?: (frame: VideoFrame) => Promise<ImageBitmap>;
+}
+export declare class StreamingFrameProducer implements VideoFrameProvider {
+    private readonly _src;
+    private readonly _fps;
+    private readonly _lookaheadFrames;
+    private readonly _usPerFrame;
+    private readonly _cache;
+    private readonly _convert;
+    private readonly _manager;
+    private _state;
+    private _lastPlayhead;
+    private _latestPlayhead;
+    private _feedWatermark;
+    private _highestDecodedFrame;
+    private _ticksSinceDecodeAdvance;
+    private _resetInProgress;
+    private _lastLoggedPlayhead;
+    private _idleTimer;
+    private _idleCallback;
+    private _openPromise;
+    private _openError;
+    private _reopening;
+    constructor(opts: StreamingFrameProducerOptions);
+    getCurrent(sourceFrame: number): ProvidedFrame | null;
+    private _cacheKeysSnapshot;
+    setPlayhead(sourceFrame: number, opts?: {
+        lookaheadFrames?: number;
+    }): void;
+    markIdle(): void;
+    markActive(): void;
+    dispose(): void;
+    get state(): ProviderState;
+    get decoderState(): string;
+    get cacheSize(): number;
+    get openError(): Error | null;
+    get openPromise(): Promise<void> | null;
+    setIdleCallback(cb: (() => void) | null): void;
+    private _handleDiscontinuity;
+    private _feedWindow;
+    private _copyAndCache;
+    private _clearIdleTimer;
+}
+export {};