@elah/core 0.1.0

package/dist/media/video/VideoDecoderManager.js

@@ -0,0 +1,342 @@
+import { MediabunnyDemuxer, } from './demuxer/MediabunnyDemuxer';
+const DEFAULT_IDLE_TIMEOUT_MS = 5000;
+const DEFAULT_FPS = 30;
+const MAX_DECODE_QUEUE_DEPTH = 4;
+function _vdmTrace(msg, data) {
+    if (typeof globalThis !== 'undefined' && globalThis.__VDM_DEBUG__) {
+        if (data) {
+            console.log(`[VDM-TRACE] ${msg}`, data);
+        }
+        else {
+            console.log(`[VDM-TRACE] ${msg}`);
+        }
+    }
+}
+const VALID_TRANSITIONS = {
+    Idle: ['Opening', 'Disposed'],
+    Opening: ['Ready', 'Errored', 'Disposed'],
+    Ready: ['Resetting', 'Draining', 'Disposed', 'Errored'],
+    Resetting: ['Ready', 'Errored', 'Disposed'],
+    Draining: ['Idle', 'Errored', 'Disposed'],
+    Disposed: [],
+    Errored: ['Idle', 'Disposed'],
+};
+export class VideoDecoderManager {
+    constructor(options = {}) {
+        this.onFrame = null;
+        this._state = 'Idle';
+        this._src = null;
+        this._demuxer = null;
+        this._decoder = null;
+        this._idleTimer = null;
+        this._idleCallback = null;
+        this._feedGeneration = 0;
+        this._feedActive = false;
+        this._feedPendingRange = null;
+        this._demuxerFactory = options.demuxerFactory;
+        this._idleTimeoutMs = options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS;
+        this._fps = options.fps ?? DEFAULT_FPS;
+        this._onStateChange = options.onStateChange ?? null;
+        this._onDroppedFrame = options.onDroppedFrame ?? null;
+        this._onError = options.onError ?? null;
+        this._decoderFactory = options.decoderFactory ?? ((output, error) => {
+            if (typeof VideoDecoder === 'undefined') {
+                throw new Error('VideoDecoderManager: VideoDecoder not available');
+            }
+            return new VideoDecoder({ output, error });
+        });
+    }
+    get fps() {
+        return this._fps;
+    }
+    get state() {
+        return this._state;
+    }
+    get src() {
+        return this._src;
+    }
+    setIdleCallback(cb) {
+        this._idleCallback = cb;
+    }
+    async open(src) {
+        this._assertTransition('Opening');
+        this._src = src;
+        try {
+            this._demuxer = new MediabunnyDemuxer(this._demuxerFactory);
+            await this._demuxer.open(src);
+            const config = {
+                ...this._demuxer.getConfig(),
+                optimizeForLatency: true,
+            };
+            const usPerFrame = 1000000 / this._fps;
+            this._decoder = this._decoderFactory((frame) => {
+                if (this._state === 'Disposed') {
+                    frame.close();
+                    return;
+                }
+                if (this.onFrame) {
+                    const sourceFrameIdx = Math.round(frame.timestamp / usPerFrame);
+                    _vdmTrace('decoder.output → onFrame', {
+                        sourceFrameIdx,
+                        timestampUs: frame.timestamp,
+                        decoderQueueSize: this._decoder
+                            ? this._decoder.decodeQueueSize
+                            : undefined,
+                        state: this._state,
+                    });
+                    this.onFrame(frame, sourceFrameIdx);
+                }
+                else {
+                    frame.close();
+                }
+            }, (error) => {
+                _vdmTrace('decoder.error', { message: error.message });
+                this._handleError(error);
+            });
+            this._decoder.configure(config);
+            this._transition('Ready');
+        }
+        catch (error) {
+            this._handleError(error);
+            throw error;
+        }
+    }
+    async reopen(src) {
+        if (this._state === 'Errored') {
+            this._cleanupResources();
+            this._transition('Idle');
+        }
+        if (this._state !== 'Idle') {
+            throw new Error(`VideoDecoderManager: reopen() invalid from state ${this._state}`);
+        }
+        await this.open(src);
+    }
+    feed(timeRangeUs) {
+        if (this._state !== 'Ready') {
+            _vdmTrace('feed dropped — state not Ready', {
+                state: this._state,
+                timeRangeUs,
+            });
+            return;
+        }
+        if (this._feedActive) {
+            if (this._feedPendingRange === null) {
+                this._feedPendingRange = timeRangeUs;
+            }
+            else {
+                this._feedPendingRange = [
+                    Math.min(this._feedPendingRange[0], timeRangeUs[0]),
+                    Math.max(this._feedPendingRange[1], timeRangeUs[1]),
+                ];
+            }
+            _vdmTrace('feed coalesced into pending', {
+                incomingRange: timeRangeUs,
+                coalescedPending: this._feedPendingRange,
+                decoderQueueSize: this._decoder
+                    ? this._decoder.decodeQueueSize
+                    : undefined,
+            });
+            return;
+        }
+        this._feedActive = true;
+        const gen = this._feedGeneration;
+        _vdmTrace('feed → _feedAsync start', {
+            timeRangeUs,
+            gen,
+            decoderQueueSize: this._decoder
+                ? this._decoder.decodeQueueSize
+                : undefined,
+        });
+        void this._feedAsync(timeRangeUs, gen);
+    }
+    async reset(toKeyframeUs) {
+        if (this._state === 'Disposed' || this._state === 'Errored')
+            return;
+        if (this._state !== 'Ready') {
+            throw new Error(`VideoDecoderManager: reset() invalid from state ${this._state}`);
+        }
+        this._feedGeneration++;
+        this._feedActive = false;
+        this._feedPendingRange = null;
+        this._assertTransition('Resetting');
+        try {
+            await this._demuxer.seekToKeyframe(toKeyframeUs);
+            this._decoder.reset();
+            this._decoder.configure({
+                ...this._demuxer.getConfig(),
+                optimizeForLatency: true,
+            });
+            this._transition('Ready');
+        }
+        catch (error) {
+            this._handleError(error);
+            throw error;
+        }
+    }
+    async debugFlush() {
+        if (this._state !== 'Ready' || !this._decoder)
+            return;
+        _vdmTrace('debugFlush → decoder.flush()', {
+            decoderQueueSize: this._decoder.decodeQueueSize,
+        });
+        await this._decoder.flush();
+        _vdmTrace('debugFlush complete', {});
+    }
+    async drain() {
+        if (this._state === 'Disposed')
+            return;
+        this._assertTransition('Draining');
+        try {
+            await this._decoder.flush();
+            this._cleanupResources();
+            this._transition('Idle');
+        }
+        catch (error) {
+            this._handleError(error);
+            throw error;
+        }
+    }
+    markIdle() {
+        if (this._state === 'Disposed')
+            return;
+        this._clearIdleTimer();
+        this._idleTimer = setTimeout(() => {
+            this._idleCallback?.();
+        }, this._idleTimeoutMs);
+    }
+    markActive() {
+        if (this._state === 'Disposed')
+            return;
+        this._clearIdleTimer();
+    }
+    dispose() {
+        if (this._state === 'Disposed')
+            return;
+        this._feedGeneration++;
+        this._feedActive = false;
+        this._feedPendingRange = null;
+        this._clearIdleTimer();
+        this._cleanupResources();
+        this._src = null;
+        this._transition('Disposed');
+    }
+    async _feedAsync(timeRangeUs, gen) {
+        let packetCount = 0;
+        let firstTimestampUs = null;
+        let lastTimestampUs = null;
+        let stateNotReadyExit = false;
+        try {
+            for await (const chunk of this._demuxer.packets(timeRangeUs)) {
+                if (gen !== this._feedGeneration) {
+                    _vdmTrace('_feedAsync stale-gen exit during iteration', {
+                        gen,
+                        currentGen: this._feedGeneration,
+                        packetsConsumed: packetCount,
+                    });
+                    return;
+                }
+                if (this._state !== 'Ready') {
+                    _vdmTrace('_feedAsync state-not-Ready exit during iteration', {
+                        state: this._state,
+                        packetsConsumed: packetCount,
+                    });
+                    stateNotReadyExit = true;
+                    break;
+                }
+                if (firstTimestampUs === null)
+                    firstTimestampUs = chunk.timestamp;
+                lastTimestampUs = chunk.timestamp;
+                const queueDepth = this._decoder.decodeQueueSize;
+                if (queueDepth !== undefined && queueDepth >= MAX_DECODE_QUEUE_DEPTH) {
+                    await new Promise(resolve => setTimeout(resolve, 0));
+                    if (gen !== this._feedGeneration)
+                        return;
+                    if (this._state !== 'Ready') {
+                        stateNotReadyExit = true;
+                        break;
+                    }
+                }
+                this._decoder.decode(chunk);
+                packetCount++;
+            }
+        }
+        catch (error) {
+            if (gen === this._feedGeneration && this._state !== 'Disposed') {
+                this._feedActive = false;
+                this._feedPendingRange = null;
+                _vdmTrace('_feedAsync ERROR', {
+                    gen,
+                    packetCount,
+                    error: error instanceof Error ? error.message : String(error),
+                });
+                const err = error instanceof Error ? error : new Error(String(error));
+                this._onDroppedFrame?.(0);
+                this._handleError(err);
+            }
+            return;
+        }
+        if (stateNotReadyExit) {
+            if (gen === this._feedGeneration) {
+                this._feedActive = false;
+                this._feedPendingRange = null;
+            }
+            return;
+        }
+        if (gen !== this._feedGeneration)
+            return;
+        const pending = this._feedPendingRange;
+        this._feedPendingRange = null;
+        this._feedActive = false;
+        _vdmTrace('_feedAsync completed', {
+            timeRangeUs,
+            packetCount,
+            firstTimestampUs,
+            lastTimestampUs,
+            hasPending: pending !== null,
+            pending,
+            decoderQueueSize: this._decoder
+                ? this._decoder.decodeQueueSize
+                : undefined,
+        });
+        if (pending !== null && this._state === 'Ready') {
+            this._feedActive = true;
+            _vdmTrace('_feedAsync chaining pending', { pending });
+            void this._feedAsync(pending, this._feedGeneration);
+        }
+    }
+    _assertTransition(target) {
+        if (this._state === 'Disposed') {
+            throw new Error(`VideoDecoderManager: invalid transition ${this._state} → ${target}`);
+        }
+        const allowed = VALID_TRANSITIONS[this._state];
+        if (!allowed.includes(target)) {
+            throw new Error(`VideoDecoderManager: invalid transition ${this._state} → ${target}`);
+        }
+        this._transition(target);
+    }
+    _transition(next) {
+        this._state = next;
+        this._onStateChange?.(next);
+    }
+    _handleError(error) {
+        const err = error instanceof Error ? error : new Error(String(error));
+        if (this._state !== 'Disposed' &&
+            this._state !== 'Idle' &&
+            this._state !== 'Errored') {
+            this._transition('Errored');
+        }
+        this._onError?.(err);
+    }
+    _cleanupResources() {
+        this._decoder?.close();
+        this._decoder = null;
+        this._demuxer?.dispose();
+        this._demuxer = null;
+    }
+    _clearIdleTimer() {
+        if (this._idleTimer !== null) {
+            clearTimeout(this._idleTimer);
+            this._idleTimer = null;
+        }
+    }
+}

package/dist/media/video/VideoFrameProvider.d.ts

@@ -0,0 +1,101 @@
+import type { DemuxerFactory } from './demuxer/MediabunnyDemuxer';
+import type { VideoDecoderFactory } from './VideoDecoderManager';
+export type ProvidedFrame = VideoFrame | ImageBitmap;
+export interface VideoFrameProvider {
+    getCurrent(sourceFrame: number): ProvidedFrame | null;
+    setPlayhead(sourceFrame: number, opts?: {
+        lookaheadFrames?: number;
+    }): void;
+    markIdle(): void;
+    markActive(): void;
+    dispose(): void;
+}
+export interface MetricsHook {
+    onHit?: (sourceFrame: number) => void;
+    onMiss?: (sourceFrame: number) => void;
+    onDecodeLatency?: (sourceFrame: number, ms: number) => void;
+}
+export interface MockVideoFrameProviderOptions {
+    maxFrames?: number;
+    idleTimeoutMs?: number;
+    frameWidth?: number;
+    frameHeight?: number;
+    metrics?: MetricsHook;
+    cacheHooks?: import('./FrameCache').FrameCacheHooks;
+    lookaheadFrames?: number;
+}
+export interface SyntheticVideoFrameProviderOptions {
+    maxFrames?: number;
+    idleTimeoutMs?: number;
+    frameWidth?: number;
+    frameHeight?: number;
+    fps?: number;
+    metrics?: MetricsHook;
+    cacheHooks?: import('./FrameCache').FrameCacheHooks;
+    lookaheadFrames?: number;
+}
+type ProviderState = 'active' | 'idle' | 'disposed';
+export declare class MockVideoFrameProvider implements VideoFrameProvider {
+    private readonly _cache;
+    private readonly _idleTimeoutMs;
+    private readonly _frameWidth;
+    private readonly _frameHeight;
+    private readonly _metrics;
+    private readonly _defaultLookahead;
+    private _state;
+    private readonly _pending;
+    private _idleTimer;
+    private _idleCallback;
+    constructor(options?: MockVideoFrameProviderOptions);
+    setIdleCallback(cb: (() => void) | null): void;
+    getCurrent(sourceFrame: number): VideoFrame | null;
+    setPlayhead(sourceFrame: number, opts?: {
+        lookaheadFrames?: number;
+    }): void;
+    markIdle(): void;
+    markActive(): void;
+    dispose(): void;
+    get state(): ProviderState;
+    get cacheSize(): number;
+    private _scheduleFrame;
+    private _clearIdleTimer;
+    private _createMockFrame;
+}
+export declare class SyntheticVideoFrameProvider implements VideoFrameProvider {
+    private readonly _cache;
+    private readonly _idleTimeoutMs;
+    private readonly _frameWidth;
+    private readonly _frameHeight;
+    private readonly _fps;
+    private readonly _metrics;
+    private readonly _canvas;
+    private readonly _ctx;
+    private readonly _defaultLookahead;
+    private _state;
+    private readonly _pending;
+    private _idleTimer;
+    private _idleCallback;
+    constructor(options?: SyntheticVideoFrameProviderOptions);
+    setIdleCallback(cb: (() => void) | null): void;
+    getCurrent(sourceFrame: number): VideoFrame | null;
+    setPlayhead(sourceFrame: number, opts?: {
+        lookaheadFrames?: number;
+    }): void;
+    markIdle(): void;
+    markActive(): void;
+    dispose(): void;
+    get state(): ProviderState;
+    get cacheSize(): number;
+    private _scheduleFrame;
+    private _clearIdleTimer;
+    private _createSyntheticFrame;
+}
+export interface VideoFrameProviderDeps {
+    demuxerFactory?: DemuxerFactory;
+    decoderFactory?: VideoDecoderFactory;
+    fps?: number;
+    lookaheadFrames?: number;
+    frameConverter?: (frame: VideoFrame) => Promise<ImageBitmap>;
+}
+export declare function createVideoFrameProvider(src: string, deps?: VideoFrameProviderDeps): VideoFrameProvider;
+export {};

package/dist/media/video/VideoFrameProvider.js

@@ -0,0 +1,257 @@
+import { FrameCache } from './FrameCache';
+import { GpuDebugCounters } from '../../renderer/gpu/debug/GpuDebugCounters';
+import { StreamingFrameProducer } from './StreamingFrameProducer';
+const DEFAULT_IDLE_TIMEOUT_MS = 5000;
+const DEFAULT_FRAME_WIDTH = 640;
+const DEFAULT_FRAME_HEIGHT = 360;
+const DEFAULT_FPS = 30;
+const DEFAULT_LOOKAHEAD_FRAMES = 8;
+function canUseSyntheticFrames() {
+    return (typeof OffscreenCanvas !== 'undefined' &&
+        typeof VideoFrame !== 'undefined');
+}
+export class MockVideoFrameProvider {
+    constructor(options = {}) {
+        this._state = 'active';
+        this._pending = new Set();
+        this._idleTimer = null;
+        this._idleCallback = null;
+        this._cache = new FrameCache({
+            maxFrames: options.maxFrames,
+            hooks: options.cacheHooks,
+        });
+        this._idleTimeoutMs = options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS;
+        this._frameWidth = options.frameWidth ?? 320;
+        this._frameHeight = options.frameHeight ?? 240;
+        this._metrics = options.metrics ?? {};
+        this._defaultLookahead = options.lookaheadFrames ?? DEFAULT_LOOKAHEAD_FRAMES;
+    }
+    setIdleCallback(cb) {
+        this._idleCallback = cb;
+    }
+    getCurrent(sourceFrame) {
+        if (this._state === 'disposed')
+            return null;
+        this._cache.setPivot(sourceFrame);
+        const frame = this._cache.get(sourceFrame);
+        if (frame !== null) {
+            this._metrics.onHit?.(sourceFrame);
+            GpuDebugCounters.cacheHits++;
+        }
+        else {
+            this._metrics.onMiss?.(sourceFrame);
+            GpuDebugCounters.cacheMisses++;
+        }
+        GpuDebugCounters.cacheSize = this._cache.size;
+        return frame;
+    }
+    setPlayhead(sourceFrame, opts) {
+        if (this._state === 'disposed')
+            return;
+        this._cache.setPivot(sourceFrame);
+        const lookahead = opts?.lookaheadFrames ?? this._defaultLookahead;
+        for (let i = 0; i <= lookahead; i++) {
+            this._scheduleFrame(sourceFrame + i);
+        }
+    }
+    markIdle() {
+        if (this._state === 'disposed')
+            return;
+        this._state = 'idle';
+        this._clearIdleTimer();
+        this._idleTimer = setTimeout(() => {
+            this._idleCallback?.();
+        }, this._idleTimeoutMs);
+    }
+    markActive() {
+        if (this._state === 'disposed')
+            return;
+        this._clearIdleTimer();
+        this._state = 'active';
+    }
+    dispose() {
+        if (this._state === 'disposed')
+            return;
+        this._state = 'disposed';
+        this._clearIdleTimer();
+        this._pending.clear();
+        this._cache.dispose();
+        GpuDebugCounters.cacheSize = 0;
+    }
+    get state() {
+        return this._state;
+    }
+    get cacheSize() {
+        return this._cache.size;
+    }
+    _scheduleFrame(sourceFrame) {
+        if (this._pending.has(sourceFrame))
+            return;
+        if (this._cache.has(sourceFrame))
+            return;
+        this._pending.add(sourceFrame);
+        const startedAt = performance.now();
+        setTimeout(() => {
+            if (this._state === 'disposed') {
+                this._pending.delete(sourceFrame);
+                return;
+            }
+            const frame = this._createMockFrame();
+            this._cache.put(sourceFrame, frame);
+            this._pending.delete(sourceFrame);
+            GpuDebugCounters.cacheSize = this._cache.size;
+            this._metrics.onDecodeLatency?.(sourceFrame, performance.now() - startedAt);
+            GpuDebugCounters.recordDecodeLatency(performance.now() - startedAt);
+        }, 0);
+    }
+    _clearIdleTimer() {
+        if (this._idleTimer !== null) {
+            clearTimeout(this._idleTimer);
+            this._idleTimer = null;
+        }
+    }
+    _createMockFrame() {
+        return {
+            displayWidth: this._frameWidth,
+            displayHeight: this._frameHeight,
+            close: () => { },
+        };
+    }
+}
+export class SyntheticVideoFrameProvider {
+    constructor(options = {}) {
+        this._state = 'active';
+        this._pending = new Set();
+        this._idleTimer = null;
+        this._idleCallback = null;
+        this._cache = new FrameCache({
+            maxFrames: options.maxFrames,
+            hooks: options.cacheHooks,
+        });
+        this._idleTimeoutMs = options.idleTimeoutMs ?? DEFAULT_IDLE_TIMEOUT_MS;
+        this._frameWidth = options.frameWidth ?? DEFAULT_FRAME_WIDTH;
+        this._frameHeight = options.frameHeight ?? DEFAULT_FRAME_HEIGHT;
+        this._fps = options.fps ?? DEFAULT_FPS;
+        this._metrics = options.metrics ?? {};
+        this._defaultLookahead = options.lookaheadFrames ?? DEFAULT_LOOKAHEAD_FRAMES;
+        this._canvas = new OffscreenCanvas(this._frameWidth, this._frameHeight);
+        const ctx = this._canvas.getContext('2d');
+        if (!ctx) {
+            throw new Error('SyntheticVideoFrameProvider: 2D context unavailable');
+        }
+        this._ctx = ctx;
+    }
+    setIdleCallback(cb) {
+        this._idleCallback = cb;
+    }
+    getCurrent(sourceFrame) {
+        if (this._state === 'disposed')
+            return null;
+        this._cache.setPivot(sourceFrame);
+        const frame = this._cache.get(sourceFrame);
+        if (frame !== null) {
+            this._metrics.onHit?.(sourceFrame);
+            GpuDebugCounters.cacheHits++;
+        }
+        else {
+            this._metrics.onMiss?.(sourceFrame);
+            GpuDebugCounters.cacheMisses++;
+        }
+        GpuDebugCounters.cacheSize = this._cache.size;
+        return frame;
+    }
+    setPlayhead(sourceFrame, opts) {
+        if (this._state === 'disposed')
+            return;
+        this._cache.setPivot(sourceFrame);
+        const lookahead = opts?.lookaheadFrames ?? this._defaultLookahead;
+        for (let i = 0; i <= lookahead; i++) {
+            this._scheduleFrame(sourceFrame + i);
+        }
+    }
+    markIdle() {
+        if (this._state === 'disposed')
+            return;
+        this._state = 'idle';
+        this._clearIdleTimer();
+        this._idleTimer = setTimeout(() => {
+            this._idleCallback?.();
+        }, this._idleTimeoutMs);
+    }
+    markActive() {
+        if (this._state === 'disposed')
+            return;
+        this._clearIdleTimer();
+        this._state = 'active';
+    }
+    dispose() {
+        if (this._state === 'disposed')
+            return;
+        this._state = 'disposed';
+        this._clearIdleTimer();
+        this._pending.clear();
+        this._cache.dispose();
+        GpuDebugCounters.cacheSize = 0;
+    }
+    get state() {
+        return this._state;
+    }
+    get cacheSize() {
+        return this._cache.size;
+    }
+    _scheduleFrame(sourceFrame) {
+        if (this._pending.has(sourceFrame))
+            return;
+        if (this._cache.has(sourceFrame))
+            return;
+        this._pending.add(sourceFrame);
+        const startedAt = performance.now();
+        setTimeout(() => {
+            if (this._state === 'disposed') {
+                this._pending.delete(sourceFrame);
+                return;
+            }
+            const frame = this._createSyntheticFrame(sourceFrame);
+            this._cache.put(sourceFrame, frame);
+            this._pending.delete(sourceFrame);
+            GpuDebugCounters.cacheSize = this._cache.size;
+            const latency = performance.now() - startedAt;
+            this._metrics.onDecodeLatency?.(sourceFrame, latency);
+            GpuDebugCounters.recordDecodeLatency(latency);
+        }, 0);
+    }
+    _clearIdleTimer() {
+        if (this._idleTimer !== null) {
+            clearTimeout(this._idleTimer);
+            this._idleTimer = null;
+        }
+    }
+    _createSyntheticFrame(sourceFrame) {
+        const hue = (sourceFrame * 37) % 360;
+        this._ctx.fillStyle = `hsl(${hue}, 70%, 45%)`;
+        this._ctx.fillRect(0, 0, this._frameWidth, this._frameHeight);
+        this._ctx.fillStyle = '#ffffff';
+        this._ctx.font = 'bold 48px monospace';
+        this._ctx.textAlign = 'center';
+        this._ctx.textBaseline = 'middle';
+        this._ctx.fillText(String(sourceFrame), this._frameWidth / 2, this._frameHeight / 2);
+        const timestamp = Math.round((sourceFrame / this._fps) * 1000000);
+        return new VideoFrame(this._canvas, { timestamp });
+    }
+}
+export function createVideoFrameProvider(src, deps) {
+    if (deps?.demuxerFactory) {
+        return new StreamingFrameProducer({
+            src,
+            fps: deps.fps,
+            lookaheadFrames: deps.lookaheadFrames,
+            demuxerFactory: deps.demuxerFactory,
+            decoderFactory: deps.decoderFactory,
+            frameConverter: deps.frameConverter,
+        });
+    }
+    if (canUseSyntheticFrames()) {
+        return new SyntheticVideoFrameProvider();
+    }
+    return new MockVideoFrameProvider();
+}