@livepeer-frameworks/player-core 0.0.4 → 0.1.0

package/dist/types/players/WebCodecsPlayer/types.d.ts

@@ -143,6 +143,8 @@ export interface CreatePipelineMessage {
     track: TrackInfo;
     opts: {
         optimizeForLatency: boolean;
+        /** Payload format: 'avcc' (length-prefixed) or 'annexb' (start-code delimited) */
+        payloadFormat?: 'avcc' | 'annexb';
     };
     uid?: number;
 }
@@ -181,9 +183,10 @@ export interface ClosePipelineMessage {
 }
 export interface FrameTimingMessage {
     type: 'frametiming';
-    action: 'setSpeed' | 'reset';
+    action: 'setSpeed' | 'reset' | 'setPaused';
     speed?: number;
     tweak?: number;
+    paused?: boolean;
     uid?: number;
 }
 export interface SeekWorkerMessage {
@@ -196,7 +199,19 @@ export interface DebuggingMessage {
     value: boolean | 'verbose';
     uid?: number;
 }
-export type MainToWorkerMessage = CreatePipelineMessage | ConfigurePipelineMessage | ReceiveChunkMessage | SetWritableMessage | CreateGeneratorMessage | ClosePipelineMessage | FrameTimingMessage | SeekWorkerMessage | DebuggingMessage;
+export interface FrameStepMessage {
+    type: 'framestep';
+    direction: -1 | 1;
+    uid?: number;
+}
+export interface WriteFrameResponseMessage {
+    type: 'writeframe';
+    idx: number;
+    uid?: number;
+    status: 'ok' | 'error';
+    error?: string;
+}
+export type MainToWorkerMessage = CreatePipelineMessage | ConfigurePipelineMessage | ReceiveChunkMessage | SetWritableMessage | CreateGeneratorMessage | ClosePipelineMessage | FrameTimingMessage | SeekWorkerMessage | DebuggingMessage | FrameStepMessage | WriteFrameResponseMessage;
 export interface AddTrackMessage {
     type: 'addtrack';
     idx: number;
@@ -228,6 +243,7 @@ export interface SendEventMessage {
     type: 'sendevent';
     kind: string;
     message?: string;
+    time?: number;
     idx?: number;
     uid?: number;
 }
@@ -243,7 +259,13 @@ export interface AckMessage {
     status?: 'ok' | 'error';
     error?: string;
 }
-export type WorkerToMainMessage = AddTrackMessage | RemoveTrackMessage | SetPlaybackRateMessage | ClosedMessage | LogMessage | SendEventMessage | StatsMessage | AckMessage;
+export interface WriteFrameMessage {
+    type: 'writeframe';
+    idx: number;
+    frame: AudioData;
+    uid?: number;
+}
+export type WorkerToMainMessage = AddTrackMessage | RemoveTrackMessage | SetPlaybackRateMessage | ClosedMessage | LogMessage | SendEventMessage | StatsMessage | AckMessage | WriteFrameMessage;
 export interface FrameTimingStats {
     /** Timestamp when frame entered decoder (microseconds) */
     in: number;

package/dist/types/players/WebCodecsPlayer/worker/types.d.ts

@@ -10,6 +10,8 @@ export interface CreateMessage {
     track: TrackInfo;
     opts: {
         optimizeForLatency: boolean;
+        /** Payload format: 'avcc' (length-prefixed) or 'annexb' (start-code delimited) */
+        payloadFormat?: 'avcc' | 'annexb';
     };
     uid: number;
 }
@@ -48,9 +50,10 @@ export interface CloseMessage {
 }
 export interface FrameTimingMessage {
     type: 'frametiming';
-    action: 'setSpeed' | 'reset';
+    action: 'setSpeed' | 'reset' | 'setPaused';
     speed?: number;
     tweak?: number;
+    paused?: boolean;
     uid: number;
 }
 export interface SeekMessage {
@@ -58,12 +61,17 @@ export interface SeekMessage {
     seekTime: number;
     uid: number;
 }
+export interface FrameStepMessage {
+    type: 'framestep';
+    direction: -1 | 1;
+    uid: number;
+}
 export interface DebuggingMessage {
     type: 'debugging';
     value: boolean | 'verbose';
     uid: number;
 }
-export type MainToWorkerMessage = CreateMessage | ConfigureMessage | ReceiveMessage | SetWritableMessage | CreateGeneratorMessage | CloseMessage | FrameTimingMessage | SeekMessage | DebuggingMessage;
+export type MainToWorkerMessage = CreateMessage | ConfigureMessage | ReceiveMessage | SetWritableMessage | CreateGeneratorMessage | CloseMessage | FrameTimingMessage | SeekMessage | FrameStepMessage | DebuggingMessage;
 export interface AddTrackMessage {
     type: 'addtrack';
     idx: number;
@@ -97,6 +105,7 @@ export interface SendEventMessage {
     type: 'sendevent';
     kind: string;
     message?: string;
+    time?: number;
     idx?: number;
     uid: number;
 }
@@ -159,6 +168,13 @@ export interface PipelineState {
         data: Uint8Array;
     }>;
     outputQueue: DecodedFrame[];
+    /** Recent video frames for backward/forward stepping (video only) */
+    frameHistory?: Array<{
+        frame: VideoFrame;
+        timestamp: number;
+    }>;
+    /** Cursor into frameHistory for step navigation */
+    historyCursor?: number | null;
     stats: {
         framesIn: number;
         framesDecoded: number;
@@ -172,6 +188,8 @@ export interface PipelineState {
         lastChunkBytes: string;
     };
     optimizeForLatency: boolean;
+    /** Payload format: 'avcc' (length-prefixed) or 'annexb' (start-code delimited) */
+    payloadFormat: 'avcc' | 'annexb';
 }
 export interface ScheduleResult {
     /** Whether frame should be output now */

package/dist/types/types.d.ts

@@ -50,6 +50,8 @@ export interface PlayerOptions {
     hlsConfig?: HlsJsConfig;
     /** DASH.js configuration override (merged with defaults) */
     dashConfig?: DashJsConfig;
+    /** Video.js VHS configuration override (merged with defaults) */
+    vhsConfig?: VhsConfig;
     /** WebRTC configuration (ICE servers, etc.) */
     rtcConfig?: RTCConfiguration;
     /** String to append to all request URLs (auth tokens, tracking params) */
@@ -98,6 +100,21 @@ export interface DashJsConfig {
     };
     [key: string]: unknown;
 }
+/** Video.js VHS (http-streaming) configuration subset */
+export interface VhsConfig {
+    /** Start with lowest quality for faster initial playback */
+    enableLowInitialPlaylist?: boolean;
+    /** Initial bandwidth estimate in bits per second (e.g., 5_000_000 for 5 Mbps) */
+    bandwidth?: number;
+    /** Persist bandwidth estimate in localStorage across sessions */
+    useBandwidthFromLocalStorage?: boolean;
+    /** Enable partial segment appends for lower latency */
+    handlePartialData?: boolean;
+    /** Time delta for live range safety calculations (seconds) */
+    liveRangeSafeTimeDelta?: number;
+    /** Pass-through for other VHS options */
+    [key: string]: unknown;
+}
 export type StreamProtocol = 'WHEP' | 'HLS' | 'DASH' | 'MP4' | 'WEBM' | 'RTMP' | 'MIST_HTML';
 export interface OutputCapabilities {
     supportsSeek: boolean;
@@ -138,7 +155,7 @@ export interface ContentMetadata {
     durationSeconds?: number;
     thumbnailUrl?: string;
     createdAt?: string;
-    status?: 'AVAILABLE' | 'PROCESSING' | 'ERROR' | 'OFFLINE';
+    status?: 'AVAILABLE' | 'PROCESSING' | 'ERROR' | 'OFFLINE' | 'ONLINE' | 'INITIALIZING' | 'BOOTING' | 'WAITING_FOR_DATA' | 'SHUTTING_DOWN' | 'INVALID';
     viewers?: number;
     isLive?: boolean;
     recordingSizeBytes?: number;
@@ -147,6 +164,19 @@ export interface ContentMetadata {
     dvrStatus?: 'recording' | 'completed';
     /** Native container format: mp4, m3u8, webm, etc. */
     format?: string;
+    /** MistServer authoritative snapshot (merged into this metadata) */
+    mist?: MistStreamInfo;
+    /** Parsed track summary (derived from Mist metadata when available) */
+    tracks?: Array<{
+        type: 'video' | 'audio' | 'meta';
+        codec?: string;
+        width?: number;
+        height?: number;
+        bitrate?: number;
+        fps?: number;
+        channels?: number;
+        sampleRate?: number;
+    }>;
 }
 export interface ContentEndpoints {
     primary: EndpointInfo;
@@ -360,4 +390,5 @@ export interface PlayerMetadata {
         channels?: number;
         sampleRate?: number;
     }>;
+    mist?: MistStreamInfo;
 }

package/dist/types/vanilla/FrameWorksPlayer.d.ts

@@ -6,11 +6,11 @@
  *
  * @example
  * ```typescript
- * import { FrameWorksPlayer } from '@livepeer-frameworks/player/vanilla';
- * import '@livepeer-frameworks/player/player.css';
+ * import { FrameWorksPlayer } from '@livepeer-frameworks/player-core/vanilla';
+ * import '@livepeer-frameworks/player-core/player.css';
  *
  * const player = new FrameWorksPlayer('#player', {
- *   contentId: 'my-stream',
+ *   contentId: 'pk_...',
  *   contentType: 'live',
  *   gatewayUrl: 'https://gateway.example.com/graphql',
  *   onStateChange: (state) => console.log('State:', state),
@@ -30,7 +30,7 @@ export interface FrameWorksPlayerOptions {
     /** Content identifier (stream name) */
     contentId: string;
     /** Content type */
-    contentType: ContentType;
+    contentType?: ContentType;
     /** Pre-resolved endpoints (skip gateway) */
     endpoints?: ContentEndpoints;
     /** Gateway URL (required if endpoints not provided) */
@@ -57,7 +57,7 @@ export interface FrameWorksPlayerOptions {
 }
 interface LegacyConfig {
     contentId: string;
-    contentType: ContentType;
+    contentType?: ContentType;
     thumbnailUrl?: string | null;
     options?: {
         gatewayUrl?: string;

package/dist/types/vanilla/index.d.ts

@@ -3,11 +3,11 @@
  *
  * @example
  * ```typescript
- * import { FrameWorksPlayer } from '@livepeer-frameworks/player/vanilla';
- * import '@livepeer-frameworks/player/player.css';
+ * import { FrameWorksPlayer } from '@livepeer-frameworks/player-core/vanilla';
+ * import '@livepeer-frameworks/player-core/player.css';
  *
  * const player = new FrameWorksPlayer('#player', {
- *   contentId: 'my-stream',
+ *   contentId: 'pk_...',
  *   contentType: 'live',
  *   gatewayUrl: 'https://gateway.example.com/graphql',
  * });

package/dist/workers/decoder.worker.js

@@ -57,6 +57,65 @@
         trackBaseTimes.clear();
         log(`Reset all track baseTimes`);
     }
+    function cloneVideoFrame(frame) {
+        try {
+            if ('clone' in frame) {
+                return frame.clone();
+            }
+            return new VideoFrame(frame);
+        }
+        catch {
+            return null;
+        }
+    }
+    function pushFrameHistory(pipeline, frame, timestamp) {
+        if (pipeline.track.type !== 'video')
+            return;
+        if (!pipeline.frameHistory)
+            pipeline.frameHistory = [];
+        const cloned = cloneVideoFrame(frame);
+        if (!cloned)
+            return;
+        pipeline.frameHistory.push({ frame: cloned, timestamp });
+        // Trim history
+        while (pipeline.frameHistory.length > MAX_FRAME_HISTORY) {
+            const entry = pipeline.frameHistory.shift();
+            if (entry) {
+                try {
+                    entry.frame.close();
+                }
+                catch { }
+            }
+        }
+        pipeline.historyCursor = pipeline.frameHistory.length - 1;
+    }
+    function alignHistoryCursorToLastOutput(pipeline) {
+        if (!pipeline.frameHistory || pipeline.frameHistory.length === 0)
+            return;
+        const lastTs = pipeline.stats.lastOutputTimestamp;
+        if (!Number.isFinite(lastTs)) {
+            pipeline.historyCursor = pipeline.frameHistory.length - 1;
+            return;
+        }
+        // Find first history entry greater than last output, then step back one
+        const idx = pipeline.frameHistory.findIndex(entry => entry.timestamp > lastTs);
+        if (idx === -1) {
+            pipeline.historyCursor = pipeline.frameHistory.length - 1;
+            return;
+        }
+        pipeline.historyCursor = Math.max(0, idx - 1);
+    }
+    function getPrimaryVideoPipeline() {
+        let selected = null;
+        for (const pipeline of pipelines.values()) {
+            if (pipeline.track.type === 'video') {
+                if (!selected || pipeline.idx < selected.idx) {
+                    selected = pipeline;
+                }
+            }
+        }
+        return selected;
+    }
     // Stats update interval
     let statsTimer = null;
     const STATS_INTERVAL_MS = 250;
@@ -64,6 +123,9 @@
     // Per Chrome WebCodecs best practices: drop when decodeQueueSize > 2
     // This ensures decoder doesn't fall too far behind before corrective action
     const MAX_DECODER_QUEUE_SIZE = 2;
+    const MAX_FRAME_HISTORY = 60;
+    const MAX_PAUSED_OUTPUT_QUEUE = 120;
+    const MAX_PAUSED_INPUT_QUEUE = 600;
     // ============================================================================
     // Logging
     // ============================================================================
@@ -113,6 +175,9 @@
             case 'seek':
                 handleSeek(msg);
                 break;
+            case 'framestep':
+                handleFrameStep(msg);
+                break;
             case 'debugging':
                 debugging = msg.value;
                 log(`Debugging set to: ${msg.value}`);
@@ -137,6 +202,8 @@
             writer: null,
             inputQueue: [],
             outputQueue: [],
+            frameHistory: track.type === 'video' ? [] : undefined,
+            historyCursor: track.type === 'video' ? null : undefined,
             stats: {
                 framesIn: 0,
                 framesDecoded: 0,
@@ -151,6 +218,7 @@
                 lastChunkBytes: '',
             },
             optimizeForLatency: opts.optimizeForLatency,
+            payloadFormat: opts.payloadFormat || 'avcc',
         };
         pipelines.set(idx, pipeline);
         // Start stats reporting if not already running
@@ -230,7 +298,11 @@
             hardwareAcceleration: 'prefer-hardware',
         };
         // Pass description directly from WebSocket INIT data (per reference rawws.js line 1052)
-        if (description && description.byteLength > 0) {
+        // For Annex B format (ws/video/h264), SPS/PPS comes inline in the bitstream - skip description
+        if (pipeline.payloadFormat === 'annexb') {
+            log(`Annex B mode - SPS/PPS inline in bitstream, no description needed`);
+        }
+        else if (description && description.byteLength > 0) {
             config.description = description;
             log(`Configuring with description (${description.byteLength} bytes)`);
         }
@@ -374,7 +446,7 @@
     // Frame Input/Output
     // ============================================================================
     function handleReceive(msg) {
-        const { idx, chunk, uid } = msg;
+        const { idx, chunk } = msg;
         const pipeline = pipelines.get(idx);
         if (!pipeline) {
             logVerbose(`Received chunk for unknown pipeline ${idx}`);
@@ -386,6 +458,15 @@
             logVerbose(`Queued chunk for track ${idx} (configured=${pipeline.configured}, decoder=${!!pipeline.decoder})`);
             return;
         }
+        // If paused and output queue is saturated, queue input to preserve per-frame stepping
+        if (frameTiming.paused && pipeline.outputQueue.length >= MAX_PAUSED_OUTPUT_QUEUE) {
+            pipeline.inputQueue.push(chunk);
+            if (pipeline.inputQueue.length > MAX_PAUSED_INPUT_QUEUE) {
+                pipeline.inputQueue.splice(0, pipeline.inputQueue.length - MAX_PAUSED_INPUT_QUEUE);
+                logVerbose(`Trimmed paused input queue for track ${idx} to ${MAX_PAUSED_INPUT_QUEUE}`);
+            }
+            return;
+        }
         // Log only first 3 chunks per track to confirm receiving
         if (pipeline.stats.framesIn < 3) {
             log(`Received chunk ${pipeline.stats.framesIn} for track ${idx}: type=${chunk.type}, ts=${chunk.timestamp / 1000}ms, size=${chunk.data.byteLength}`);
@@ -410,6 +491,8 @@
      * Based on Chrome WebCodecs best practices: drop when decodeQueueSize > 2
      */
     function shouldDropFramesDueToDecoderPressure(pipeline) {
+        if (frameTiming.paused)
+            return false;
         if (!pipeline.decoder)
             return false;
         const queueSize = pipeline.decoder.decodeQueueSize;
@@ -515,6 +598,9 @@
         }
     }
     function processOutputQueue(pipeline) {
+        if (frameTiming.paused) {
+            return;
+        }
         // Check if pipeline is closed (e.g., player destroyed) - clean up queued frames
         if (pipeline.closed) {
             while (pipeline.outputQueue.length > 0) {
@@ -618,7 +704,7 @@
         // Schedule check for when frame should be ready
         return { shouldOutput: false, earliness: -delay, checkDelayMs: Math.max(1, Math.floor(delay)) };
     }
-    function outputFrame(pipeline, entry) {
+    function outputFrame(pipeline, entry, options) {
         if (!pipeline.writer || pipeline.closed) {
             entry.frame.close();
             return;
@@ -631,6 +717,10 @@
         if (pipeline.stats.framesOut <= 3) {
             log(`Output frame ${pipeline.stats.framesOut} for track ${pipeline.idx}: ts=${entry.timestamp}μs`);
         }
+        // Store history for frame stepping (video only)
+        if (pipeline.track.type === 'video' && !(options?.skipHistory)) {
+            pushFrameHistory(pipeline, entry.frame, entry.timestamp);
+        }
         // Write returns a Promise - handle rejection to avoid unhandled promise errors
         // Frame ownership is transferred to the stream, so we don't need to close() on success
         pipeline.writer.write(entry.frame).then(() => {
@@ -639,6 +729,7 @@
                 type: 'sendevent',
                 kind: 'timeupdate',
                 idx: pipeline.idx,
+                time: entry.timestamp / 1e6,
                 uid: uidCounter++,
             };
             self.postMessage(message);
@@ -742,13 +833,14 @@
                                 }
                             };
                             self.addEventListener('message', handler);
-                            // Send frame to main thread
-                            self.postMessage({
+                            // Send frame to main thread (transfer AudioData)
+                            const msg = {
                                 type: 'writeframe',
                                 idx,
                                 frame,
                                 uid: frameUid,
-                            }, [frame]);
+                            };
+                            self.postMessage(msg, { transfer: [frame] });
                         });
                     },
                     close: () => Promise.resolve(),
@@ -763,6 +855,7 @@
                 self.postMessage(message);
                 log(`Set up frame relay for track ${idx} (Safari audio)`);
             }
+            // @ts-ignore - MediaStreamTrackGenerator may not be in standard types
         }
         else if (typeof MediaStreamTrackGenerator !== 'undefined') {
             // Chrome/Edge: use MediaStreamTrackGenerator in worker
@@ -832,12 +925,96 @@
             frameTiming.speed.combined = frameTiming.speed.main * frameTiming.speed.tweak;
             log(`Speed set to ${frameTiming.speed.combined} (main: ${frameTiming.speed.main}, tweak: ${frameTiming.speed.tweak})`);
         }
+        else if (action === 'setPaused') {
+            frameTiming.paused = msg.paused === true;
+            log(`Frame timing paused=${frameTiming.paused}`);
+        }
         else if (action === 'reset') {
             frameTiming.seeking = false;
             log('Frame timing reset (seek complete)');
         }
         sendAck(uid);
     }
+    function handleFrameStep(msg) {
+        const { direction, uid } = msg;
+        log(`FrameStep request dir=${direction} paused=${frameTiming.paused}`);
+        if (!frameTiming.paused) {
+            log(`FrameStep ignored (not paused)`);
+            sendAck(uid);
+            return;
+        }
+        const pipeline = getPrimaryVideoPipeline();
+        if (!pipeline || !pipeline.writer || pipeline.closed) {
+            log(`FrameStep ignored (pipeline missing or closed)`);
+            sendAck(uid);
+            return;
+        }
+        pipeline.frameHistory = pipeline.frameHistory ?? [];
+        if (pipeline.historyCursor === null || pipeline.historyCursor === undefined) {
+            alignHistoryCursorToLastOutput(pipeline);
+        }
+        log(`FrameStep pipeline idx=${pipeline.idx} outQueue=${pipeline.outputQueue.length} history=${pipeline.frameHistory.length} cursor=${pipeline.historyCursor}`);
+        if (direction < 0) {
+            const nextIndex = (pipeline.historyCursor ?? 0) - 1;
+            if (nextIndex < 0 || pipeline.frameHistory.length === 0) {
+                log(`FrameStep back: no history`);
+                sendAck(uid);
+                return;
+            }
+            pipeline.historyCursor = nextIndex;
+            const entry = pipeline.frameHistory[nextIndex];
+            const clone = entry ? cloneVideoFrame(entry.frame) : null;
+            if (!clone) {
+                log(`FrameStep back: failed to clone frame`);
+                sendAck(uid);
+                return;
+            }
+            log(`FrameStep back: output ts=${entry.timestamp}`);
+            outputFrame(pipeline, { frame: clone, timestamp: entry.timestamp, decodedAt: performance.now() }, { skipHistory: true });
+            sendAck(uid);
+            return;
+        }
+        if (direction > 0) {
+            // If we're stepping forward within history (after stepping back), use history
+            const cursor = pipeline.historyCursor;
+            if (cursor !== null && cursor !== undefined && cursor < pipeline.frameHistory.length - 1) {
+                pipeline.historyCursor = cursor + 1;
+                const entry = pipeline.frameHistory[pipeline.historyCursor];
+                const clone = entry ? cloneVideoFrame(entry.frame) : null;
+                if (!clone) {
+                    log(`FrameStep forward: failed to clone frame`);
+                    sendAck(uid);
+                    return;
+                }
+                log(`FrameStep forward (history): output ts=${entry.timestamp}`);
+                outputFrame(pipeline, { frame: clone, timestamp: entry.timestamp, decodedAt: performance.now() }, { skipHistory: true });
+                sendAck(uid);
+                return;
+            }
+            // Otherwise, output the next queued frame
+            if (pipeline.outputQueue.length > 1) {
+                const wasSorted = pipeline.outputQueue.every((entry, i, arr) => i === 0 || arr[i - 1].timestamp <= entry.timestamp);
+                if (!wasSorted) {
+                    pipeline.outputQueue.sort((a, b) => a.timestamp - b.timestamp);
+                }
+            }
+            const lastTs = pipeline.stats.lastOutputTimestamp;
+            let idx = pipeline.outputQueue.findIndex(e => e.timestamp > lastTs);
+            if (idx === -1 && pipeline.outputQueue.length > 0)
+                idx = 0;
+            if (idx === -1) {
+                log(`FrameStep forward: no queued frame available`);
+                sendAck(uid);
+                return;
+            }
+            const entry = pipeline.outputQueue.splice(idx, 1)[0];
+            log(`FrameStep forward (queue): output ts=${entry.timestamp}`);
+            outputFrame(pipeline, entry);
+            sendAck(uid);
+            return;
+        }
+        sendAck(uid);
+    }
     // ============================================================================
     // Cleanup
     // ============================================================================