@livepeer-frameworks/player-core 0.0.3

package/src/players/WebCodecsPlayer/JitterBuffer.ts

@@ -0,0 +1,299 @@
+/**
+ * JitterBuffer - Network Jitter Estimation
+ *
+ * Tracks network jitter to inform buffer sizing decisions.
+ * Ported from legacy rawws.js JitterTracker with improvements:
+ * - Per-track jitter tracking (audio/video can differ)
+ * - TypeScript types
+ * - Better edge case handling
+ *
+ * Algorithm:
+ * 1. Track arrival time vs media time for last N chunks
+ * 2. Calculate jitter = (mediaTimePassed / speed) - clockTimePassed
+ * 3. Maintain sliding window of peak jitter per second
+ * 4. Weighted average: (avgPeak + maxPeak * 2) / 3 + 1ms
+ * 5. Limit lowering rate to prevent oscillation
+ */
+
+import type { JitterState } from './types';
+
+/** Default sliding window size for chunk tracking */
+const DEFAULT_CHUNK_WINDOW = 8;
+
+/** Default sliding window size for peak tracking */
+const DEFAULT_PEAK_WINDOW = 8;
+
+/** Interval between peak calculations (ms) */
+const PEAK_INTERVAL_MS = 1000;
+
+/** Maximum jitter decrease per interval (ms) */
+const MAX_JITTER_DECREASE = 500;
+
+/** Initial jitter estimate (ms) */
+const INITIAL_JITTER = 120;
+
+/** Minimum jitter floor (ms) */
+const MIN_JITTER = 1;
+
+interface ChunkTiming {
+  /** Wall clock time when chunk arrived (performance.now()) */
+  receiveTime: number;
+  /** Media timestamp from chunk (ms) */
+  mediaTime: number;
+}
+
+export interface JitterTrackerOptions {
+  /** Initial jitter estimate (ms) */
+  initialJitter?: number;
+  /** Sliding window size for chunks */
+  chunkWindowSize?: number;
+  /** Sliding window size for peaks */
+  peakWindowSize?: number;
+}
+
+/**
+ * JitterTracker - Estimates network jitter for a single track
+ */
+export class JitterTracker {
+  /** Sliding window of chunk timings */
+  private chunks: ChunkTiming[] = [];
+
+  /** Current playback speed (1 = realtime) */
+  private speed = 1;
+
+  /** Last time a peak was recorded */
+  private lastPeakTime = 0;
+
+  /** Maximum jitter observed in current interval */
+  private currentPeak = 0;
+
+  /** Sliding window of peak jitter values */
+  private peaks: number[] = [];
+
+  /** Weighted average jitter estimate */
+  private maxJitter: number;
+
+  /** Configuration */
+  private readonly chunkWindowSize: number;
+  private readonly peakWindowSize: number;
+
+  constructor(options: JitterTrackerOptions = {}) {
+    this.maxJitter = options.initialJitter ?? INITIAL_JITTER;
+    this.chunkWindowSize = options.chunkWindowSize ?? DEFAULT_CHUNK_WINDOW;
+    this.peakWindowSize = options.peakWindowSize ?? DEFAULT_PEAK_WINDOW;
+  }
+
+  /**
+   * Add a received chunk to jitter calculation
+   *
+   * @param mediaTime - Media timestamp from chunk (ms)
+   * @param receiveTime - Wall clock time (performance.now())
+   */
+  addChunk(mediaTime: number, receiveTime: number = performance.now()): void {
+    // Add to sliding window
+    this.chunks.push({ receiveTime, mediaTime });
+    if (this.chunks.length > this.chunkWindowSize) {
+      this.chunks.shift();
+    }
+
+    // Calculate instantaneous jitter
+    const jitter = this.calculateJitter();
+    if (jitter > this.currentPeak) {
+      this.currentPeak = jitter;
+    }
+
+    // Update peaks every second
+    const now = performance.now();
+    if (now > this.lastPeakTime + PEAK_INTERVAL_MS) {
+      this.recordPeak();
+      this.lastPeakTime = now;
+    }
+  }
+
+  /**
+   * Calculate current instantaneous jitter
+   */
+  private calculateJitter(): number {
+    if (this.chunks.length <= 1) {
+      return 0;
+    }
+
+    // Skip calculation during fast-forward
+    if (this.speed === 0 || !isFinite(this.speed)) {
+      return 0;
+    }
+
+    const oldest = this.chunks[0];
+    const newest = this.chunks[this.chunks.length - 1];
+
+    // Time passed on wall clock
+    const clockTimePassed = newest.receiveTime - oldest.receiveTime;
+
+    // Time that should have passed based on media timestamps
+    const mediaTimePassed = newest.mediaTime - oldest.mediaTime;
+
+    // Jitter = expected - actual
+    // Positive jitter means chunks arriving faster than expected (buffering)
+    // Negative jitter means chunks arriving slower than expected (starving)
+    const jitter = mediaTimePassed / this.speed - clockTimePassed;
+
+    return Math.max(0, jitter);
+  }
+
+  /**
+   * Record current peak and update weighted average
+   */
+  private recordPeak(): void {
+    // Add current peak to sliding window
+    this.peaks.push(this.currentPeak);
+    if (this.peaks.length > this.peakWindowSize) {
+      this.peaks.shift();
+    }
+
+    // Reset for next interval
+    this.currentPeak = 0;
+
+    // Calculate new weighted average
+    if (this.peaks.length > 0) {
+      const maxPeak = Math.max(...this.peaks);
+      const avgPeak =
+        this.peaks.reduce((sum, p) => sum + p, 0) / this.peaks.length;
+
+      // Weighted: emphasize max peak for safety
+      let weighted = (avgPeak + maxPeak * 2) / 3 + MIN_JITTER;
+
+      // Limit rate of decrease to prevent oscillation
+      if (this.maxJitter > weighted + MAX_JITTER_DECREASE) {
+        weighted = this.maxJitter - MAX_JITTER_DECREASE;
+      }
+
+      // Smooth transition
+      this.maxJitter = (this.maxJitter + weighted) / 2;
+    }
+  }
+
+  /**
+   * Get current jitter estimate (ms)
+   */
+  get(): number {
+    return this.maxJitter;
+  }
+
+  /**
+   * Get detailed jitter state
+   */
+  getState(): JitterState {
+    return {
+      current: this.calculateJitter(),
+      peak: this.currentPeak,
+      weighted: this.maxJitter,
+    };
+  }
+
+  /**
+   * Set playback speed for jitter calculation
+   */
+  setSpeed(speed: number | 'auto'): void {
+    const newSpeed = speed === 'auto' ? 1 : speed;
+    if (newSpeed !== this.speed) {
+      this.speed = newSpeed;
+      this.reset();
+    }
+  }
+
+  /**
+   * Reset jitter tracking (e.g., after seek)
+   */
+  reset(): void {
+    this.chunks = [];
+    this.currentPeak = 0;
+    // Don't reset maxJitter - keep the learned estimate
+  }
+
+  /**
+   * Full reset including learned jitter estimate
+   */
+  fullReset(): void {
+    this.reset();
+    this.peaks = [];
+    this.maxJitter = INITIAL_JITTER;
+    this.lastPeakTime = 0;
+  }
+}
+
+/**
+ * MultiTrackJitterTracker - Manages jitter tracking for multiple tracks
+ */
+export class MultiTrackJitterTracker {
+  private trackers = new Map<number, JitterTracker>();
+  private globalSpeed = 1;
+  private options: JitterTrackerOptions;
+
+  constructor(options: JitterTrackerOptions = {}) {
+    this.options = options;
+  }
+
+  /**
+   * Add a chunk for a specific track
+   */
+  addChunk(trackIndex: number, mediaTime: number, receiveTime?: number): void {
+    let tracker = this.trackers.get(trackIndex);
+    if (!tracker) {
+      tracker = new JitterTracker(this.options);
+      tracker.setSpeed(this.globalSpeed);
+      this.trackers.set(trackIndex, tracker);
+    }
+    tracker.addChunk(mediaTime, receiveTime);
+  }
+
+  /**
+   * Get maximum jitter across all tracks
+   */
+  getMax(): number {
+    let max = 0;
+    for (const tracker of this.trackers.values()) {
+      max = Math.max(max, tracker.get());
+    }
+    return max;
+  }
+
+  /**
+   * Get jitter for a specific track
+   */
+  getForTrack(trackIndex: number): number {
+    return this.trackers.get(trackIndex)?.get() ?? INITIAL_JITTER;
+  }
+
+  /**
+   * Set playback speed for all trackers
+   */
+  setSpeed(speed: number | 'auto'): void {
+    this.globalSpeed = speed === 'auto' ? 1 : speed;
+    for (const tracker of this.trackers.values()) {
+      tracker.setSpeed(speed);
+    }
+  }
+
+  /**
+   * Reset all trackers
+   */
+  reset(): void {
+    for (const tracker of this.trackers.values()) {
+      tracker.reset();
+    }
+  }
+
+  /**
+   * Remove a track's tracker
+   */
+  removeTrack(trackIndex: number): void {
+    this.trackers.delete(trackIndex);
+  }
+
+  /**
+   * Clear all trackers
+   */
+  clear(): void {
+    this.trackers.clear();
+  }
+}

package/src/players/WebCodecsPlayer/LatencyProfiles.ts

@@ -0,0 +1,151 @@
+/**
+ * Latency Profiles for WebCodecs Player
+ *
+ * Presets for trading off latency vs stability.
+ *
+ * Buffer calculation: desiredBuffer = keepAway + serverDelay + (jitter * jitterMultiplier)
+ *
+ * Speed tweaking:
+ *   - If buffer > desired * speedUpThreshold → speed up to maxSpeedUp
+ *   - If buffer < desired * speedDownThreshold → slow down to minSpeedDown
+ */
+
+import type { LatencyProfile, LatencyProfileName } from './types';
+
+/**
+ * Ultra-low latency profile
+ * Target: <200ms end-to-end latency
+ * Use case: Real-time conferencing, remote control
+ * Trade-offs: May stutter on poor networks
+ */
+export const ULTRA_LOW_PROFILE: LatencyProfile = {
+  name: 'Ultra Low Latency',
+  keepAway: 50,              // 50ms base buffer
+  jitterMultiplier: 1.0,     // Full jitter protection (no extra margin)
+  speedUpThreshold: 1.5,     // Speed up when buffer > 150% of desired
+  speedDownThreshold: 0.5,   // Slow down when buffer < 50% of desired
+  maxSpeedUp: 1.08,          // 8% speed up max
+  minSpeedDown: 0.95,        // 5% slow down max
+  audioBufferMs: 100,        // 100ms audio ring buffer
+  optimizeForLatency: true,  // Tell decoders to optimize for latency
+};
+
+/**
+ * Low latency profile
+ * Target: ~300-500ms end-to-end latency
+ * Use case: Live sports, gaming streams
+ * Trade-offs: Balanced latency/stability
+ */
+export const LOW_PROFILE: LatencyProfile = {
+  name: 'Low Latency',
+  keepAway: 100,             // 100ms base buffer
+  jitterMultiplier: 1.2,     // 20% extra jitter protection
+  speedUpThreshold: 2.0,     // Speed up when buffer > 200% of desired
+  speedDownThreshold: 0.6,   // Slow down when buffer < 60% of desired
+  maxSpeedUp: 1.05,          // 5% speed up max (legacy default)
+  minSpeedDown: 0.98,        // 2% slow down max (legacy default)
+  audioBufferMs: 150,        // 150ms audio ring buffer
+  optimizeForLatency: true,
+};
+
+/**
+ * Balanced profile
+ * Target: ~500-1000ms end-to-end latency
+ * Use case: General live streaming
+ * Trade-offs: Prioritizes stability over latency
+ */
+export const BALANCED_PROFILE: LatencyProfile = {
+  name: 'Balanced',
+  keepAway: 200,             // 200ms base buffer
+  jitterMultiplier: 1.5,     // 50% extra jitter protection
+  speedUpThreshold: 2.5,     // Speed up when buffer > 250% of desired
+  speedDownThreshold: 0.5,   // Slow down when buffer < 50% of desired
+  maxSpeedUp: 1.03,          // 3% speed up max
+  minSpeedDown: 0.97,        // 3% slow down max
+  audioBufferMs: 200,        // 200ms audio ring buffer
+  optimizeForLatency: false, // Let decoders optimize for quality
+};
+
+/**
+ * Quality priority profile
+ * Target: ~1-2s end-to-end latency
+ * Use case: VOD, recorded content, poor networks
+ * Trade-offs: Maximum stability, higher latency
+ */
+export const QUALITY_PROFILE: LatencyProfile = {
+  name: 'Quality Priority',
+  keepAway: 500,             // 500ms base buffer (legacy VOD default)
+  jitterMultiplier: 2.0,     // Double jitter protection
+  speedUpThreshold: 3.0,     // Speed up when buffer > 300% of desired
+  speedDownThreshold: 0.4,   // Slow down when buffer < 40% of desired
+  maxSpeedUp: 1.02,          // 2% speed up max
+  minSpeedDown: 0.98,        // 2% slow down max
+  audioBufferMs: 300,        // 300ms audio ring buffer
+  optimizeForLatency: false,
+};
+
+/**
+ * All available latency profiles
+ */
+export const LATENCY_PROFILES: Record<LatencyProfileName, LatencyProfile> = {
+  'ultra-low': ULTRA_LOW_PROFILE,
+  'low': LOW_PROFILE,
+  'balanced': BALANCED_PROFILE,
+  'quality': QUALITY_PROFILE,
+};
+
+/**
+ * Get a latency profile by name
+ * @param name - Profile name
+ * @returns The profile, or 'low' as default
+ */
+export function getLatencyProfile(name?: LatencyProfileName): LatencyProfile {
+  if (name && name in LATENCY_PROFILES) {
+    return LATENCY_PROFILES[name];
+  }
+  return LATENCY_PROFILES['low'];
+}
+
+/**
+ * Merge a custom partial profile with a base profile
+ * @param base - Base profile name or profile object
+ * @param custom - Partial overrides
+ * @returns Merged profile
+ */
+export function mergeLatencyProfile(
+  base: LatencyProfileName | LatencyProfile,
+  custom?: Partial<LatencyProfile>
+): LatencyProfile {
+  const baseProfile = typeof base === 'string' ? getLatencyProfile(base) : base;
+
+  if (!custom) {
+    return baseProfile;
+  }
+
+  return {
+    ...baseProfile,
+    ...custom,
+    name: custom.name ?? `${baseProfile.name} (Custom)`,
+  };
+}
+
+/**
+ * Select appropriate profile based on stream type
+ * @param isLive - Whether the stream is live
+ * @param preferLowLatency - Whether to prefer low latency (e.g., WebRTC source)
+ * @returns Recommended profile name
+ */
+export function selectDefaultProfile(
+  isLive: boolean,
+  preferLowLatency = false
+): LatencyProfileName {
+  if (!isLive) {
+    return 'quality';
+  }
+
+  if (preferLowLatency) {
+    return 'low';
+  }
+
+  return 'balanced';
+}

package/src/players/WebCodecsPlayer/RawChunkParser.ts

@@ -0,0 +1,151 @@
+/**
+ * RawChunkParser - Binary Frame Header Parser
+ *
+ * Parses the 12-byte binary header from MistServer's raw WebSocket stream.
+ *
+ * Header format:
+ *   Byte 0:     Track index (uint8)
+ *   Byte 1:     Chunk type: 0=delta, 1=key, 2=init
+ *   Bytes 2-9:  Timestamp in milliseconds (uint64 big-endian)
+ *   Bytes 10-11: Offset in milliseconds (int16 big-endian, signed)
+ *
+ * The offset is server-calculated and used for A/V synchronization.
+ * Combined presentation time = timestamp + offset
+ */
+
+import type { RawChunk, ChunkType } from './types';
+
+const HEADER_LENGTH = 12;
+
+/**
+ * Parse chunk type byte to string
+ */
+function parseChunkType(typeByte: number): ChunkType {
+  switch (typeByte) {
+    case 1:
+      return 'key';
+    case 2:
+      return 'init';
+    default:
+      return 'delta';
+  }
+}
+
+/**
+ * Parse a raw binary chunk from MistServer
+ *
+ * @param data - ArrayBuffer containing header + frame data
+ * @returns Parsed RawChunk object
+ * @throws Error if data is too short
+ */
+export function parseRawChunk(data: ArrayBuffer): RawChunk {
+  if (data.byteLength < HEADER_LENGTH) {
+    throw new Error(
+      `Invalid chunk: expected at least ${HEADER_LENGTH} bytes, got ${data.byteLength}`
+    );
+  }
+
+  const headerView = new DataView(data, 0, HEADER_LENGTH);
+
+  // Byte 0: track index
+  const trackIndex = headerView.getUint8(0);
+
+  // Byte 1: chunk type
+  const type = parseChunkType(headerView.getUint8(1));
+
+  // Bytes 2-9: timestamp (uint64 big-endian)
+  // getBigUint64 returns BigInt, convert to Number (safe for values < 2^53)
+  const timestampBig = headerView.getBigUint64(2, false); // false = big-endian
+  const timestamp = Number(timestampBig);
+
+  // Bytes 10-11: offset (int16 big-endian, signed)
+  const offset = headerView.getInt16(10, false); // false = big-endian
+
+  // Remaining bytes: actual frame data
+  // Use slice() to create a copy (like reference rawws.js line 462)
+  // This ensures the data survives buffer transfers
+  const frameData = new Uint8Array(data.slice(HEADER_LENGTH));
+
+  return {
+    trackIndex,
+    type,
+    timestamp,
+    offset,
+    data: frameData,
+  };
+}
+
+/**
+ * Calculate the presentation timestamp for a chunk
+ * This combines the server timestamp with the sync offset
+ *
+ * @param chunk - Parsed raw chunk
+ * @returns Presentation timestamp in microseconds (for WebCodecs API)
+ */
+export function getPresentationTimestamp(chunk: RawChunk): number {
+  // timestamp and offset are in milliseconds
+  // WebCodecs expects microseconds
+  return (chunk.timestamp + chunk.offset) * 1000;
+}
+
+/**
+ * Check if this chunk is a keyframe
+ */
+export function isKeyframe(chunk: RawChunk): boolean {
+  return chunk.type === 'key';
+}
+
+/**
+ * Check if this chunk contains codec initialization data
+ */
+export function isInitData(chunk: RawChunk): boolean {
+  return chunk.type === 'init';
+}
+
+/**
+ * Format chunk for debug logging
+ */
+export function formatChunkForLog(chunk: RawChunk): string {
+  const pts = (chunk.timestamp + chunk.offset) / 1000; // seconds
+  const ptsFormatted = pts.toFixed(3);
+  return `[Track ${chunk.trackIndex}] ${chunk.type.toUpperCase()} @ ${ptsFormatted}s (${chunk.data.byteLength} bytes)`;
+}
+
+/**
+ * RawChunkParser class for stateful parsing with validation
+ */
+export class RawChunkParser {
+  private debug: boolean;
+
+  constructor(options: { debug?: boolean } = {}) {
+    this.debug = options.debug ?? false;
+  }
+
+  /**
+   * Parse binary data from WebSocket
+   *
+   * @param data - ArrayBuffer from WebSocket message
+   * @returns Parsed chunk or null if invalid
+   */
+  parse(data: ArrayBuffer): RawChunk | null {
+    try {
+      const chunk = parseRawChunk(data);
+
+      if (this.debug) {
+        console.log('▶️', formatChunkForLog(chunk));
+      }
+
+      return chunk;
+    } catch (err) {
+      console.error('▶️ Failed to parse chunk:', err);
+      return null;
+    }
+  }
+
+  /**
+   * Set debug mode
+   */
+  setDebug(enabled: boolean | 'verbose'): void {
+    this.debug = enabled === 'verbose' || enabled === true;
+  }
+}