avbridge 1.0.0

package/src/strategies/hybrid/index.ts

@@ -0,0 +1,139 @@
+import type { MediaContext, PlaybackSession } from "../../types.js";
+import { VideoRenderer } from "../fallback/video-renderer.js";
+import { AudioOutput } from "../fallback/audio-output.js";
+import { startHybridDecoder, type HybridDecoderHandles } from "./decoder.js";
+
+/**
+ * Hybrid strategy session.
+ *
+ * Uses libav.js for demuxing + WebCodecs VideoDecoder for hardware-accelerated
+ * video decode + libav.js software decode for audio. Same canvas + Web Audio
+ * output as the fallback strategy.
+ *
+ * Falls back to the pure-WASM fallback strategy if WebCodecs fails (via the
+ * onFatalError callback that the player wires to its escalation mechanism).
+ */
+
+const READY_AUDIO_BUFFER_SECONDS = 0.3;
+const READY_TIMEOUT_SECONDS = 10;
+
+export async function createHybridSession(
+  ctx: MediaContext,
+  target: HTMLVideoElement,
+): Promise<PlaybackSession> {
+  // Normalize the source so URL inputs go through the libav HTTP block
+  // reader instead of being buffered into memory.
+  const { normalizeSource } = await import("../../util/source.js");
+  const source = await normalizeSource(ctx.source);
+
+  const fps = ctx.videoTracks[0]?.fps ?? 30;
+  const audio = new AudioOutput();
+  const renderer = new VideoRenderer(target, audio, fps);
+
+  let handles: HybridDecoderHandles;
+  try {
+    handles = await startHybridDecoder({
+      source,
+      filename: ctx.name ?? "input.bin",
+      context: ctx,
+      renderer,
+      audio,
+    });
+  } catch (err) {
+    audio.destroy();
+    renderer.destroy();
+    throw err;
+  }
+
+  // Patch <video> element for the unified player layer
+  Object.defineProperty(target, "currentTime", {
+    configurable: true,
+    get: () => audio.now(),
+    set: (v: number) => { void doSeek(v); },
+  });
+  if (ctx.duration && Number.isFinite(ctx.duration)) {
+    Object.defineProperty(target, "duration", {
+      configurable: true,
+      get: () => ctx.duration ?? NaN,
+    });
+  }
+
+  async function waitForBuffer(): Promise<void> {
+    const start = performance.now();
+    while (true) {
+      const audioReady = audio.isNoAudio() || audio.bufferAhead() >= READY_AUDIO_BUFFER_SECONDS;
+      if (audioReady && renderer.hasFrames()) {
+        return;
+      }
+      if ((performance.now() - start) / 1000 > READY_TIMEOUT_SECONDS) return;
+      await new Promise((r) => setTimeout(r, 50));
+    }
+  }
+
+  async function doSeek(timeSec: number): Promise<void> {
+    const wasPlaying = audio.isPlaying();
+    await audio.pause().catch(() => {});
+    await handles.seek(timeSec).catch((err) =>
+      console.warn("[avbridge] hybrid decoder seek failed:", err),
+    );
+    await audio.reset(timeSec);
+    renderer.flush();
+    if (wasPlaying) {
+      await waitForBuffer();
+      await audio.start();
+    }
+  }
+
+  // Store the fatal error handler so the player can wire escalation
+  let fatalErrorHandler: ((reason: string) => void) | null = null;
+  handles.onFatalError((reason) => fatalErrorHandler?.(reason));
+
+  return {
+    strategy: "hybrid",
+
+    async play() {
+      if (!audio.isPlaying()) {
+        await waitForBuffer();
+        await audio.start();
+      }
+    },
+
+    pause() {
+      void audio.pause();
+    },
+
+    async seek(time) {
+      await doSeek(time);
+    },
+
+    async setAudioTrack(_id) {
+      // Post-MVP for hybrid strategy
+    },
+
+    async setSubtitleTrack(_id) {
+      // Post-MVP for hybrid strategy
+    },
+
+    getCurrentTime() {
+      return audio.now();
+    },
+
+    onFatalError(handler: (reason: string) => void) {
+      fatalErrorHandler = handler;
+    },
+
+    async destroy() {
+      await handles.destroy();
+      renderer.destroy();
+      audio.destroy();
+      try {
+        delete (target as unknown as Record<string, unknown>).currentTime;
+        delete (target as unknown as Record<string, unknown>).duration;
+      } catch { /* ignore */ }
+    },
+
+    getRuntimeStats() {
+      return handles.stats();
+    },
+  };
+}

package/src/strategies/native.ts

@@ -0,0 +1,107 @@
+import type { MediaContext, PlaybackSession } from "../types.js";
+
+/**
+ * Simplest strategy: hand the source to the browser. Works for any
+ * MP4/WebM/MP3/etc. that the user agent already plays.
+ *
+ * The only complexity is that the source might be a `File`/`Blob` (use
+ * `URL.createObjectURL`), an `ArrayBuffer`/`Uint8Array` (wrap in a Blob first),
+ * or a string URL (assign directly).
+ */
+export async function createNativeSession(
+  context: MediaContext,
+  video: HTMLVideoElement,
+): Promise<PlaybackSession> {
+  const { url, revoke } = sourceToVideoUrl(context.source);
+  video.src = url;
+
+  // Wait for metadata so the player resolves only once playback is actually
+  // ready. We expose errors via the player's "error" event, not by throwing
+  // here, because failure here often means we should escalate to remux.
+  await new Promise<void>((resolve, reject) => {
+    const onMeta = () => {
+      cleanup();
+      resolve();
+    };
+    const onError = () => {
+      cleanup();
+      reject(new Error(`<video> failed to load: ${video.error?.message ?? "unknown"}`));
+    };
+    const cleanup = () => {
+      video.removeEventListener("loadedmetadata", onMeta);
+      video.removeEventListener("error", onError);
+    };
+    video.addEventListener("loadedmetadata", onMeta);
+    video.addEventListener("error", onError);
+  });
+
+  let stats = { framesDecoded: 0, framesDropped: 0 };
+
+  return {
+    strategy: "native",
+    async play() {
+      await video.play();
+    },
+    pause() {
+      video.pause();
+    },
+    async seek(time) {
+      video.currentTime = time;
+    },
+    async setAudioTrack(id) {
+      // HTMLMediaElement.audioTracks is not exposed in all browsers, so we
+      // try-catch and no-op if not available.
+      const tracks = (video as unknown as { audioTracks?: { length: number; [i: number]: { id: string; enabled: boolean } } }).audioTracks;
+      if (!tracks) return;
+      for (let i = 0; i < tracks.length; i++) {
+        tracks[i].enabled = tracks[i].id === String(id) || i === id;
+      }
+    },
+    async setSubtitleTrack(id) {
+      const tracks = video.textTracks;
+      for (let i = 0; i < tracks.length; i++) {
+        tracks[i].mode = i === id ? "showing" : "disabled";
+      }
+    },
+    async destroy() {
+      video.pause();
+      video.removeAttribute("src");
+      video.load();
+      revoke?.();
+    },
+    getCurrentTime() {
+      return video.currentTime || 0;
+    },
+    getRuntimeStats() {
+      // getVideoPlaybackQuality is the standard hook; not all UAs implement it.
+      const q = (video as unknown as { getVideoPlaybackQuality?: () => VideoPlaybackQuality }).getVideoPlaybackQuality?.();
+      if (q) {
+        stats = {
+          framesDecoded: q.totalVideoFrames,
+          framesDropped: q.droppedVideoFrames,
+        };
+      }
+      return { ...stats, decoderType: "native" };
+    },
+  };
+}
+
+function sourceToVideoUrl(source: unknown): { url: string; revoke?: () => void } {
+  if (source instanceof Blob) {
+    const url = URL.createObjectURL(source);
+    return { url, revoke: () => URL.revokeObjectURL(url) };
+  }
+  if (source instanceof ArrayBuffer || source instanceof Uint8Array) {
+    const blob = new Blob([source as BlobPart]);
+    const url = URL.createObjectURL(blob);
+    return { url, revoke: () => URL.revokeObjectURL(url) };
+  }
+  if (typeof source === "string") return { url: source };
+  if (source instanceof URL) return { url: source.toString() };
+  throw new TypeError("native strategy: unsupported source type");
+}
+
+interface VideoPlaybackQuality {
+  totalVideoFrames: number;
+  droppedVideoFrames: number;
+}

package/src/strategies/remux/annexb.ts

@@ -0,0 +1,112 @@
+/**
+ * H.264/HEVC bitstream conversion helpers.
+ *
+ * Demuxers from MP4-family containers (mediabunny) hand us packets in **AVCC**
+ * format: each NAL unit prefixed with a 4-byte big-endian length.
+ *
+ * Demuxers from elementary-stream/AVI/TS hand us **Annex B**: NAL units
+ * separated by `00 00 00 01` (or `00 00 01`) start codes.
+ *
+ * MSE expects AVCC inside fragmented MP4. So when the source side emits Annex
+ * B, we need to convert before muxing. Going the other way (AVCC → Annex B) is
+ * useful for feeding `VideoDecoder` configured with `description` omitted.
+ */
+
+const START_CODE_4 = new Uint8Array([0, 0, 0, 1]);
+
+/** True if the bytes look like Annex B (start with `00 00 00 01` or `00 00 01`). */
+export function isAnnexB(bytes: Uint8Array): boolean {
+  if (bytes.length < 3) return false;
+  if (bytes[0] === 0 && bytes[1] === 0 && bytes[2] === 1) return true;
+  if (bytes.length >= 4 && bytes[0] === 0 && bytes[1] === 0 && bytes[2] === 0 && bytes[3] === 1) return true;
+  return false;
+}
+
+/**
+ * Walk an Annex B byte stream and yield each NAL unit (without start code).
+ * This is the standard byte-by-byte scan; no SIMD tricks because the typical
+ * frame is small.
+ */
+export function* iterateAnnexBNalus(bytes: Uint8Array): Generator<Uint8Array> {
+  const length = bytes.length;
+  let i = 0;
+  let nalStart = -1;
+
+  while (i < length) {
+    // Look for start code at position i
+    let scLen = 0;
+    if (i + 3 < length && bytes[i] === 0 && bytes[i + 1] === 0 && bytes[i + 2] === 0 && bytes[i + 3] === 1) {
+      scLen = 4;
+    } else if (i + 2 < length && bytes[i] === 0 && bytes[i + 1] === 0 && bytes[i + 2] === 1) {
+      scLen = 3;
+    }
+
+    if (scLen > 0) {
+      if (nalStart >= 0) {
+        yield bytes.subarray(nalStart, i);
+      }
+      nalStart = i + scLen;
+      i += scLen;
+    } else {
+      i += 1;
+    }
+  }
+
+  if (nalStart >= 0 && nalStart < length) {
+    yield bytes.subarray(nalStart, length);
+  }
+}
+
+/**
+ * Convert an Annex B byte stream to AVCC. Each NALU is prefixed with its
+ * 4-byte big-endian length.
+ */
+export function annexBToAvcc(annexB: Uint8Array): Uint8Array {
+  const nalus: Uint8Array[] = [];
+  let total = 0;
+  for (const nal of iterateAnnexBNalus(annexB)) {
+    nalus.push(nal);
+    total += 4 + nal.length;
+  }
+  const out = new Uint8Array(total);
+  let off = 0;
+  for (const nal of nalus) {
+    const len = nal.length;
+    out[off++] = (len >>> 24) & 0xff;
+    out[off++] = (len >>> 16) & 0xff;
+    out[off++] = (len >>> 8) & 0xff;
+    out[off++] = len & 0xff;
+    out.set(nal, off);
+    off += len;
+  }
+  return out;
+}
+
+/**
+ * Convert AVCC (4-byte length-prefixed) NALUs to Annex B. Each NALU is
+ * prefixed with `00 00 00 01`.
+ */
+export function avccToAnnexB(avcc: Uint8Array): Uint8Array {
+  const out: Uint8Array[] = [];
+  let total = 0;
+  let i = 0;
+  while (i + 4 <= avcc.length) {
+    const len =
+      (avcc[i] << 24) | (avcc[i + 1] << 16) | (avcc[i + 2] << 8) | avcc[i + 3];
+    i += 4;
+    if (i + len > avcc.length) {
+      throw new Error(`avccToAnnexB: NAL length ${len} overflows buffer at offset ${i}`);
+    }
+    out.push(START_CODE_4);
+    out.push(avcc.subarray(i, i + len));
+    total += 4 + len;
+    i += len;
+  }
+  const merged = new Uint8Array(total);
+  let off = 0;
+  for (const chunk of out) {
+    merged.set(chunk, off);
+    off += chunk.length;
+  }
+  return merged;
+}

package/src/strategies/remux/index.ts

@@ -0,0 +1,79 @@
+import type { MediaContext, PlaybackSession } from "../../types.js";
+import { createRemuxPipeline, type RemuxPipeline } from "./pipeline.js";
+
+/**
+ * Strategy entry: build the remux pipeline, then expose a {@link PlaybackSession}
+ * that delegates to the underlying `<video>` element for playback control and
+ * to the pipeline for source-side seek invalidation.
+ */
+export async function createRemuxSession(
+  context: MediaContext,
+  video: HTMLVideoElement,
+): Promise<PlaybackSession> {
+  let pipeline: RemuxPipeline;
+  try {
+    pipeline = await createRemuxPipeline(context, video);
+  } catch (err) {
+    throw new Error(
+      `remux strategy failed to start: ${(err as Error).message}. The container or codec combination is not supported by mediabunny + MSE on this browser.`,
+    );
+  }
+
+  // Don't pump yet — wait for the first play() or seek() to start from the
+  // right position. The player's strategy-switch flow calls seek(currentTime)
+  // immediately after creation, so pumping from 0 here would be wasted work.
+  let started = false;
+  let wantPlay = false;
+
+  return {
+    strategy: "remux",
+    async play() {
+      wantPlay = true;
+      if (!started) {
+        // First play — start the pump. The deferred seek in MseSink will
+        // call video.play() once data is available (via autoPlay flag).
+        started = true;
+        await pipeline.start(video.currentTime || 0, true);
+        return;
+      }
+      await video.play();
+    },
+    pause() {
+      wantPlay = false;
+      video.pause();
+    },
+    async seek(time) {
+      if (!started) {
+        started = true;
+        // autoPlay=true so playback starts as soon as data arrives at
+        // the seek target (handles the strategy-switch case where play()
+        // is called right after seek()).
+        await pipeline.seek(time, wantPlay);
+        return;
+      }
+      const wasPlaying = !video.paused;
+      await pipeline.seek(time, wasPlaying || wantPlay);
+    },
+    async setAudioTrack(_id) {
+      // v1: single-track output. Multi-audio remuxing is post-MVP.
+    },
+    async setSubtitleTrack(id) {
+      const tracks = video.textTracks;
+      for (let i = 0; i < tracks.length; i++) {
+        tracks[i].mode = i === id ? "showing" : "disabled";
+      }
+    },
+    getCurrentTime() {
+      return video.currentTime || 0;
+    },
+    async destroy() {
+      video.pause();
+      await pipeline.destroy();
+      video.removeAttribute("src");
+      video.load();
+    },
+    getRuntimeStats() {
+      return pipeline.stats();
+    },
+  };
+}

package/src/strategies/remux/mse.ts

@@ -0,0 +1,234 @@
+/**
+ * MediaSource Extensions plumbing. Wraps a `MediaSource` + single
+ * `SourceBuffer` with an append queue that respects `updateend` backpressure.
+ */
+
+export interface MseSinkOptions {
+  mime: string;
+  video: HTMLVideoElement;
+  /** Called once the MediaSource is open and ready for appends. */
+  onReady?: () => void;
+}
+
+export class MseSink {
+  private mediaSource: MediaSource;
+  private sourceBuffer: SourceBuffer | null = null;
+  private queue: ArrayBuffer[] = [];
+  private endOfStreamCalled = false;
+  private destroyed = false;
+  private readyPromise: Promise<void>;
+  private resolveReady!: () => void;
+  private rejectReady!: (err: Error) => void;
+  private objectUrl: string;
+
+  constructor(private readonly options: MseSinkOptions) {
+    if (typeof MediaSource === "undefined") {
+      throw new Error("MSE not supported in this environment");
+    }
+    if (!MediaSource.isTypeSupported(options.mime)) {
+      throw new Error(`MSE does not support MIME "${options.mime}" — cannot remux`);
+    }
+
+    this.mediaSource = new MediaSource();
+    this.objectUrl = URL.createObjectURL(this.mediaSource);
+    options.video.src = this.objectUrl;
+
+    this.readyPromise = new Promise((resolve, reject) => {
+      this.resolveReady = resolve;
+      this.rejectReady = reject;
+    });
+
+    this.mediaSource.addEventListener("sourceopen", () => {
+      try {
+        this.sourceBuffer = this.mediaSource.addSourceBuffer(options.mime);
+        this.sourceBuffer.mode = "segments";
+        this.sourceBuffer.addEventListener("updateend", () => this.pump());
+        this.resolveReady();
+        options.onReady?.();
+      } catch (err) {
+        this.rejectReady(err instanceof Error ? err : new Error(String(err)));
+      }
+    });
+  }
+
+  ready(): Promise<void> {
+    return this.readyPromise;
+  }
+
+  /** Queue a chunk of fMP4 bytes (init segment or media segment). */
+  append(chunk: ArrayBuffer | Uint8Array): void {
+    if (this.destroyed) return;
+    const ab = chunk instanceof Uint8Array
+      ? (chunk.buffer.slice(chunk.byteOffset, chunk.byteOffset + chunk.byteLength) as ArrayBuffer)
+      : chunk;
+    this.queue.push(ab);
+    this.pump();
+  }
+
+  private pump(): void {
+    const sb = this.sourceBuffer;
+    if (!sb || sb.updating) return;
+
+    // Apply deferred actions once the SourceBuffer has any data. Deferred
+    // seek and deferred autoplay are independent — both can fire here, or
+    // either alone. Setting `currentTime` before data exists causes the
+    // browser to snap back to the nearest buffered range; calling `play()`
+    // before data exists puts the video into a stuck waiting state.
+    if (sb.buffered.length > 0) {
+      if (this.pendingSeekTime !== null) {
+        this.options.video.currentTime = this.pendingSeekTime;
+        this.pendingSeekTime = null;
+      } else if (!this.hasSnappedToFirstBuffered) {
+        // First data arrival with no pending seek. If currentTime is
+        // outside the first buffered range (typical for MPEG-TS sources
+        // whose PTS doesn't start at 0), snap into the buffered range
+        // so the video element doesn't wait forever for nonexistent data.
+        const v = this.options.video;
+        const firstStart = sb.buffered.start(0);
+        const firstEnd = sb.buffered.end(0);
+        if (v.currentTime < firstStart || v.currentTime > firstEnd) {
+          v.currentTime = firstStart;
+        }
+        this.hasSnappedToFirstBuffered = true;
+      }
+      if (this.playOnSeek) {
+        this.playOnSeek = false;
+        this.options.video.play().catch(() => { /* ignore — autoplay may be blocked */ });
+      }
+    }
+
+    const next = this.queue.shift();
+    if (!next) return;
+    try {
+      sb.appendBuffer(next);
+    } catch (err) {
+      // QuotaExceededError → evict the oldest few seconds and retry once.
+      if ((err as DOMException).name === "QuotaExceededError") {
+        this.evict();
+        try {
+          sb.appendBuffer(next);
+          return;
+        } catch {
+          /* fall through to error */
+        }
+      }
+      this.rejectReady(err instanceof Error ? err : new Error(String(err)));
+    }
+  }
+
+  private evict(): void {
+    const sb = this.sourceBuffer;
+    if (!sb || sb.buffered.length === 0) return;
+    const start = sb.buffered.start(0);
+    const current = this.options.video.currentTime;
+    // Drop everything that's at least 10s behind the current position.
+    if (current - start > 10) {
+      try {
+        sb.remove(start, current - 10);
+      } catch {
+        /* ignore */
+      }
+    }
+  }
+
+  /** Indicate the source is finished. Future seeks past the end will fail. */
+  endOfStream(): void {
+    if (this.endOfStreamCalled || this.destroyed) return;
+    this.endOfStreamCalled = true;
+    const tryEnd = () => {
+      if (this.queue.length > 0 || this.sourceBuffer?.updating) {
+        // Wait for the queue to drain.
+        this.sourceBuffer?.addEventListener("updateend", tryEnd, { once: true });
+        return;
+      }
+      try {
+        if (this.mediaSource.readyState === "open") {
+          this.mediaSource.endOfStream();
+        }
+      } catch {
+        /* ignore */
+      }
+    };
+    tryEnd();
+  }
+
+  /** Seconds of media buffered ahead of the current playback position. */
+  bufferedAhead(): number {
+    const sb = this.sourceBuffer;
+    if (!sb || sb.buffered.length === 0) return 0;
+    const current = this.options.video.currentTime;
+    for (let i = 0; i < sb.buffered.length; i++) {
+      if (sb.buffered.start(i) <= current && sb.buffered.end(i) > current) {
+        return sb.buffered.end(i) - current;
+      }
+    }
+    return 0;
+  }
+
+  /** Total seconds of media buffered across all ranges. */
+  totalBuffered(): number {
+    const sb = this.sourceBuffer;
+    if (!sb || sb.buffered.length === 0) return 0;
+    let total = 0;
+    for (let i = 0; i < sb.buffered.length; i++) {
+      total += sb.buffered.end(i) - sb.buffered.start(i);
+    }
+    return total;
+  }
+
+  /** Number of chunks waiting in the append queue. */
+  queueLength(): number {
+    return this.queue.length;
+  }
+
+  /** Time to seek to once the SourceBuffer has data at this position. */
+  private pendingSeekTime: number | null = null;
+  /** Whether to resume playback after the deferred seek completes. */
+  private playOnSeek = false;
+  /**
+   * On the very first data arrival, if `currentTime` falls outside the first
+   * buffered range, snap it to the start of that range. MPEG-TS sources
+   * commonly start their PTS at a non-zero value (e.g. ~1.5s); without this
+   * snap, the video element sits at `currentTime=0` waiting forever for
+   * data that doesn't exist.
+   */
+  private hasSnappedToFirstBuffered = false;
+
+  /** Request that playback resumes automatically once the deferred seek fires. */
+  setPlayOnSeek(play: boolean): void {
+    this.playOnSeek = play;
+  }
+
+  /**
+   * Discard all buffered media and schedule a deferred seek. The actual
+   * `video.currentTime` assignment happens in `pump()` once the SourceBuffer
+   * has data at the target position — setting it earlier causes the browser
+   * to snap back to the nearest buffered range.
+   */
+  invalidate(seekTime: number): void {
+    const sb = this.sourceBuffer;
+    // Clear the pending queue — stale fragments from the old pump position.
+    this.queue = [];
+    this.pendingSeekTime = seekTime;
+    this.hasSnappedToFirstBuffered = true; // explicit seek overrides the auto-snap
+    if (!sb || sb.buffered.length === 0) return;
+    try {
+      const start = sb.buffered.start(0);
+      const end = sb.buffered.end(sb.buffered.length - 1);
+      sb.remove(start, end);
+    } catch {
+      /* ignore — sourcebuffer may be in updating state */
+    }
+  }
+
+  destroy(): void {
+    this.destroyed = true;
+    this.queue = [];
+    try {
+      if (this.mediaSource.readyState === "open") this.mediaSource.endOfStream();
+    } catch {
+      /* ignore */
+    }
+    URL.revokeObjectURL(this.objectUrl);
+  }
+}