avbridge 2.3.0 → 2.6.0

package/src/strategies/fallback/decoder.ts

@@ -29,11 +29,22 @@ import { AudioOutput } from "./audio-output.js";
 import type { MediaContext } from "../../types.js";
 import { pickLibavVariant } from "./variant-routing.js";
 import { dbg } from "../../util/debug.js";
+import {
+  sanitizeFrameTimestamp,
+  libavFrameToInterleavedFloat32,
+} from "../../util/libav-demux.js";
 
 export interface DecoderHandles {
   destroy(): Promise<void>;
   /** Seek to the given time in seconds. Returns once the new pump has been kicked off. */
   seek(timeSec: number): Promise<void>;
+  /**
+   * Switch the active audio track. The decoder tears down the current audio
+   * decoder, initializes one for the stream whose container id matches
+   * `trackId` (== libav `stream.index`), seeks the demuxer to `timeSec`, and
+   * restarts the pump. No-op if the track is already active.
+   */
+  setAudioTrack(trackId: number, timeSec: number): Promise<void>;
   stats(): Record<string, unknown>;
 }
 
@@ -63,7 +74,15 @@ export async function startDecoder(opts: StartDecoderOptions): Promise<DecoderHa
 
   const [fmt_ctx, streams] = await libav.ff_init_demuxer_file(opts.filename);
   const videoStream = streams.find((s) => s.codec_type === libav.AVMEDIA_TYPE_VIDEO) ?? null;
-  const audioStream = streams.find((s) => s.codec_type === libav.AVMEDIA_TYPE_AUDIO) ?? null;
+  // Audio stream is mutable so setAudioTrack() can swap it. Default to the
+  // track the context picked first (matches probe ordering). We resolve by
+  // container id so the selection survives stream reordering.
+  const firstAudioTrackId = opts.context.audioTracks[0]?.id;
+  let audioStream: LibavStream | null =
+    (firstAudioTrackId != null
+      ? streams.find((s) => s.codec_type === libav.AVMEDIA_TYPE_AUDIO && s.index === firstAudioTrackId)
+      : undefined) ??
+    streams.find((s) => s.codec_type === libav.AVMEDIA_TYPE_AUDIO) ?? null;
 
   if (!videoStream && !audioStream) {
     throw new Error("fallback decoder: file has no decodable streams");
@@ -376,7 +395,7 @@ export async function startDecoder(opts: StartDecoderOptions): Promise<DecoderHa
 
     for (const f of frames) {
       if (myToken !== pumpToken || destroyed) return;
-      const bridgeOpts = sanitizeFrameTimestamp(
+      sanitizeFrameTimestamp(
         f,
         () => {
           const ts = syntheticVideoUs;
@@ -385,8 +404,10 @@ export async function startDecoder(opts: StartDecoderOptions): Promise<DecoderHa
         },
         videoTimeBase,
       );
+      // sanitizeFrameTimestamp normalizes pts to µs, so the bridge can
+      // always use the 1/1e6 timebase.
       try {
-        const vf = bridge.laFrameToVideoFrame(f, bridgeOpts);
+        const vf = bridge.laFrameToVideoFrame(f, { timeBase: [1, 1_000_000] });
         opts.renderer.enqueue(vf);
         videoFramesDecoded++;
       } catch (err) {
@@ -455,6 +476,78 @@ export async function startDecoder(opts: StartDecoderOptions): Promise<DecoderHa
       try { await inputHandle.detach(); } catch { /* ignore */ }
     },
 
+    async setAudioTrack(trackId, timeSec) {
+      if (audioStream && audioStream.index === trackId) return;
+      const newStream = streams.find(
+        (s) => s.codec_type === libav.AVMEDIA_TYPE_AUDIO && s.index === trackId,
+      );
+      if (!newStream) {
+        console.warn("[avbridge] fallback: setAudioTrack — no stream with id", trackId);
+        return;
+      }
+
+      // Stop the pump before touching libav state. Same discipline as seek().
+      const newToken = ++pumpToken;
+      if (pumpRunning) {
+        try { await pumpRunning; } catch { /* ignore */ }
+      }
+      if (destroyed) return;
+
+      // Tear down the old audio decoder and init a fresh one for the new stream.
+      if (audioDec) {
+        try { await libav.ff_free_decoder?.(audioDec.c, audioDec.pkt, audioDec.frame); } catch { /* ignore */ }
+        audioDec = null;
+      }
+      try {
+        const [, c, pkt, frame] = await libav.ff_init_decoder(newStream.codec_id, {
+          codecpar: newStream.codecpar,
+        });
+        audioDec = { c, pkt, frame };
+        audioTimeBase = newStream.time_base_num && newStream.time_base_den
+          ? [newStream.time_base_num, newStream.time_base_den]
+          : undefined;
+      } catch (err) {
+        console.warn(
+          "[avbridge] fallback: setAudioTrack init failed — falling back to no-audio mode:",
+          (err as Error).message,
+        );
+        audioDec = null;
+        opts.audio.setNoAudio();
+      }
+
+      audioStream = newStream;
+
+      // Re-seek so packets resume from the user's current position for the
+      // new track (and the same video position).
+      try {
+        const tsUs = Math.floor(timeSec * 1_000_000);
+        const [tsLo, tsHi] = libav.f64toi64
+          ? libav.f64toi64(tsUs)
+          : [tsUs | 0, Math.floor(tsUs / 0x100000000)];
+        await libav.av_seek_frame(
+          fmt_ctx,
+          -1,
+          tsLo,
+          tsHi,
+          libav.AVSEEK_FLAG_BACKWARD ?? 0,
+        );
+      } catch (err) {
+        console.warn("[avbridge] fallback: setAudioTrack seek failed:", err);
+      }
+
+      // Flush the video decoder too — we just moved the demuxer back to a
+      // keyframe boundary.
+      try { if (videoDec) await libav.avcodec_flush_buffers?.(videoDec.c); } catch { /* ignore */ }
+      await flushBSF();
+
+      syntheticVideoUs = Math.round(timeSec * 1_000_000);
+      syntheticAudioUs = Math.round(timeSec * 1_000_000);
+
+      pumpRunning = pumpLoop(newToken).catch((err) =>
+        console.error("[avbridge] fallback pump failed (post-setAudioTrack):", err),
+      );
+    },
+
     async seek(timeSec) {
       // Cancel the current pump and wait for it to actually exit before
       // we start moving file pointers around — concurrent ff_decode_multi
@@ -537,176 +630,6 @@ export async function startDecoder(opts: StartDecoderOptions): Promise<DecoderHa
   };
 }
 
-// ─────────────────────────────────────────────────────────────────────────────
-// Frame timestamp sanitizer.
-//
-// libav can hand back decoded frames with `pts = AV_NOPTS_VALUE` (encoded as
-// ptshi = -2147483648, pts = 0) for inputs whose demuxer can't determine
-// presentation times. AVI is the canonical example. The bridge's
-// `laFrameToVideoFrame` then multiplies pts × 1e6 × tbNum / tbDen and
-// overflows int64, throwing "Value is outside the 'long long' value range".
-//
-// Fix: replace any invalid pts with a synthetic microsecond counter, force
-// the frame's pts/ptshi to that value, and tell the bridge to use a 1/1e6
-// timebase so it does an identity conversion.
-// ─────────────────────────────────────────────────────────────────────────────
-
-interface BridgeOpts {
-  timeBase?: [number, number];
-  transfer?: boolean;
-}
-
-function sanitizeFrameTimestamp(
-  frame: LibavFrame,
-  nextUs: () => number,
-  fallbackTimeBase?: [number, number],
-): BridgeOpts {
-  const lo = frame.pts ?? 0;
-  const hi = frame.ptshi ?? 0;
-  const isInvalid = (hi === -2147483648 && lo === 0) || !Number.isFinite(lo);
-  if (isInvalid) {
-    const us = nextUs();
-    frame.pts = us;
-    frame.ptshi = 0;
-    return { timeBase: [1, 1_000_000] };
-  }
-  const tb = fallbackTimeBase ?? [1, 1_000_000];
-  const pts64 = hi * 0x100000000 + lo;
-  const us = Math.round((pts64 * 1_000_000 * tb[0]) / tb[1]);
-  if (Number.isFinite(us) && Math.abs(us) <= Number.MAX_SAFE_INTEGER) {
-    frame.pts = us;
-    frame.ptshi = us < 0 ? -1 : 0;
-    return { timeBase: [1, 1_000_000] };
-  }
-  const fallback = nextUs();
-  frame.pts = fallback;
-  frame.ptshi = 0;
-  return { timeBase: [1, 1_000_000] };
-}
-
-// ─────────────────────────────────────────────────────────────────────────────
-// libav decoded `Frame` → interleaved Float32Array (the format AudioOutput
-// schedules).
-// ─────────────────────────────────────────────────────────────────────────────
-
-const AV_SAMPLE_FMT_U8 = 0;
-const AV_SAMPLE_FMT_S16 = 1;
-const AV_SAMPLE_FMT_S32 = 2;
-const AV_SAMPLE_FMT_FLT = 3;
-const AV_SAMPLE_FMT_U8P = 5;
-const AV_SAMPLE_FMT_S16P = 6;
-const AV_SAMPLE_FMT_S32P = 7;
-const AV_SAMPLE_FMT_FLTP = 8;
-
-interface InterleavedSamples {
-  data: Float32Array;
-  channels: number;
-  sampleRate: number;
-}
-
-function libavFrameToInterleavedFloat32(frame: LibavFrame): InterleavedSamples | null {
-  const channels = frame.channels ?? frame.ch_layout_nb_channels ?? 1;
-  const sampleRate = frame.sample_rate ?? 44100;
-  const nbSamples = frame.nb_samples ?? 0;
-  if (nbSamples === 0) return null;
-
-  const out = new Float32Array(nbSamples * channels);
-
-  switch (frame.format) {
-    case AV_SAMPLE_FMT_FLTP: {
-      const planes = ensurePlanes(frame.data, channels);
-      for (let ch = 0; ch < channels; ch++) {
-        const plane = asFloat32(planes[ch]);
-        for (let i = 0; i < nbSamples; i++) out[i * channels + ch] = plane[i];
-      }
-      return { data: out, channels, sampleRate };
-    }
-    case AV_SAMPLE_FMT_FLT: {
-      const flat = asFloat32(frame.data);
-      for (let i = 0; i < nbSamples * channels; i++) out[i] = flat[i];
-      return { data: out, channels, sampleRate };
-    }
-    case AV_SAMPLE_FMT_S16P: {
-      const planes = ensurePlanes(frame.data, channels);
-      for (let ch = 0; ch < channels; ch++) {
-        const plane = asInt16(planes[ch]);
-        for (let i = 0; i < nbSamples; i++) out[i * channels + ch] = plane[i] / 32768;
-      }
-      return { data: out, channels, sampleRate };
-    }
-    case AV_SAMPLE_FMT_S16: {
-      const flat = asInt16(frame.data);
-      for (let i = 0; i < nbSamples * channels; i++) out[i] = flat[i] / 32768;
-      return { data: out, channels, sampleRate };
-    }
-    case AV_SAMPLE_FMT_S32P: {
-      const planes = ensurePlanes(frame.data, channels);
-      for (let ch = 0; ch < channels; ch++) {
-        const plane = asInt32(planes[ch]);
-        for (let i = 0; i < nbSamples; i++) out[i * channels + ch] = plane[i] / 2147483648;
-      }
-      return { data: out, channels, sampleRate };
-    }
-    case AV_SAMPLE_FMT_S32: {
-      const flat = asInt32(frame.data);
-      for (let i = 0; i < nbSamples * channels; i++) out[i] = flat[i] / 2147483648;
-      return { data: out, channels, sampleRate };
-    }
-    case AV_SAMPLE_FMT_U8P: {
-      const planes = ensurePlanes(frame.data, channels);
-      for (let ch = 0; ch < channels; ch++) {
-        const plane = asUint8(planes[ch]);
-        for (let i = 0; i < nbSamples; i++) out[i * channels + ch] = (plane[i] - 128) / 128;
-      }
-      return { data: out, channels, sampleRate };
-    }
-    case AV_SAMPLE_FMT_U8: {
-      const flat = asUint8(frame.data);
-      for (let i = 0; i < nbSamples * channels; i++) out[i] = (flat[i] - 128) / 128;
-      return { data: out, channels, sampleRate };
-    }
-    default:
-      if (!(globalThis as { __avbridgeLoggedSampleFmt?: number }).__avbridgeLoggedSampleFmt) {
-        (globalThis as { __avbridgeLoggedSampleFmt?: number }).__avbridgeLoggedSampleFmt = frame.format;
-        console.warn(`[avbridge] unsupported audio sample format from libav: ${frame.format}`);
-      }
-      return null;
-  }
-}
-
-function ensurePlanes(data: unknown, channels: number): unknown[] {
-  if (Array.isArray(data)) return data;
-  const arr = data as { length: number; subarray?: (a: number, b: number) => unknown };
-  const len = arr.length;
-  const perChannel = Math.floor(len / channels);
-  const planes: unknown[] = [];
-  for (let ch = 0; ch < channels; ch++) {
-    planes.push(arr.subarray ? arr.subarray(ch * perChannel, (ch + 1) * perChannel) : arr);
-  }
-  return planes;
-}
-
-function asFloat32(x: unknown): Float32Array {
-  if (x instanceof Float32Array) return x;
-  const ta = x as { buffer: ArrayBuffer; byteOffset: number; byteLength: number };
-  return new Float32Array(ta.buffer, ta.byteOffset, ta.byteLength / 4);
-}
-function asInt16(x: unknown): Int16Array {
-  if (x instanceof Int16Array) return x;
-  const ta = x as { buffer: ArrayBuffer; byteOffset: number; byteLength: number };
-  return new Int16Array(ta.buffer, ta.byteOffset, ta.byteLength / 2);
-}
-function asInt32(x: unknown): Int32Array {
-  if (x instanceof Int32Array) return x;
-  const ta = x as { buffer: ArrayBuffer; byteOffset: number; byteLength: number };
-  return new Int32Array(ta.buffer, ta.byteOffset, ta.byteLength / 4);
-}
-function asUint8(x: unknown): Uint8Array {
-  if (x instanceof Uint8Array) return x;
-  const ta = x as { buffer: ArrayBuffer; byteOffset: number; byteLength: number };
-  return new Uint8Array(ta.buffer, ta.byteOffset, ta.byteLength);
-}
-
 // ─────────────────────────────────────────────────────────────────────────────
 // Bridge loader (lazy via the static-import wrapper).
 // ─────────────────────────────────────────────────────────────────────────────

package/src/strategies/fallback/index.ts

@@ -3,6 +3,7 @@ import { VideoRenderer } from "./video-renderer.js";
 import { AudioOutput } from "./audio-output.js";
 import { startDecoder, type DecoderHandles } from "./decoder.js";
 import { dbg } from "../../util/debug.js";
+import { makeTimeRanges } from "../../util/time-ranges.js";
 
 /**
  * Fallback strategy session.
@@ -127,6 +128,30 @@ export async function createFallbackSession(
       get: () => ctx.duration ?? NaN,
     });
   }
+  // Synthesize HTMLMediaElement parity surfaces that the canvas strategies
+  // can't otherwise answer truthfully (the inner <video> has no src, so
+  // its own readyState/seekable are zero/empty).
+  //
+  // readyState: HAVE_NOTHING (0) until the first frame lands; then
+  // HAVE_CURRENT_DATA (2) once the cold-start gate is released (both
+  // audio+video ready). Simplified from the full five-level spec — we
+  // don't distinguish HAVE_FUTURE_DATA vs HAVE_ENOUGH_DATA since our
+  // pump semantics make those essentially the same state.
+  Object.defineProperty(target, "readyState", {
+    configurable: true,
+    get: (): number => {
+      if (!renderer.hasFrames()) return 0; // HAVE_NOTHING
+      if (!audio.isPlaying() && audio.bufferAhead() <= 0 && !audio.isNoAudio()) return 1; // HAVE_METADATA
+      return 2; // HAVE_CURRENT_DATA (or better — but 2 is the honest lower bound)
+    },
+  });
+  // seekable: a progressive source is fully seekable once we have duration.
+  Object.defineProperty(target, "seekable", {
+    configurable: true,
+    get: () => makeTimeRanges(ctx.duration && Number.isFinite(ctx.duration) && ctx.duration > 0
+      ? [[0, ctx.duration]]
+      : []),
+  });
 
   /**
    * Wait until the decoder has produced enough buffered output to start
@@ -256,8 +281,25 @@ export async function createFallbackSession(
       await doSeek(time);
     },
 
-    async setAudioTrack(_id) {
-      // Multi-track audio is post-MVP for the fallback strategy.
+    async setAudioTrack(id) {
+      // Verify the id refers to a real track.
+      if (!ctx.audioTracks.some((t) => t.id === id)) {
+        console.warn("[avbridge] fallback: setAudioTrack — unknown track id", id);
+        return;
+      }
+      const wasPlaying = audio.isPlaying();
+      const currentTime = audio.now();
+      // Suspend audio, rebuild the decoder + seek, reset audio output, re-gate.
+      await audio.pause().catch(() => {});
+      await handles.setAudioTrack(id, currentTime).catch((err) =>
+        console.warn("[avbridge] fallback: handles.setAudioTrack failed:", err),
+      );
+      await audio.reset(currentTime);
+      renderer.flush();
+      if (wasPlaying) {
+        await waitForBuffer();
+        await audio.start();
+      }
     },
 
     async setSubtitleTrack(_id) {
@@ -277,6 +319,8 @@ export async function createFallbackSession(
         delete (target as unknown as Record<string, unknown>).paused;
         delete (target as unknown as Record<string, unknown>).volume;
         delete (target as unknown as Record<string, unknown>).muted;
+        delete (target as unknown as Record<string, unknown>).readyState;
+        delete (target as unknown as Record<string, unknown>).seekable;
       } catch { /* ignore */ }
     },
 

package/src/strategies/fallback/libav-import.ts

@@ -23,5 +23,13 @@ export interface BridgeModule {
   audioStreamToConfig(libav: unknown, stream: unknown): Promise<AudioDecoderConfig | null>;
   packetToEncodedVideoChunk(pkt: unknown, stream: unknown): EncodedVideoChunk;
   packetToEncodedAudioChunk(pkt: unknown, stream: unknown): EncodedAudioChunk;
-  libavFrameToVideoFrame?(frame: unknown, stream: unknown): VideoFrame | null;
+  /**
+   * Convert a libav-decoded frame (software OR hardware decode) into a
+   * WebCodecs VideoFrame. `opts.timeBase` overrides the frame's per-packet
+   * timebase; useful when callers have already normalized pts to µs.
+   */
+  laFrameToVideoFrame(
+    frame: unknown,
+    opts?: { timeBase?: [number, number]; transfer?: boolean },
+  ): VideoFrame;
 }

package/src/strategies/fallback/video-renderer.ts

@@ -1,4 +1,5 @@
 import type { ClockSource } from "./audio-output.js";
+import { SubtitleOverlay } from "../../subtitles/render.js";
 
 /**
  * Renders decoded `VideoFrame`s into a 2D canvas overlaid on the user's
@@ -47,6 +48,16 @@ export class VideoRenderer {
   /** Cumulative count of ticks where PTS mode painted a frame. */
   private ticksPainted = 0;
 
+  /**
+   * Subtitle overlay div attached to the stage wrapper alongside the
+   * canvas. Created lazily when subtitle tracks are attached via the
+   * target's `<track>` children. Canvas strategies (hybrid, fallback)
+   * hide the <video>, so we can't rely on the browser's native cue
+   * rendering; we read TextTrack.cues and render into this overlay.
+   */
+  private subtitleOverlay: SubtitleOverlay | null = null;
+  private subtitleTrack: TextTrack | null = null;
+
   /**
    * Calibration offset (microseconds) between video PTS and audio clock.
    * Video PTS and AudioContext.currentTime can drift ~0.1% relative to
@@ -111,6 +122,15 @@ export class VideoRenderer {
     }
     target.style.visibility = "hidden";
 
+    // Create a subtitle overlay on the same parent as the canvas so cues
+    // appear over the rendered video. Shows nothing until a TextTrack
+    // gets attached via attachSubtitleTracks.
+    const overlayParent = parent instanceof HTMLElement ? parent : document.body;
+    this.subtitleOverlay = new SubtitleOverlay(overlayParent);
+    // Watch for <track> children on the target <video>. When one is
+    // added, grab its TextTrack and poll cues from it each tick.
+    this.watchTextTracks(target);
+
     const ctx = this.canvas.getContext("2d");
     if (!ctx) throw new Error("video renderer: failed to acquire 2D context");
     this.ctx = ctx;
@@ -156,10 +176,95 @@ export class VideoRenderer {
     }
   }
 
+  /**
+   * Watch the target <video>'s textTracks list. When a track is added,
+   * grab it and start polling cues on each render tick. Existing tracks
+   * (if any) are picked up immediately.
+   */
+  private watchTextTracks(target: HTMLVideoElement): void {
+    const pick = () => {
+      if (this.subtitleTrack) return;
+      const tracks = target.textTracks;
+      if (isDebug()) {
+        // eslint-disable-next-line no-console
+        console.log(`[avbridge:subs] watchTextTracks pick() — ${tracks.length} tracks`);
+      }
+      for (let i = 0; i < tracks.length; i++) {
+        const t = tracks[i];
+        if (isDebug()) {
+          // eslint-disable-next-line no-console
+          console.log(`[avbridge:subs] track ${i}: kind=${t.kind} mode=${t.mode} cues=${t.cues?.length ?? 0}`);
+        }
+        if (t.kind === "subtitles" || t.kind === "captions") {
+          this.subtitleTrack = t;
+          t.mode = "hidden"; // hidden means "cues available via API, don't render"
+          if (isDebug()) {
+            // eslint-disable-next-line no-console
+            console.log(`[avbridge:subs] picked track, mode=hidden`);
+          }
+          // Listen for cue load completion
+          const trackEl = target.querySelector(`track[srclang="${t.language}"]`) as HTMLTrackElement | null;
+          if (trackEl) {
+            trackEl.addEventListener("load", () => {
+              if (isDebug()) {
+                // eslint-disable-next-line no-console
+                console.log(`[avbridge:subs] track element loaded, cues=${t.cues?.length ?? 0}`);
+              }
+            });
+            trackEl.addEventListener("error", (ev) => {
+              // eslint-disable-next-line no-console
+              console.warn(`[avbridge:subs] track element error:`, ev);
+            });
+          }
+          break;
+        }
+      }
+    };
+    pick();
+    if (typeof target.textTracks.addEventListener === "function") {
+      target.textTracks.addEventListener("addtrack", (e) => {
+        if (isDebug()) {
+          // eslint-disable-next-line no-console
+          console.log("[avbridge:subs] addtrack event fired");
+        }
+        void e;
+        pick();
+      });
+    }
+  }
+
+  private _loggedCues = false;
+
+  /** Find the active cue (if any) for the given media time. */
+  private updateSubtitles(): void {
+    if (!this.subtitleOverlay || !this.subtitleTrack) return;
+    const cues = this.subtitleTrack.cues;
+    if (!cues || cues.length === 0) return;
+    if (isDebug() && !this._loggedCues) {
+      this._loggedCues = true;
+      // eslint-disable-next-line no-console
+      console.log(`[avbridge:subs] cues available: ${cues.length}, first start=${cues[0].startTime}, last end=${cues[cues.length-1].endTime}`);
+    }
+    const t = this.clock.now();
+    let activeText = "";
+    for (let i = 0; i < cues.length; i++) {
+      const c = cues[i];
+      if (t >= c.startTime && t <= c.endTime) {
+        const vttCue = c as VTTCue & { text?: string };
+        activeText = vttCue.text ?? "";
+        break;
+      }
+    }
+    // Strip VTT tags for plain rendering (e.g. <c.en> voice tags)
+    this.subtitleOverlay.setText(activeText.replace(/<[^>]+>/g, ""));
+  }
+
   private tick(): void {
     if (this.destroyed) return;
     this.rafHandle = requestAnimationFrame(this.tick);
 
+    this.updateSubtitles();
+
     if (this.queue.length === 0) return;
 
     const playing = this.clock.isPlaying();
@@ -331,6 +436,8 @@ export class VideoRenderer {
     this.destroyed = true;
     if (this.rafHandle != null) cancelAnimationFrame(this.rafHandle);
     this.flush();
+    if (this.subtitleOverlay) { this.subtitleOverlay.destroy(); this.subtitleOverlay = null; }
+    this.subtitleTrack = null;
     this.canvas.remove();
     this.target.style.visibility = "";
   }