avbridge 2.1.1 → 2.2.0

package/src/strategies/fallback/decoder.ts

@@ -135,6 +135,20 @@ export async function startDecoder(opts: StartDecoderOptions): Promise<DecoderHa
   let videoFramesDecoded = 0;
   let audioFramesDecoded = 0;
 
+  // Decode-rate watchdog. Samples framesDecoded every second and
+  // compares against realtime expected frames for the source fps. If
+  // the decoder sustains less than 60% of realtime for more than
+  // 5 seconds (counting only time since the first frame emerged),
+  // emits a one-shot diagnostic so users know why playback is
+  // stuttering instead of guessing. A second one-shot fires if the
+  // renderer's overflow-drop rate exceeds 10% of decoded frames —
+  // that symptom means the decoder is BURSTING faster than the
+  // renderer can drain, which is a different bug from "decoder slow".
+  let watchdogFirstFrameMs = 0;
+  let watchdogSlowSinceMs = 0;
+  let watchdogSlowWarned = false;
+  let watchdogOverflowWarned = false;
+
   // Synthetic timestamp counters. Reset on seek.
   let syntheticVideoUs = 0;
   let syntheticAudioUs = 0;
@@ -150,10 +164,18 @@ export async function startDecoder(opts: StartDecoderOptions): Promise<DecoderHa
       let readErr: number;
       let packets: Record<number, LibavPacket[]>;
       try {
-        // Smaller batch = fewer frames per decode round = less queue burst.
-        // 16 KB ≈ 4 video packets + ~12 audio packets at typical DivX
-        // bitrates. The renderer drains ~1 frame per 33ms rAF tick, so
-        // keeping bursts ≤ 4-6 frames prevents queue overflow.
+        // Batch size tunes the tradeoff between JS↔WASM call overhead
+        // (small = more crossings per second) and queue burstiness
+        // (large = decoder hands the renderer big bursts at once that
+        // can blow past the renderer's 64-frame hard cap before the
+        // per-batch `queueHighWater` throttle runs).
+        //
+        // We tried 64 KB and saw ~30% overflow drops on RMVB:rv40 at
+        // 1024x768 because one decode batch regularly produced >30
+        // frames. 16 KB keeps each batch ≈ 4-6 video packets at
+        // typical bitrates, so the worst-case queue spike stays under
+        // `queueHighWater` and the throttle has a chance to apply
+        // backpressure *between* batches rather than within one.
         [readErr, packets] = await libav.ff_read_frame_multi(fmt_ctx, readPkt, {
           limit: 16 * 1024,
         });
@@ -167,16 +189,82 @@ export async function startDecoder(opts: StartDecoderOptions): Promise<DecoderHa
       const videoPackets = videoStream ? packets[videoStream.index] : undefined;
       const audioPackets = audioStream ? packets[audioStream.index] : undefined;
 
-      if (videoDec && videoPackets && videoPackets.length > 0) {
-        await decodeVideoBatch(videoPackets, myToken);
-      }
-      if (myToken !== pumpToken || destroyed) return;
+      // Decode audio BEFORE video. On software-decode-bound content
+      // (rv40/mpeg4/wmv3 @ 720p+) a single video batch can take
+      // 200-400 ms of wall time; if the scheduler hasn't been fed
+      // during that window, audio output runs dry and the user hears
+      // clicks/gaps. Audio is time-critical; video can drop a frame
+      // and nobody notices. Audio decode is also typically <1 ms per
+      // packet for cook/mp3/aac, so doing it first barely delays
+      // video decoding at all.
       if (audioDec && audioPackets && audioPackets.length > 0) {
         await decodeAudioBatch(audioPackets, myToken);
       }
+      if (myToken !== pumpToken || destroyed) return;
+      if (videoDec && videoPackets && videoPackets.length > 0) {
+        await decodeVideoBatch(videoPackets, myToken);
+      }
 
       packetsRead += (videoPackets?.length ?? 0) + (audioPackets?.length ?? 0);
 
+      // ── Decode-rate watchdog ──────────────────────────────────────
+      if (videoFramesDecoded > 0) {
+        if (watchdogFirstFrameMs === 0) {
+          watchdogFirstFrameMs = performance.now();
+        }
+        const elapsedSinceFirst = (performance.now() - watchdogFirstFrameMs) / 1000;
+
+        // 1. Slow-decode detection (sustained <60% of realtime fps).
+        if (elapsedSinceFirst > 1 && !watchdogSlowWarned) {
+          const expectedFrames = elapsedSinceFirst * videoFps;
+          const ratio = videoFramesDecoded / expectedFrames;
+          if (ratio < 0.6) {
+            if (watchdogSlowSinceMs === 0) watchdogSlowSinceMs = performance.now();
+            if ((performance.now() - watchdogSlowSinceMs) / 1000 > 5) {
+              watchdogSlowWarned = true;
+              console.warn(
+                "[avbridge:decode-rate]",
+                `decoder is running slower than realtime: ` +
+                `${videoFramesDecoded} frames in ${elapsedSinceFirst.toFixed(1)}s ` +
+                `(${(videoFramesDecoded / elapsedSinceFirst).toFixed(1)} fps vs ${videoFps} fps source — ` +
+                `${(ratio * 100).toFixed(0)}% of realtime). ` +
+                `Playback will stutter. Typical causes: software decode of a codec with no WebCodecs support ` +
+                `(rv40, mpeg4 @ 720p+, wmv3), or a resolution too large for single-threaded WASM to keep up with.`,
+              );
+            }
+          } else {
+            watchdogSlowSinceMs = 0;
+          }
+        }
+
+        // 2. Overflow-drop detection (>10% of decoded frames dropped
+        //    by the renderer's hard cap). This means the decoder
+        //    produces BURSTS — it's fast enough on average but one
+        //    batch delivers >30 frames at a time, overflowing before
+        //    the queueHighWater throttle can apply backpressure.
+        //    Symptom is different from "decoder slow": here the fps
+        //    ratio looks fine but the user sees choppy playback.
+        if (
+          !watchdogOverflowWarned &&
+          videoFramesDecoded > 100 // wait for a meaningful sample
+        ) {
+          const rendererStats = opts.renderer.stats() as { framesDroppedOverflow?: number };
+          const overflow = rendererStats.framesDroppedOverflow ?? 0;
+          if (overflow / videoFramesDecoded > 0.1) {
+            watchdogOverflowWarned = true;
+            console.warn(
+              "[avbridge:overflow-drop]",
+              `renderer is dropping ${overflow}/${videoFramesDecoded} frames ` +
+              `(${((overflow / videoFramesDecoded) * 100).toFixed(0)}%) because the decoder ` +
+              `is producing bursts faster than the canvas can drain. Symptom: choppy ` +
+              `playback despite decoder keeping up on average. Fix would be smaller ` +
+              `read batches in the pump loop or a lower queueHighWater cap — see ` +
+              `src/strategies/fallback/decoder.ts.`,
+            );
+          }
+        }
+      }
+
       // Throttle: don't run too far ahead of playback. Two backpressure
       // signals:
       //   - Audio buffer (mediaTimeOfNext - now()) > 2 sec — we have

package/src/strategies/fallback/index.ts

@@ -2,6 +2,7 @@ import type { MediaContext, PlaybackSession } from "../../types.js";
 import { VideoRenderer } from "./video-renderer.js";
 import { AudioOutput } from "./audio-output.js";
 import { startDecoder, type DecoderHandles } from "./decoder.js";
+import { dbg } from "../../util/debug.js";
 
 /**
  * Fallback strategy session.
@@ -30,8 +31,27 @@ import { startDecoder, type DecoderHandles } from "./decoder.js";
  * seek(t)` — none of the buffering choreography leaks out.
  */
 
-const READY_AUDIO_BUFFER_SECONDS = 0.3;
-const READY_TIMEOUT_SECONDS = 10;
+// Gate for cold-start playback. We want to start playing as soon as
+// there's any decoded output — the decoder will keep pumping during
+// playback, so more-is-better buffering only helps for fast decoders.
+//
+// For software-decode-bound content (rv40 / wmv3 / mpeg4 @ 720p+ on
+// single-threaded WASM), the decoder may run *slower* than realtime.
+// Waiting for a large audio-buffer threshold is actively wrong in that
+// case: it will never be reached, so the old gate would sit out its
+// full 10-second timeout before playing anything. An aggressive gate
+// ships the first frame to the screen fast, at the cost of the audio
+// clock racing a little ahead of video in the first few seconds —
+// which is the same situation we'd have been in after the timeout
+// anyway.
+//
+// READY_AUDIO_BUFFER_SECONDS: minimum audio queued before start. Set
+// low enough that a slow decoder still reaches it before the user
+// loses patience; 40 ms ≈ 2 cook packets or ~2 AAC packets.
+// READY_TIMEOUT_SECONDS: hard safety. If even 40 ms of audio can't be
+// produced in 3 s, give up and play whatever we have.
+const READY_AUDIO_BUFFER_SECONDS = 0.04;
+const READY_TIMEOUT_SECONDS = 3;
 
 export async function createFallbackSession(
   ctx: MediaContext,
@@ -86,16 +106,80 @@ export async function createFallbackSession(
    * playback smoothly. Returns early on timeout so we don't hang forever
    * if the decoder is producing nothing (e.g. immediately past EOF after
    * a seek to the end).
+   *
+   * The gate has three exit paths in order of preference:
+   *
+   *   1. **Fully ready** — audio buffer ≥ target AND ≥1 video frame.
+   *      The happy path for fast decoders (native + remux never reach
+   *      this function; this is fallback only).
+   *
+   *   2. **Video-ready, audio grace period elapsed** — we have video
+   *      frames but the audio scheduler is still empty. RM/AVI
+   *      containers commonly deliver a video GOP before their first
+   *      audio packet, so "no audio yet" ≠ "no audio coming". We give
+   *      the demuxer a 500 ms grace window from first-frame, then
+   *      start regardless. Audio will be scheduled at its correct
+   *      media time once its packets arrive.
+   *
+   *   3. **Hard timeout** — after {@link READY_TIMEOUT_SECONDS} seconds
+   *      with neither condition met, start anyway and emit an
+   *      unconditional diagnostic so the specific underflow is visible.
+   *
+   * Path #2 is what fixed the "RMVB sits on the play button for 10 s
+   * with audio=0ms, frames=N" case — the gate was waiting on audio
+   * packets that were several seconds behind in the file stream, and
+   * the timeout was the only way out.
    */
   async function waitForBuffer(): Promise<void> {
     const start = performance.now();
+    let firstFrameAtMs = 0;
+    dbg.info("cold-start",
+      `gate entry: want audio ≥ ${READY_AUDIO_BUFFER_SECONDS * 1000}ms + 1 frame`,
+    );
     while (true) {
-      const audioReady = audio.isNoAudio() || audio.bufferAhead() >= READY_AUDIO_BUFFER_SECONDS;
-      if (audioReady && renderer.hasFrames()) {
+      const audioAhead = audio.isNoAudio() ? Infinity : audio.bufferAhead();
+      const audioReady = audio.isNoAudio() || audioAhead >= READY_AUDIO_BUFFER_SECONDS;
+      const hasFrames = renderer.hasFrames();
+      const nowMs = performance.now();
+
+      if (hasFrames && firstFrameAtMs === 0) firstFrameAtMs = nowMs;
+
+      // Happy path: both ready.
+      if (audioReady && hasFrames) {
+        dbg.info("cold-start",
+          `gate satisfied in ${(nowMs - start).toFixed(0)}ms ` +
+          `(audio=${(audioAhead * 1000).toFixed(0)}ms, frames=${renderer.queueDepth()})`,
+        );
         return;
       }
-      if ((performance.now() - start) / 1000 > READY_TIMEOUT_SECONDS) {
-        // Give up waiting; play whatever we have.
+
+      // Grace path: have video, still waiting for audio that's
+      // on its way (first 500 ms after first-frame).
+      if (
+        hasFrames &&
+        firstFrameAtMs > 0 &&
+        nowMs - firstFrameAtMs >= 500
+      ) {
+        dbg.info("cold-start",
+          `gate released on video-only grace at ${(nowMs - start).toFixed(0)}ms ` +
+          `(frames=${renderer.queueDepth()}, audio=${(audioAhead * 1000).toFixed(0)}ms — ` +
+          `demuxer hasn't delivered audio packets yet, starting anyway and letting ` +
+          `the audio scheduler catch up at its media-time anchor)`,
+        );
+        return;
+      }
+
+      // Hard timeout.
+      if ((nowMs - start) / 1000 > READY_TIMEOUT_SECONDS) {
+        dbg.diag("cold-start",
+          `gate TIMEOUT after ${READY_TIMEOUT_SECONDS}s — ` +
+          `audio=${(audioAhead * 1000).toFixed(0)}ms ` +
+          `(needed ${READY_AUDIO_BUFFER_SECONDS * 1000}ms), ` +
+          `frames=${renderer.queueDepth()} (needed ≥1). ` +
+          `Decoder produced nothing in ${READY_TIMEOUT_SECONDS}s — either a corrupt source, ` +
+          `a missing codec, or WASM is catastrophically slow on this file. ` +
+          `Check getDiagnostics().runtime for decode counters.`,
+        );
         return;
       }
       await new Promise((r) => setTimeout(r, 50));

package/src/strategies/fallback/libav-loader.ts

@@ -22,6 +22,8 @@
  * breaks libav's sibling-binary loading.
  */
 
+import { dbg } from "../../util/debug.js";
+
 export type LibavVariant = "webcodecs" | "default" | "avbridge";
 
 export interface LoadLibavOptions {
@@ -85,12 +87,22 @@ export function loadLibav(
 async function loadVariant(
   variant: LibavVariant,
   wantThreads: boolean,
+): Promise<LibavInstance> {
+  return dbg.timed("libav-load", `load "${variant}" (threads=${wantThreads})`, 5000, () =>
+    loadVariantInner(variant, wantThreads),
+  );
+}
+
+async function loadVariantInner(
+  variant: LibavVariant,
+  wantThreads: boolean,
 ): Promise<LibavInstance> {
   const key = cacheKey(variant, wantThreads);
   const base = `${libavBaseUrl()}/${variant}`;
   // The custom variant is named `libav-avbridge.mjs`; the npm variants follow
   // the same convention (`libav-webcodecs.mjs`, `libav-default.mjs`).
   const variantUrl = `${base}/libav-${variant}.mjs`;
+  dbg.info("libav-load", `fetching ${variantUrl}`);
 
   // Preflight HEAD-ish check: issue a bytes=0-0 range request so a missing
   // file fails fast with a clear error instead of hanging deep inside the

package/src/strategies/fallback/video-renderer.ts

@@ -53,11 +53,36 @@ export class VideoRenderer {
     this.canvas = document.createElement("canvas");
     this.canvas.style.cssText =
       "position:absolute;left:0;top:0;width:100%;height:100%;background:black;";
-    const parent = target.parentElement;
-    if (parent && getComputedStyle(parent).position === "static") {
-      parent.style.position = "relative";
+
+    // Attach the canvas next to the video. When the video lives inside an
+    // `<avbridge-video>` shadow root, `target.parentElement` is the
+    // positioned `<div part="stage">` wrapper the element created
+    // precisely for this purpose. When the video is used standalone
+    // (legacy `createPlayer({ target: videoEl })` path), we fall back to
+    // `parentNode` — which handles plain Elements, and also ShadowRoots
+    // if someone inserts a bare <video> inside their own shadow DOM
+    // without a wrapper.
+    const parent: ParentNode | null =
+      (target.parentElement as ParentNode | null) ?? target.parentNode;
+    if (parent && parent instanceof HTMLElement) {
+      if (getComputedStyle(parent).position === "static") {
+        parent.style.position = "relative";
+      }
+    }
+    if (parent) {
+      parent.insertBefore(this.canvas, target);
+    } else {
+      // No parent at all — the target is detached. Fall back to appending
+      // the canvas to document.body so at least the frames are visible
+      // somewhere while the consumer fixes their DOM layout. This is a
+      // loud fallback: log a warning so the misuse is obvious.
+      // eslint-disable-next-line no-console
+      console.warn(
+        "[avbridge] fallback renderer: target <video> has no parent; " +
+        "appending canvas to document.body as a fallback.",
+      );
+      document.body.appendChild(this.canvas);
     }
-    parent?.insertBefore(this.canvas, target);
     target.style.visibility = "hidden";
 
     const ctx = this.canvas.getContext("2d");

package/src/strategies/remux/pipeline.ts

@@ -187,7 +187,16 @@ export async function createRemuxPipeline(
       const vTs = !vNext.done ? vNext.value.timestamp : Number.POSITIVE_INFINITY;
       const aTs = !aNext.done ? aNext.value.timestamp : Number.POSITIVE_INFINITY;
 
-      if (!vNext.done && vTs <= aTs) {
+      // Mediabunny's muxer requires the first packet on a fresh Output to
+      // be a key packet. We fetched `startVideoPacket` via
+      // `videoSink.getKeyPacket(fromTime)` so the first video packet is
+      // guaranteed to be a keyframe — but a demuxer can hand us an audio
+      // packet with a lower timestamp, which mediabunny rejects with
+      // "First packet must be a key packet." Force the first video
+      // packet out before we let any audio through.
+      const forceVideoFirst = firstVideo && !vNext.done;
+
+      if (!vNext.done && (forceVideoFirst || vTs <= aTs)) {
         await videoSource.add(
           vNext.value,
           firstVideo && videoConfig ? { decoderConfig: videoConfig } : undefined,

package/src/types.ts

@@ -23,6 +23,7 @@ export type ContainerKind =
   | "avi"
   | "asf"
   | "flv"
+  | "rm" // RealMedia (.rm / .rmvb)
   | "ogg"
   | "wav"
   | "mp3"
@@ -41,7 +42,10 @@ export type VideoCodec =
   | "mpeg4" // MPEG-4 Part 2 (DivX/Xvid)
   | "wmv3"
   | "vc1"
-  | "rv40"
+  | "rv10" // RealVideo 1.0 (H.263-like)
+  | "rv20" // RealVideo G2
+  | "rv30" // RealVideo 8
+  | "rv40" // RealVideo 9/10
   | "mpeg2"
   | "mpeg1"
   | "theora"
@@ -60,6 +64,11 @@ export type AudioCodec =
   | "wmav2"
   | "wmapro"
   | "alac"
+  | "cook" // RealAudio Cooker (G2/RealAudio 8)
+  | "ra_144" // RealAudio 1.0 (14.4 kbps)
+  | "ra_288" // RealAudio 2.0 (28.8 kbps)
+  | "sipr" // RealAudio Sipr (voice codec)
+  | "atrac3" // Sony ATRAC3 (sometimes seen in .rm)
   | (string & {});
 
 export interface VideoTrackInfo {

package/src/util/debug.ts

@@ -0,0 +1,131 @@
+/**
+ * Debug + self-diagnosis helper.
+ *
+ * avbridge has a lot of async stages (probe → classify → libav load →
+ * strategy.execute → decoder pump → cold-start gate → first paint) and
+ * when something's slow or wrong the symptom — "it hangs", "it stutters",
+ * "it plays audio without video" — is usually nowhere near the actual
+ * cause. This module gives us two things:
+ *
+ * 1. **Gated verbose logging.** `dbg.info(tag, ...)` etc. are no-ops
+ *    unless the consumer sets `globalThis.AVBRIDGE_DEBUG = true` (or
+ *    uses the matching `?avbridge_debug` URL search param at dev time).
+ *    When enabled, every log is prefixed with `[avbridge:<tag>]` so the
+ *    console is filterable.
+ *
+ * 2. **Unconditional self-diagnosis.** `dbg.warnIf(cond, tag, ...)`
+ *    always fires when a suspicious condition is detected, even with
+ *    debug off. These are the things we *know* mean something is
+ *    broken or degraded and the user would want to know about — e.g.
+ *    the cold-start gate timing out, the decoder running slower than
+ *    realtime, a libav variant taking longer to load than any network
+ *    should take, >20% of packets getting rejected.
+ *
+ * The guiding principle: **if a symptom caused more than 10 minutes of
+ * human debugging once, add a targeted warning so the next instance
+ * self-identifies in the console.** This module is where those
+ * warnings live.
+ */
+
+/** Read the debug flag fresh on every call so it's runtime-toggleable. */
+function isDebugEnabled(): boolean {
+  if (typeof globalThis === "undefined") return false;
+  const g = globalThis as { AVBRIDGE_DEBUG?: unknown };
+  if (g.AVBRIDGE_DEBUG === true) return true;
+  // Convenience: if running in a browser with a `?avbridge_debug` search
+  // param, flip the flag on automatically. Useful for demos and quick
+  // user reproduction without editing code.
+  if (typeof location !== "undefined" && typeof URLSearchParams !== "undefined") {
+    try {
+      const p = new URLSearchParams(location.search);
+      if (p.has("avbridge_debug")) {
+        g.AVBRIDGE_DEBUG = true;
+        return true;
+      }
+    } catch { /* ignore */ }
+  }
+  return false;
+}
+
+function fmt(tag: string): string {
+  return `[avbridge:${tag}]`;
+}
+
+/* eslint-disable no-console */
+
+export const dbg = {
+  /** Verbose — only when debug is enabled. The hot-path normal case. */
+  info(tag: string, ...args: unknown[]): void {
+    if (isDebugEnabled()) console.info(fmt(tag), ...args);
+  },
+
+  /** Warning — only when debug is enabled. Non-fatal oddities. */
+  warn(tag: string, ...args: unknown[]): void {
+    if (isDebugEnabled()) console.warn(fmt(tag), ...args);
+  },
+
+  /**
+   * Self-diagnosis warning. **Always** emits regardless of debug flag.
+   * Use this only for conditions that mean something is actually wrong
+   * or degraded — not for routine chatter.
+   */
+  diag(tag: string, ...args: unknown[]): void {
+    console.warn(fmt(tag), ...args);
+  },
+
+  /**
+   * Timing helper: wraps an async call and logs its elapsed time when
+   * debug is on. The callback runs whether debug is on or off — this is
+   * just for the `dbg.info` at the end.
+   *
+   * Also unconditionally fires `dbg.diag` if the elapsed time exceeds
+   * `slowMs`, so "the bootstrap took 8 seconds" shows up even without
+   * debug mode enabled.
+   */
+  async timed<T>(
+    tag: string,
+    label: string,
+    slowMs: number,
+    fn: () => Promise<T>,
+  ): Promise<T> {
+    const start = performance.now();
+    try {
+      const result = await fn();
+      const elapsed = performance.now() - start;
+      if (isDebugEnabled()) {
+        console.info(fmt(tag), `${label} ${elapsed.toFixed(0)}ms`);
+      }
+      if (elapsed > slowMs) {
+        console.warn(
+          fmt(tag),
+          `${label} took ${elapsed.toFixed(0)}ms (>${slowMs}ms expected) — ` +
+          `this is unusually slow; possible causes: ${hintForTag(tag)}`,
+        );
+      }
+      return result;
+    } catch (err) {
+      const elapsed = performance.now() - start;
+      console.warn(
+        fmt(tag),
+        `${label} FAILED after ${elapsed.toFixed(0)}ms:`,
+        err,
+      );
+      throw err;
+    }
+  },
+};
+
+function hintForTag(tag: string): string {
+  switch (tag) {
+    case "probe":
+      return "slow network (range request), large sniff window, or libav cold-start";
+    case "libav-load":
+      return "large .wasm download, misconfigured AVBRIDGE_LIBAV_BASE, or server-side MIME type";
+    case "bootstrap":
+      return "probe+classify+strategy-init chain; enable AVBRIDGE_DEBUG for a phase breakdown";
+    case "cold-start":
+      return "decoder is producing output slower than realtime — check framesDecoded in getDiagnostics()";
+    default:
+      return "unknown stage — enable globalThis.AVBRIDGE_DEBUG for more detail";
+  }
+}

package/src/util/source.ts

@@ -215,6 +215,10 @@ export function sniffContainerFromBytes(head: Uint8Array): ContainerKind {
   ) return "asf";
   // FLV: 46 4C 56
   if (head[0] === 0x46 && head[1] === 0x4c && head[2] === 0x56) return "flv";
+  // RealMedia (.rm / .rmvb): ".RMF" — 2E 52 4D 46
+  if (head[0] === 0x2e && head[1] === 0x52 && head[2] === 0x4d && head[3] === 0x46) {
+    return "rm";
+  }
   // OggS: 4F 67 67 53
   if (head[0] === 0x4f && head[1] === 0x67 && head[2] === 0x67 && head[3] === 0x53) return "ogg";
   // FLAC: 66 4C 61 43