avbridge 2.2.1 → 2.5.0

package/src/strategies/fallback/video-renderer.ts

@@ -1,4 +1,5 @@
 import type { ClockSource } from "./audio-output.js";
+import { SubtitleOverlay } from "../../subtitles/render.js";
 
 /**
  * Renders decoded `VideoFrame`s into a 2D canvas overlaid on the user's
@@ -20,6 +21,13 @@ import type { ClockSource } from "./audio-output.js";
  * decoder was still warming up, and every frame was already in the past by
  * the time it landed in the queue.
  */
+// Periodic debug log — throttled to once per second so it doesn't
+// flood the console at 60Hz rAF rate.
+function isDebug(): boolean {
+  return typeof globalThis !== "undefined" && !!(globalThis as Record<string, unknown>).AVBRIDGE_DEBUG;
+}
+let lastDebugLog = 0;
+
 export class VideoRenderer {
   private canvas: HTMLCanvasElement;
   private ctx: CanvasRenderingContext2D;
@@ -35,6 +43,31 @@ export class VideoRenderer {
   private lastPaintWall = 0;
   /** Minimum ms between paints — paces video at roughly source fps. */
   private paintIntervalMs: number;
+  /** Cumulative count of frames skipped because all PTS are in the future. */
+  private ticksWaiting = 0;
+  /** Cumulative count of ticks where PTS mode painted a frame. */
+  private ticksPainted = 0;
+
+  /**
+   * Subtitle overlay div attached to the stage wrapper alongside the
+   * canvas. Created lazily when subtitle tracks are attached via the
+   * target's `<track>` children. Canvas strategies (hybrid, fallback)
+   * hide the <video>, so we can't rely on the browser's native cue
+   * rendering; we read TextTrack.cues and render into this overlay.
+   */
+  private subtitleOverlay: SubtitleOverlay | null = null;
+  private subtitleTrack: TextTrack | null = null;
+
+  /**
+   * Calibration offset (microseconds) between video PTS and audio clock.
+   * Video PTS and AudioContext.currentTime can drift ~0.1% relative to
+   * each other (different clock domains). Over 45 minutes that's 2.6s.
+   * We measure the offset on the first painted frame and update it
+   * periodically so the PTS comparison stays calibrated.
+   */
+  private ptsCalibrationUs = 0;
+  private ptsCalibrated = false;
+  private lastCalibrationWall = 0;
 
   /** Resolves once the first decoded frame has been enqueued. */
   readonly firstFrameReady: Promise<void>;
@@ -89,6 +122,15 @@ export class VideoRenderer {
     }
     target.style.visibility = "hidden";
 
+    // Create a subtitle overlay on the same parent as the canvas so cues
+    // appear over the rendered video. Shows nothing until a TextTrack
+    // gets attached via attachSubtitleTracks.
+    const overlayParent = parent instanceof HTMLElement ? parent : document.body;
+    this.subtitleOverlay = new SubtitleOverlay(overlayParent);
+    // Watch for <track> children on the target <video>. When one is
+    // added, grab its TextTrack and poll cues from it each tick.
+    this.watchTextTracks(target);
+
     const ctx = this.canvas.getContext("2d");
     if (!ctx) throw new Error("video renderer: failed to acquire 2D context");
     this.ctx = ctx;
@@ -134,10 +176,95 @@ export class VideoRenderer {
     }
   }
 
+  /**
+   * Watch the target <video>'s textTracks list. When a track is added,
+   * grab it and start polling cues on each render tick. Existing tracks
+   * (if any) are picked up immediately.
+   */
+  private watchTextTracks(target: HTMLVideoElement): void {
+    const pick = () => {
+      if (this.subtitleTrack) return;
+      const tracks = target.textTracks;
+      if (isDebug()) {
+        // eslint-disable-next-line no-console
+        console.log(`[avbridge:subs] watchTextTracks pick() — ${tracks.length} tracks`);
+      }
+      for (let i = 0; i < tracks.length; i++) {
+        const t = tracks[i];
+        if (isDebug()) {
+          // eslint-disable-next-line no-console
+          console.log(`[avbridge:subs] track ${i}: kind=${t.kind} mode=${t.mode} cues=${t.cues?.length ?? 0}`);
+        }
+        if (t.kind === "subtitles" || t.kind === "captions") {
+          this.subtitleTrack = t;
+          t.mode = "hidden"; // hidden means "cues available via API, don't render"
+          if (isDebug()) {
+            // eslint-disable-next-line no-console
+            console.log(`[avbridge:subs] picked track, mode=hidden`);
+          }
+          // Listen for cue load completion
+          const trackEl = target.querySelector(`track[srclang="${t.language}"]`) as HTMLTrackElement | null;
+          if (trackEl) {
+            trackEl.addEventListener("load", () => {
+              if (isDebug()) {
+                // eslint-disable-next-line no-console
+                console.log(`[avbridge:subs] track element loaded, cues=${t.cues?.length ?? 0}`);
+              }
+            });
+            trackEl.addEventListener("error", (ev) => {
+              // eslint-disable-next-line no-console
+              console.warn(`[avbridge:subs] track element error:`, ev);
+            });
+          }
+          break;
+        }
+      }
+    };
+    pick();
+    if (typeof target.textTracks.addEventListener === "function") {
+      target.textTracks.addEventListener("addtrack", (e) => {
+        if (isDebug()) {
+          // eslint-disable-next-line no-console
+          console.log("[avbridge:subs] addtrack event fired");
+        }
+        void e;
+        pick();
+      });
+    }
+  }
+
+  private _loggedCues = false;
+
+  /** Find the active cue (if any) for the given media time. */
+  private updateSubtitles(): void {
+    if (!this.subtitleOverlay || !this.subtitleTrack) return;
+    const cues = this.subtitleTrack.cues;
+    if (!cues || cues.length === 0) return;
+    if (isDebug() && !this._loggedCues) {
+      this._loggedCues = true;
+      // eslint-disable-next-line no-console
+      console.log(`[avbridge:subs] cues available: ${cues.length}, first start=${cues[0].startTime}, last end=${cues[cues.length-1].endTime}`);
+    }
+    const t = this.clock.now();
+    let activeText = "";
+    for (let i = 0; i < cues.length; i++) {
+      const c = cues[i];
+      if (t >= c.startTime && t <= c.endTime) {
+        const vttCue = c as VTTCue & { text?: string };
+        activeText = vttCue.text ?? "";
+        break;
+      }
+    }
+    // Strip VTT tags for plain rendering (e.g. <c.en> voice tags)
+    this.subtitleOverlay.setText(activeText.replace(/<[^>]+>/g, ""));
+  }
+
   private tick(): void {
     if (this.destroyed) return;
     this.rafHandle = requestAnimationFrame(this.tick);
 
+    this.updateSubtitles();
+
     if (this.queue.length === 0) return;
 
     const playing = this.clock.isPlaying();
@@ -154,45 +281,109 @@ export class VideoRenderer {
       return;
     }
 
-    // Wall-clock-paced painting with coarse A/V drift correction.
-    //
-    // Base policy: paint one frame every `paintIntervalMs` of wall time,
-    // regardless of the frame's synthetic timestamp. This avoids the old
-    // per-frame audio-gate that caused massive overflow during decode bursts.
+    // PTS-based painting: find the latest frame whose presentation time
+    // has arrived (timestamp ≤ audio clock), paint it, and discard any
+    // older frames. This produces correct cadence at any display refresh
+    // rate and any source fps — no 3:2 pulldown artifacts.
     //
-    // Drift correction (runs every ~1 sec):
-    //   - Video > 150 ms behind audio → drop one frame (catch up)
-    //   - Video > 150 ms ahead of audio → skip one paint (let audio catch up)
-    //
-    // This keeps long-run sync robust even for legacy AVI/DivX with messy
-    // timestamps, packed B-frames, and odd frame durations. The correction
-    // is deliberately gentle (one frame at a time) so it doesn't cause
-    // visible stuttering.
-    const wallNow = performance.now();
-    if (wallNow - this.lastPaintWall < this.paintIntervalMs - 2) return;
+    // Fallback: if frame timestamps are unreliable (all zero, synthetic),
+    // fall back to wall-clock pacing as before.
+    const rawAudioNowUs = this.clock.now() * 1_000_000;
+    const headTs = this.queue[0].timestamp ?? 0;
+    const hasPts = headTs > 0 || this.queue.length > 1;
 
-    if (this.queue.length === 0) return;
+    if (hasPts) {
+      // Calibration: video PTS and audio clock (AudioContext.currentTime)
+      // live in different clock domains with a fixed offset (different epoch)
+      // plus a small rate drift (~7ms/s). We snap the offset on first paint
+      // and re-snap every 10 seconds. Between snaps, max drift is ~70ms
+      // (under 2 frames at 24fps, below lip-sync perception threshold).
+      const wallNow = performance.now();
+      if (!this.ptsCalibrated || wallNow - this.lastCalibrationWall > 10_000) {
+        this.ptsCalibrationUs = headTs - rawAudioNowUs;
+        this.ptsCalibrated = true;
+        this.lastCalibrationWall = wallNow;
+      }
+
+      const audioNowUs = rawAudioNowUs + this.ptsCalibrationUs;
+      const frameDurationUs = this.paintIntervalMs * 1000;
+      const deadlineUs = audioNowUs + frameDurationUs;
 
-    // Coarse drift correction: compare the head frame's timestamp to
-    // audio.now() every ~1 sec (every 30 frames at 30fps). The frame ts
-    // and audio.now() are both in seconds of media time. Drift beyond
-    // 150ms triggers gentle correction — one frame per check, not a burst.
-    if (this.framesPainted > 0 && this.framesPainted % 30 === 0) {
-      const audioNowUs = this.clock.now() * 1_000_000;
-      const headTs = this.queue[0].timestamp ?? 0;
-      const driftUs = headTs - audioNowUs;
-
-      if (driftUs < -150_000) {
-        // Video behind audio by > 150ms — drop one frame to catch up.
-        this.queue.shift()?.close();
-        this.framesDroppedLate++;
-        if (this.queue.length === 0) return;
-      } else if (driftUs > 150_000) {
-        // Video ahead of audio by > 150ms — skip this paint cycle.
+      let bestIdx = -1;
+      for (let i = 0; i < this.queue.length; i++) {
+        const ts = this.queue[i].timestamp ?? 0;
+        if (ts <= deadlineUs) {
+          bestIdx = i;
+        } else {
+          break;
+        }
+      }
+
+      if (bestIdx < 0) {
+        this.ticksWaiting++;
+        if (isDebug()) {
+          const now = performance.now();
+          if (now - lastDebugLog > 1000) {
+            const headPtsMs = (headTs / 1000).toFixed(1);
+            const audioMs = (audioNowUs / 1000).toFixed(1);
+            const rawDriftMs = ((headTs - rawAudioNowUs) / 1000).toFixed(1);
+            const calibMs = (this.ptsCalibrationUs / 1000).toFixed(1);
+            // eslint-disable-next-line no-console
+            console.log(
+              `[avbridge:renderer] WAIT q=${this.queue.length} headPTS=${headPtsMs}ms calibAudio=${audioMs}ms ` +
+              `rawDrift=${rawDriftMs}ms calib=${calibMs}ms painted=${this.framesPainted} dropped=${this.framesDroppedLate}`,
+            );
+            lastDebugLog = now;
+          }
+        }
         return;
       }
+
+      // Only drop frames that are more than 2 frame-durations behind.
+      const dropThresholdUs = audioNowUs - frameDurationUs * 2;
+      let dropped = 0;
+      while (bestIdx > 0) {
+        const ts = this.queue[0].timestamp ?? 0;
+        if (ts < dropThresholdUs) {
+          this.queue.shift()?.close();
+          this.framesDroppedLate++;
+          bestIdx--;
+          dropped++;
+        } else {
+          break;
+        }
+      }
+
+      this.ticksPainted++;
+
+      if (isDebug()) {
+        const now = performance.now();
+        if (now - lastDebugLog > 1000) {
+          const paintedTs = (this.queue[0]?.timestamp ?? 0);
+          const audioMs = (audioNowUs / 1000).toFixed(1);
+          const ptsMs = (paintedTs / 1000).toFixed(1);
+          const rawDriftMs = ((paintedTs - rawAudioNowUs) / 1000).toFixed(1);
+          const calibMs = (this.ptsCalibrationUs / 1000).toFixed(1);
+          // eslint-disable-next-line no-console
+          console.log(
+            `[avbridge:renderer] PAINT q=${this.queue.length} calibAudio=${audioMs}ms nextPTS=${ptsMs}ms ` +
+            `rawDrift=${rawDriftMs}ms calib=${calibMs}ms dropped=${dropped} total_drops=${this.framesDroppedLate} painted=${this.framesPainted}`,
+          );
+          lastDebugLog = now;
+        }
+      }
+
+      const frame = this.queue.shift()!;
+      this.paint(frame);
+      frame.close();
+      this.lastPaintWall = performance.now();
+      return;
     }
 
+    // Wall-clock fallback: used when timestamps are unreliable (all zero).
+    const wallNow = performance.now();
+    if (wallNow - this.lastPaintWall < this.paintIntervalMs - 2) return;
+
     const frame = this.queue.shift()!;
     this.paint(frame);
     frame.close();
@@ -222,8 +413,14 @@ export class VideoRenderer {
 
   /** Discard all queued frames. Used by seek to drop stale buffers. */
   flush(): void {
+    const count = this.queue.length;
     while (this.queue.length > 0) this.queue.shift()?.close();
     this.prerolled = false;
+    this.ptsCalibrated = false; // recalibrate at new seek position
+    if (isDebug() && count > 0) {
+      // eslint-disable-next-line no-console
+      console.log(`[avbridge:renderer] FLUSH discarded=${count} painted=${this.framesPainted} drops=${this.framesDroppedLate}`);
+    }
   }
 
   stats(): Record<string, unknown> {
@@ -239,6 +436,8 @@ export class VideoRenderer {
     this.destroyed = true;
     if (this.rafHandle != null) cancelAnimationFrame(this.rafHandle);
     this.flush();
+    if (this.subtitleOverlay) { this.subtitleOverlay.destroy(); this.subtitleOverlay = null; }
+    this.subtitleTrack = null;
     this.canvas.remove();
     this.target.style.visibility = "";
   }