avbridge 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/CHANGELOG.md +120 -0
  2. package/LICENSE +21 -0
  3. package/README.md +415 -0
  4. package/dist/avi-M5B4SHRM.cjs +164 -0
  5. package/dist/avi-M5B4SHRM.cjs.map +1 -0
  6. package/dist/avi-POCGZ4JX.js +162 -0
  7. package/dist/avi-POCGZ4JX.js.map +1 -0
  8. package/dist/chunk-5ISVAODK.js +80 -0
  9. package/dist/chunk-5ISVAODK.js.map +1 -0
  10. package/dist/chunk-F7YS2XOA.cjs +2966 -0
  11. package/dist/chunk-F7YS2XOA.cjs.map +1 -0
  12. package/dist/chunk-FKM7QBZU.js +2957 -0
  13. package/dist/chunk-FKM7QBZU.js.map +1 -0
  14. package/dist/chunk-J5MCMN3S.js +27 -0
  15. package/dist/chunk-J5MCMN3S.js.map +1 -0
  16. package/dist/chunk-L4NPOJ36.cjs +180 -0
  17. package/dist/chunk-L4NPOJ36.cjs.map +1 -0
  18. package/dist/chunk-NZU7W256.cjs +29 -0
  19. package/dist/chunk-NZU7W256.cjs.map +1 -0
  20. package/dist/chunk-PQTZS7OA.js +147 -0
  21. package/dist/chunk-PQTZS7OA.js.map +1 -0
  22. package/dist/chunk-WD2ZNQA7.js +177 -0
  23. package/dist/chunk-WD2ZNQA7.js.map +1 -0
  24. package/dist/chunk-Y5FYF5KG.cjs +153 -0
  25. package/dist/chunk-Y5FYF5KG.cjs.map +1 -0
  26. package/dist/chunk-Z2FJ5TJC.cjs +82 -0
  27. package/dist/chunk-Z2FJ5TJC.cjs.map +1 -0
  28. package/dist/element.cjs +433 -0
  29. package/dist/element.cjs.map +1 -0
  30. package/dist/element.d.cts +158 -0
  31. package/dist/element.d.ts +158 -0
  32. package/dist/element.js +431 -0
  33. package/dist/element.js.map +1 -0
  34. package/dist/index.cjs +576 -0
  35. package/dist/index.cjs.map +1 -0
  36. package/dist/index.d.cts +80 -0
  37. package/dist/index.d.ts +80 -0
  38. package/dist/index.js +554 -0
  39. package/dist/index.js.map +1 -0
  40. package/dist/libav-http-reader-FPYDBMYK.cjs +16 -0
  41. package/dist/libav-http-reader-FPYDBMYK.cjs.map +1 -0
  42. package/dist/libav-http-reader-NQJVY273.js +3 -0
  43. package/dist/libav-http-reader-NQJVY273.js.map +1 -0
  44. package/dist/libav-import-2JURFHEW.js +8 -0
  45. package/dist/libav-import-2JURFHEW.js.map +1 -0
  46. package/dist/libav-import-GST2AMPL.cjs +30 -0
  47. package/dist/libav-import-GST2AMPL.cjs.map +1 -0
  48. package/dist/libav-loader-KA2MAWLM.js +3 -0
  49. package/dist/libav-loader-KA2MAWLM.js.map +1 -0
  50. package/dist/libav-loader-ZHOERPHW.cjs +12 -0
  51. package/dist/libav-loader-ZHOERPHW.cjs.map +1 -0
  52. package/dist/player-BBwbCkdL.d.cts +365 -0
  53. package/dist/player-BBwbCkdL.d.ts +365 -0
  54. package/dist/source-SC6ZEQYR.cjs +28 -0
  55. package/dist/source-SC6ZEQYR.cjs.map +1 -0
  56. package/dist/source-ZFS4H7J3.js +3 -0
  57. package/dist/source-ZFS4H7J3.js.map +1 -0
  58. package/dist/variant-routing-GOHB2RZN.cjs +12 -0
  59. package/dist/variant-routing-GOHB2RZN.cjs.map +1 -0
  60. package/dist/variant-routing-JOBWXYKD.js +3 -0
  61. package/dist/variant-routing-JOBWXYKD.js.map +1 -0
  62. package/package.json +95 -0
  63. package/src/classify/index.ts +1 -0
  64. package/src/classify/rules.ts +214 -0
  65. package/src/convert/index.ts +2 -0
  66. package/src/convert/remux.ts +522 -0
  67. package/src/convert/transcode.ts +329 -0
  68. package/src/diagnostics.ts +99 -0
  69. package/src/element/avbridge-player.ts +576 -0
  70. package/src/element.ts +19 -0
  71. package/src/events.ts +71 -0
  72. package/src/index.ts +42 -0
  73. package/src/libav-stubs.d.ts +24 -0
  74. package/src/player.ts +455 -0
  75. package/src/plugins/builtin.ts +37 -0
  76. package/src/plugins/registry.ts +32 -0
  77. package/src/probe/avi.ts +242 -0
  78. package/src/probe/index.ts +59 -0
  79. package/src/probe/mediabunny.ts +194 -0
  80. package/src/strategies/fallback/audio-output.ts +293 -0
  81. package/src/strategies/fallback/clock.ts +7 -0
  82. package/src/strategies/fallback/decoder.ts +660 -0
  83. package/src/strategies/fallback/index.ts +170 -0
  84. package/src/strategies/fallback/libav-import.ts +27 -0
  85. package/src/strategies/fallback/libav-loader.ts +190 -0
  86. package/src/strategies/fallback/variant-routing.ts +43 -0
  87. package/src/strategies/fallback/video-renderer.ts +216 -0
  88. package/src/strategies/hybrid/decoder.ts +641 -0
  89. package/src/strategies/hybrid/index.ts +139 -0
  90. package/src/strategies/native.ts +107 -0
  91. package/src/strategies/remux/annexb.ts +112 -0
  92. package/src/strategies/remux/index.ts +79 -0
  93. package/src/strategies/remux/mse.ts +234 -0
  94. package/src/strategies/remux/pipeline.ts +254 -0
  95. package/src/subtitles/index.ts +91 -0
  96. package/src/subtitles/render.ts +62 -0
  97. package/src/subtitles/srt.ts +62 -0
  98. package/src/subtitles/vtt.ts +5 -0
  99. package/src/types-shim.d.ts +3 -0
  100. package/src/types.ts +360 -0
  101. package/src/util/codec-strings.ts +86 -0
  102. package/src/util/libav-http-reader.ts +315 -0
  103. package/src/util/source.ts +274 -0
@@ -0,0 +1,170 @@
1
+ import type { MediaContext, PlaybackSession } from "../../types.js";
2
+ import { VideoRenderer } from "./video-renderer.js";
3
+ import { AudioOutput } from "./audio-output.js";
4
+ import { startDecoder, type DecoderHandles } from "./decoder.js";
5
+
6
+ /**
7
+ * Fallback strategy session.
8
+ *
9
+ * Owns the orchestration between the libav decoder, the audio scheduler,
10
+ * and the canvas renderer. Three things make this non-trivial:
11
+ *
12
+ * 1. **Cold-start ready gate.** When `play()` is called, we wait until the
13
+ * audio scheduler has buffered enough audio (≥ 300 ms) AND the renderer
14
+ * has at least one decoded video frame, before actually telling the
15
+ * audio context to start. Without this gate, audio and the wall clock
16
+ * race ahead of the still-warming-up software decoder, and every video
17
+ * frame lands "in the past" and gets dropped.
18
+ *
19
+ * 2. **Pause / resume.** The audio context is suspended on pause and
20
+ * resumed on play. The media-time anchor is preserved across the
21
+ * suspend so the clock is continuous.
22
+ *
23
+ * 3. **Seek.** Pauses the audio scheduler, asks the decoder to cancel its
24
+ * current pump and `av_seek_frame` to the target, resets the audio
25
+ * output's media-time anchor to the seek target, flushes the renderer
26
+ * queue, then re-enters the ready gate. If we were playing before the
27
+ * seek, we automatically resume once the buffer fills.
28
+ *
29
+ * The unified player API on top of this just sees `play() / pause() /
30
+ * seek(t)` — none of the buffering choreography leaks out.
31
+ */
32
+
33
+ const READY_AUDIO_BUFFER_SECONDS = 0.3;
34
+ const READY_TIMEOUT_SECONDS = 10;
35
+
36
+ export async function createFallbackSession(
37
+ ctx: MediaContext,
38
+ target: HTMLVideoElement,
39
+ ): Promise<PlaybackSession> {
40
+ // Normalize the source so URL inputs go through the libav HTTP block
41
+ // reader instead of being buffered into memory.
42
+ const { normalizeSource } = await import("../../util/source.js");
43
+ const source = await normalizeSource(ctx.source);
44
+
45
+ const fps = ctx.videoTracks[0]?.fps ?? 30;
46
+ const audio = new AudioOutput();
47
+ const renderer = new VideoRenderer(target, audio, fps);
48
+
49
+ let handles: DecoderHandles;
50
+ try {
51
+ handles = await startDecoder({
52
+ source,
53
+ filename: ctx.name ?? "input.bin",
54
+ context: ctx,
55
+ renderer,
56
+ audio,
57
+ });
58
+ } catch (err) {
59
+ audio.destroy();
60
+ renderer.destroy();
61
+ throw err;
62
+ }
63
+
64
+ // Patch the <video> element so the unified player layer (which polls
65
+ // `target.currentTime` for `timeupdate` events and lets users assign to
66
+ // it for seeks) gets the right values from the fallback strategy.
67
+ Object.defineProperty(target, "currentTime", {
68
+ configurable: true,
69
+ get: () => audio.now(),
70
+ set: (v: number) => {
71
+ // Fire-and-forget — the user is expected to await player.seek() if
72
+ // they want to know when the seek completes.
73
+ void doSeek(v);
74
+ },
75
+ });
76
+ // Mirror duration so the demo's controls can use target.duration too.
77
+ if (ctx.duration && Number.isFinite(ctx.duration)) {
78
+ Object.defineProperty(target, "duration", {
79
+ configurable: true,
80
+ get: () => ctx.duration ?? NaN,
81
+ });
82
+ }
83
+
84
+ /**
85
+ * Wait until the decoder has produced enough buffered output to start
86
+ * playback smoothly. Returns early on timeout so we don't hang forever
87
+ * if the decoder is producing nothing (e.g. immediately past EOF after
88
+ * a seek to the end).
89
+ */
90
+ async function waitForBuffer(): Promise<void> {
91
+ const start = performance.now();
92
+ while (true) {
93
+ const audioReady = audio.isNoAudio() || audio.bufferAhead() >= READY_AUDIO_BUFFER_SECONDS;
94
+ if (audioReady && renderer.hasFrames()) {
95
+ return;
96
+ }
97
+ if ((performance.now() - start) / 1000 > READY_TIMEOUT_SECONDS) {
98
+ // Give up waiting; play whatever we have.
99
+ return;
100
+ }
101
+ await new Promise((r) => setTimeout(r, 50));
102
+ }
103
+ }
104
+
105
+ async function doSeek(timeSec: number): Promise<void> {
106
+ const wasPlaying = audio.isPlaying();
107
+ // 1. Stop audio (suspend ctx + capture media time).
108
+ await audio.pause().catch(() => {});
109
+ // 2. Tell the decoder to cancel its pump and seek the demuxer.
110
+ await handles.seek(timeSec).catch((err) =>
111
+ console.warn("[avbridge] decoder seek failed:", err),
112
+ );
113
+ // 3. Reset audio + renderer to the new media time. New samples from
114
+ // the decoder will queue against this anchor.
115
+ await audio.reset(timeSec);
116
+ renderer.flush();
117
+ // 4. If we were playing, wait for the buffer to fill again and then
118
+ // resume. If we were paused, leave it paused at the new position.
119
+ if (wasPlaying) {
120
+ await waitForBuffer();
121
+ await audio.start();
122
+ }
123
+ }
124
+
125
+ return {
126
+ strategy: "fallback",
127
+
128
+ async play() {
129
+ // Either a cold start (very first play() call) or a resume from
130
+ // pause. AudioOutput.start() handles both.
131
+ if (!audio.isPlaying()) {
132
+ await waitForBuffer();
133
+ await audio.start();
134
+ }
135
+ },
136
+
137
+ pause() {
138
+ void audio.pause();
139
+ },
140
+
141
+ async seek(time) {
142
+ await doSeek(time);
143
+ },
144
+
145
+ async setAudioTrack(_id) {
146
+ // Multi-track audio is post-MVP for the fallback strategy.
147
+ },
148
+
149
+ async setSubtitleTrack(_id) {
150
+ // Subtitle overlay support is post-MVP for the fallback strategy.
151
+ },
152
+
153
+ getCurrentTime() {
154
+ return audio.now();
155
+ },
156
+ async destroy() {
157
+ await handles.destroy();
158
+ renderer.destroy();
159
+ audio.destroy();
160
+ try {
161
+ delete (target as unknown as Record<string, unknown>).currentTime;
162
+ delete (target as unknown as Record<string, unknown>).duration;
163
+ } catch { /* ignore */ }
164
+ },
165
+
166
+ getRuntimeStats() {
167
+ return handles.stats();
168
+ },
169
+ };
170
+ }
@@ -0,0 +1,27 @@
1
+ /**
2
+ * Static-import wrapper for the libavjs-webcodecs-bridge optional peer dep.
3
+ *
4
+ * The variant itself is **not** imported here — it's loaded via a runtime
5
+ * dynamic import with `/* @vite-ignore *\/` from `libav-loader.ts`, so the
6
+ * variant's `.mjs` file is never touched by Vite's transform pipeline (which
7
+ * would otherwise pre-bundle it and break the `import.meta.url`-based path
8
+ * resolution it uses to find its sibling .wasm files).
9
+ *
10
+ * The bridge has no such issue — it's pure JS and doesn't reference sibling
11
+ * binaries — so a normal static import is fine here.
12
+ *
13
+ * TypeScript resolves `libavjs-webcodecs-bridge` via the `paths` mapping in
14
+ * tsconfig.json which redirects to `src/libav-stubs.d.ts`, sidestepping the
15
+ * polyfill source files that don't typecheck under TS 5.7.
16
+ */
17
+ import * as bridge from "libavjs-webcodecs-bridge";
18
+
19
+ export const libavBridge: BridgeModule = bridge as unknown as BridgeModule;
20
+
21
+ export interface BridgeModule {
22
+ videoStreamToConfig(libav: unknown, stream: unknown): Promise<VideoDecoderConfig | null>;
23
+ audioStreamToConfig(libav: unknown, stream: unknown): Promise<AudioDecoderConfig | null>;
24
+ packetToEncodedVideoChunk(pkt: unknown, stream: unknown): EncodedVideoChunk;
25
+ packetToEncodedAudioChunk(pkt: unknown, stream: unknown): EncodedAudioChunk;
26
+ libavFrameToVideoFrame?(frame: unknown, stream: unknown): VideoFrame | null;
27
+ }
@@ -0,0 +1,190 @@
1
+ /**
2
+ * Lazy libav.js loader supporting multiple variants.
3
+ *
4
+ * avbridge recognises three libav variants:
5
+ *
6
+ * - **webcodecs** — npm `@libav.js/variant-webcodecs`, ~5 MB. Modern formats
7
+ * only (mp4/mkv/webm/ogg/wav/...) — designed to bridge to WebCodecs.
8
+ *
9
+ * - **default** — npm `@libav.js/variant-default`, ~12 MB. Audio-only build
10
+ * (Opus, FLAC, WAV) despite the name. Useful for audio fallback.
11
+ *
12
+ * - **avbridge** — a custom build produced by `scripts/build-libav.sh` and
13
+ * landing in `vendor/libav/`. Includes the AVI/ASF/FLV/MKV demuxers plus
14
+ * the legacy decoders (WMV3, MPEG-4 Part 2, MS-MPEG4 v1/2/3, VC-1, MPEG-1/2,
15
+ * AC-3/E-AC-3, WMAv1/v2/Pro). This is the only variant that can read AVI;
16
+ * the npm variants are intentionally minimal and ship none of the legacy
17
+ * demuxers.
18
+ *
19
+ * Variant resolution always goes through a runtime URL + `/* @vite-ignore *\/`
20
+ * dynamic import. Static imports trigger Vite's optimized-deps pipeline,
21
+ * which rewrites `import.meta.url` away from the real `dist/` directory and
22
+ * breaks libav's sibling-binary loading.
23
+ */
24
+
25
+ export type LibavVariant = "webcodecs" | "default" | "avbridge";
26
+
27
+ export interface LoadLibavOptions {
28
+ /**
29
+ * Force threading on/off for this load. If unspecified, defaults to
30
+ * "true if `crossOriginIsolated`, otherwise false". Some libav.js code
31
+ * paths (notably the cross-thread reader-device protocol used during
32
+ * `avformat_find_stream_info` for AVI) are unreliable in threaded mode,
33
+ * so probing forces this to `false` while decode keeps it default.
34
+ */
35
+ threads?: boolean;
36
+ }
37
+
38
+ // Cache key includes both variant and threading mode so probe and decode
39
+ // can run different libav instances of the same variant.
40
+ const cache: Map<string, Promise<LibavInstance>> = new Map();
41
+
42
+ function cacheKey(variant: LibavVariant, threads: boolean): string {
43
+ return `${variant}:${threads ? "thr" : "wasm"}`;
44
+ }
45
+
46
+ /**
47
+ * Load (and cache) a libav.js variant. Pass `"webcodecs"` for the small
48
+ * default; pass `"default"` for the audio fallback; pass `"avbridge"` for the
49
+ * custom build that supports AVI/WMV/legacy codecs.
50
+ */
51
+ export function loadLibav(
52
+ variant: LibavVariant = "webcodecs",
53
+ opts: LoadLibavOptions = {},
54
+ ): Promise<LibavInstance> {
55
+ // Threading is OFF by default. The threaded libav.js variant is too
56
+ // fragile in practice for our usage:
57
+ // - Probe (`avformat_find_stream_info` for AVI) throws an `undefined`
58
+ // exception out of `ff_init_demuxer_file`, apparently due to the
59
+ // cross-thread reader-device protocol racing with the main thread.
60
+ // - Decode hits a `TypeError: Cannot read properties of undefined
61
+ // (reading 'apply')` inside libav.js's own worker message handler
62
+ // within seconds of starting — a bug in libav.js's threaded message
63
+ // dispatch that we can't fix from outside.
64
+ //
65
+ // Performance work for the fallback strategy needs to come from elsewhere
66
+ // (WASM SIMD, OffscreenCanvas, larger decode batches) instead of libav's
67
+ // pthreads. Threading can still be force-enabled with
68
+ // `globalThis.AVBRIDGE_LIBAV_THREADS = true` for testing if libav.js fixes
69
+ // those bugs in a future release.
70
+ const env = globalThis as { AVBRIDGE_LIBAV_THREADS?: boolean };
71
+ const wantThreads =
72
+ opts.threads !== undefined
73
+ ? opts.threads
74
+ : env.AVBRIDGE_LIBAV_THREADS === true;
75
+
76
+ const key = cacheKey(variant, wantThreads);
77
+ let entry = cache.get(key);
78
+ if (!entry) {
79
+ entry = loadVariant(variant, wantThreads);
80
+ cache.set(key, entry);
81
+ }
82
+ return entry;
83
+ }
84
+
85
+ async function loadVariant(
86
+ variant: LibavVariant,
87
+ wantThreads: boolean,
88
+ ): Promise<LibavInstance> {
89
+ const key = cacheKey(variant, wantThreads);
90
+ const base = `${libavBaseUrl()}/${variant}`;
91
+ // The custom variant is named `libav-avbridge.mjs`; the npm variants follow
92
+ // the same convention (`libav-webcodecs.mjs`, `libav-default.mjs`).
93
+ const variantUrl = `${base}/libav-${variant}.mjs`;
94
+
95
+ let mod: LoadedVariant;
96
+ try {
97
+ // @ts-ignore runtime URL
98
+ const imported: unknown = await import(/* @vite-ignore */ variantUrl);
99
+ if (!imported || typeof (imported as { LibAV?: unknown }).LibAV !== "function") {
100
+ throw new Error(`module at ${variantUrl} did not export LibAV`);
101
+ }
102
+ mod = imported as LoadedVariant;
103
+ } catch (err) {
104
+ cache.delete(key);
105
+ const hint =
106
+ variant === "avbridge"
107
+ ? `The "avbridge" variant is a custom local build. Run \`./scripts/build-libav.sh\` ` +
108
+ `to produce it (requires Emscripten; ~15-30 min the first time), then ` +
109
+ `\`npm run predemo\` to copy it into the demo asset path.`
110
+ : `Make sure the variant files are present (run \`npm run predemo\` or copy ` +
111
+ `node_modules/@libav.js/variant-${variant}/dist/* into the URL space).`;
112
+ throw new Error(
113
+ `failed to load libav.js "${variant}" variant from ${variantUrl}. ${hint} ` +
114
+ `Original error: ${(err as Error).message || String(err)}`,
115
+ );
116
+ }
117
+
118
+ try {
119
+ const inst = (await mod.LibAV(buildOpts(base, wantThreads))) as LibavInstance;
120
+ await silenceLibavLogs(inst);
121
+ return inst;
122
+ } catch (err) {
123
+ cache.delete(key);
124
+ throw chain(`LibAV() factory failed for "${variant}" variant (threads=${wantThreads})`, err);
125
+ }
126
+ }
127
+
128
+ /**
129
+ * Lower libav's internal log level so the console doesn't get flooded with
130
+ * `[mp3 @ ...] Header missing` and `Video uses a non-standard and wasteful
131
+ * way to store B-frames` warnings on every legacy file. We still get any
132
+ * actual JS-level errors via the normal Error path; this only affects
133
+ * libav's own ffmpeg log channel.
134
+ *
135
+ * AV_LOG_QUIET = -8 (no output at all). If you want to keep fatal errors,
136
+ * use AV_LOG_FATAL = 8 instead.
137
+ */
138
+ async function silenceLibavLogs(inst: LibavInstance): Promise<void> {
139
+ try {
140
+ const setLevel = (inst as { av_log_set_level?: (n: number) => Promise<void> })
141
+ .av_log_set_level;
142
+ if (typeof setLevel === "function") {
143
+ const quiet = (inst as { AV_LOG_QUIET?: number }).AV_LOG_QUIET ?? -8;
144
+ await setLevel(quiet);
145
+ }
146
+ } catch {
147
+ /* not fatal — verbose logs are noise, not an error */
148
+ }
149
+ }
150
+
151
+ function buildOpts(base: string, wantThreads: boolean): Record<string, unknown> {
152
+ // The wantThreads decision is made by `loadLibav()` so callers (probe,
153
+ // decoder) can override per-load. Decode wants pthreads for speed; probe
154
+ // forces them off because libav.js's cross-thread reader-device protocol
155
+ // is unreliable mid-`avformat_find_stream_info` for some AVI files.
156
+ return {
157
+ base,
158
+ nothreads: !wantThreads,
159
+ yesthreads: wantThreads,
160
+ };
161
+ }
162
+
163
+ function libavBaseUrl(): string {
164
+ const override =
165
+ typeof globalThis !== "undefined"
166
+ ? (globalThis as { AVBRIDGE_LIBAV_BASE?: string }).AVBRIDGE_LIBAV_BASE
167
+ : undefined;
168
+ if (override) return override;
169
+ if (typeof location !== "undefined" && location.protocol.startsWith("http")) {
170
+ return `${location.origin}/libav`;
171
+ }
172
+ return "/libav";
173
+ }
174
+
175
+ function chain(message: string, err: unknown): Error {
176
+ const inner = err instanceof Error ? err.message : String(err);
177
+ // eslint-disable-next-line no-console
178
+ console.error(`[avbridge] ${message}:`, err);
179
+ return new Error(`${message}: ${inner || "(no message — see browser console)"}`);
180
+ }
181
+
182
+ interface LoadedVariant {
183
+ LibAV: (opts?: Record<string, unknown>) => Promise<Record<string, unknown>>;
184
+ }
185
+
186
+ /** Loose structural type — the AVI probe and the fallback decoder add fields. */
187
+ export type LibavInstance = Record<string, unknown> & {
188
+ mkreadaheadfile(name: string, blob: Blob): Promise<void>;
189
+ unlinkreadaheadfile(name: string): Promise<void>;
190
+ };
@@ -0,0 +1,43 @@
1
+ import type { MediaContext, AudioCodec, VideoCodec } from "../../types.js";
2
+ import type { LibavVariant } from "./libav-loader.js";
3
+
4
+ /**
5
+ * Decide which libav.js variant to load for a given media context.
6
+ *
7
+ * - **webcodecs** (~5 MB, npm) — modern formats only, designed for the
8
+ * WebCodecs bridge. Used when the codec is browser-supported and we just
9
+ * need libav.js for demuxing or as a parser source.
10
+ *
11
+ * - **avbridge** (custom build, vendor/libav/) — has the AVI/ASF/FLV demuxers
12
+ * and the legacy decoders (WMV3, MPEG-4 Part 2, VC-1, MS-MPEG4 v1/2/3,
13
+ * AC-3, WMA*). Required for any of those formats; the npm variants ship
14
+ * none of them.
15
+ *
16
+ * Rule: pick "avbridge" if either the container or any codec is one only the
17
+ * custom build can handle. Otherwise pick "webcodecs".
18
+ */
19
+
20
+ const LEGACY_CONTAINERS = new Set(["avi", "asf", "flv"]);
21
+
22
+ const LEGACY_VIDEO_CODECS = new Set<VideoCodec>([
23
+ "wmv3",
24
+ "vc1",
25
+ "mpeg4", // MPEG-4 Part 2 / DivX / Xvid
26
+ "rv40",
27
+ "mpeg2",
28
+ "mpeg1",
29
+ "theora",
30
+ ]);
31
+
32
+ const LEGACY_AUDIO_CODECS = new Set<AudioCodec>(["wmav2", "wmapro", "ac3", "eac3"]);
33
+
34
+ export function pickLibavVariant(ctx: MediaContext): LibavVariant {
35
+ if (LEGACY_CONTAINERS.has(ctx.container)) return "avbridge";
36
+ for (const v of ctx.videoTracks) {
37
+ if (LEGACY_VIDEO_CODECS.has(v.codec)) return "avbridge";
38
+ }
39
+ for (const a of ctx.audioTracks) {
40
+ if (LEGACY_AUDIO_CODECS.has(a.codec)) return "avbridge";
41
+ }
42
+ return "webcodecs";
43
+ }
@@ -0,0 +1,216 @@
1
+ import type { ClockSource } from "./audio-output.js";
2
+
3
+ /**
4
+ * Renders decoded `VideoFrame`s into a 2D canvas overlaid on the user's
5
+ * `<video>` element. The fallback strategy never assigns a src to the video,
6
+ * so we hide it and put the canvas in its place visually.
7
+ *
8
+ * The renderer has two modes:
9
+ *
10
+ * 1. **Pre-roll** — `clock.isPlaying()` is false. The very first decoded
11
+ * frame is painted as a "poster" so the user sees something while audio
12
+ * buffers; subsequent frames stay queued without being dropped.
13
+ *
14
+ * 2. **Synced** — `clock.isPlaying()` is true. On each rAF tick, find the
15
+ * latest frame whose timestamp ≤ `clock.now() + lookahead` and paint it.
16
+ * Drop any older frames as "late."
17
+ *
18
+ * The pre-roll behavior is what fixes the cold-start "first minute is all
19
+ * dropped" problem: without it, the wall clock raced ahead while the
20
+ * decoder was still warming up, and every frame was already in the past by
21
+ * the time it landed in the queue.
22
+ */
23
+ export class VideoRenderer {
24
+ private canvas: HTMLCanvasElement;
25
+ private ctx: CanvasRenderingContext2D;
26
+ private queue: VideoFrame[] = [];
27
+ private rafHandle: number | null = null;
28
+ private destroyed = false;
29
+
30
+ private framesPainted = 0;
31
+ private framesDroppedLate = 0;
32
+ private framesDroppedOverflow = 0;
33
+ private prerolled = false;
34
+ /** Wall-clock time of the last paint, in ms (performance.now()). */
35
+ private lastPaintWall = 0;
36
+ /** Minimum ms between paints — paces video at roughly source fps. */
37
+ private paintIntervalMs: number;
38
+
39
+ /** Resolves once the first decoded frame has been enqueued. */
40
+ readonly firstFrameReady: Promise<void>;
41
+ private resolveFirstFrame!: () => void;
42
+
43
+ constructor(
44
+ private readonly target: HTMLVideoElement,
45
+ private readonly clock: ClockSource,
46
+ fps = 30,
47
+ ) {
48
+ this.paintIntervalMs = Math.max(1, 1000 / fps);
49
+ this.firstFrameReady = new Promise<void>((resolve) => {
50
+ this.resolveFirstFrame = resolve;
51
+ });
52
+
53
+ this.canvas = document.createElement("canvas");
54
+ this.canvas.style.cssText =
55
+ "position:absolute;left:0;top:0;width:100%;height:100%;background:black;";
56
+ const parent = target.parentElement;
57
+ if (parent && getComputedStyle(parent).position === "static") {
58
+ parent.style.position = "relative";
59
+ }
60
+ parent?.insertBefore(this.canvas, target);
61
+ target.style.visibility = "hidden";
62
+
63
+ const ctx = this.canvas.getContext("2d");
64
+ if (!ctx) throw new Error("video renderer: failed to acquire 2D context");
65
+ this.ctx = ctx;
66
+
67
+ this.tick = this.tick.bind(this);
68
+ this.rafHandle = requestAnimationFrame(this.tick);
69
+ }
70
+
71
+ /** True once at least one frame has been enqueued. */
72
+ hasFrames(): boolean {
73
+ return this.queue.length > 0 || this.framesPainted > 0;
74
+ }
75
+
76
+ /** Current depth of the frame queue. Used by the decoder for backpressure. */
77
+ queueDepth(): number {
78
+ return this.queue.length;
79
+ }
80
+
81
+ /**
82
+ * Soft cap for decoder backpressure. The decoder pump throttles when
83
+ * `queueDepth() >= queueHighWater`. Set high enough that normal decode
84
+ * bursts don't trigger the renderer's overflow-drop loop (which runs at
85
+ * every paint), but low enough that the decoder doesn't run unboundedly
86
+ * ahead. The hard cap in `enqueue()` is 64.
87
+ */
88
+ readonly queueHighWater = 30;
89
+
90
+ enqueue(frame: VideoFrame): void {
91
+ if (this.destroyed) {
92
+ frame.close();
93
+ return;
94
+ }
95
+ this.queue.push(frame);
96
+ if (this.queue.length === 1 && this.framesPainted === 0) {
97
+ this.resolveFirstFrame();
98
+ }
99
+ // Hard cap. Should rarely trigger because the decoder backs off at
100
+ // queueHighWater (30) and the drift correction trims gently. This is
101
+ // the last-resort defense against runaway producers.
102
+ while (this.queue.length > 60) {
103
+ this.queue.shift()?.close();
104
+ this.framesDroppedOverflow++;
105
+ }
106
+ }
107
+
108
+ private tick(): void {
109
+ if (this.destroyed) return;
110
+ this.rafHandle = requestAnimationFrame(this.tick);
111
+
112
+ if (this.queue.length === 0) return;
113
+
114
+ const playing = this.clock.isPlaying();
115
+
116
+ // Pre-roll: paint the very first frame as a poster while audio buffers.
117
+ if (!playing) {
118
+ if (!this.prerolled) {
119
+ const head = this.queue.shift()!;
120
+ this.paint(head);
121
+ head.close();
122
+ this.prerolled = true;
123
+ this.lastPaintWall = performance.now();
124
+ }
125
+ return;
126
+ }
127
+
128
+ // Wall-clock-paced painting with coarse A/V drift correction.
129
+ //
130
+ // Base policy: paint one frame every `paintIntervalMs` of wall time,
131
+ // regardless of the frame's synthetic timestamp. This avoids the old
132
+ // per-frame audio-gate that caused massive overflow during decode bursts.
133
+ //
134
+ // Drift correction (runs every ~1 sec):
135
+ // - Video > 150 ms behind audio → drop one frame (catch up)
136
+ // - Video > 150 ms ahead of audio → skip one paint (let audio catch up)
137
+ //
138
+ // This keeps long-run sync robust even for legacy AVI/DivX with messy
139
+ // timestamps, packed B-frames, and odd frame durations. The correction
140
+ // is deliberately gentle (one frame at a time) so it doesn't cause
141
+ // visible stuttering.
142
+ const wallNow = performance.now();
143
+ if (wallNow - this.lastPaintWall < this.paintIntervalMs - 2) return;
144
+
145
+ if (this.queue.length === 0) return;
146
+
147
+ // Coarse drift correction: compare the head frame's timestamp to
148
+ // audio.now() every ~1 sec (every 30 frames at 30fps). The frame ts
149
+ // and audio.now() are both in seconds of media time. Drift beyond
150
+ // 150ms triggers gentle correction — one frame per check, not a burst.
151
+ if (this.framesPainted > 0 && this.framesPainted % 30 === 0) {
152
+ const audioNowUs = this.clock.now() * 1_000_000;
153
+ const headTs = this.queue[0].timestamp ?? 0;
154
+ const driftUs = headTs - audioNowUs;
155
+
156
+ if (driftUs < -150_000) {
157
+ // Video behind audio by > 150ms — drop one frame to catch up.
158
+ this.queue.shift()?.close();
159
+ this.framesDroppedLate++;
160
+ if (this.queue.length === 0) return;
161
+ } else if (driftUs > 150_000) {
162
+ // Video ahead of audio by > 150ms — skip this paint cycle.
163
+ return;
164
+ }
165
+ }
166
+
167
+ const frame = this.queue.shift()!;
168
+ this.paint(frame);
169
+ frame.close();
170
+ this.lastPaintWall = wallNow;
171
+ }
172
+
173
+ private paint(frame: VideoFrame): void {
174
+ if (
175
+ this.canvas.width !== frame.displayWidth ||
176
+ this.canvas.height !== frame.displayHeight
177
+ ) {
178
+ this.canvas.width = frame.displayWidth;
179
+ this.canvas.height = frame.displayHeight;
180
+ }
181
+ try {
182
+ this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
183
+ this.framesPainted++;
184
+ } catch (err) {
185
+ // Log only once so a structurally broken frame format doesn't spam
186
+ // the console at 60 Hz, but we still find out about it.
187
+ if (this.framesPainted === 0 && this.framesDroppedLate === 0) {
188
+ // eslint-disable-next-line no-console
189
+ console.warn("[avbridge] canvas drawImage failed:", err);
190
+ }
191
+ }
192
+ }
193
+
194
+ /** Discard all queued frames. Used by seek to drop stale buffers. */
195
+ flush(): void {
196
+ while (this.queue.length > 0) this.queue.shift()?.close();
197
+ this.prerolled = false;
198
+ }
199
+
200
+ stats(): Record<string, unknown> {
201
+ return {
202
+ framesPainted: this.framesPainted,
203
+ framesDroppedLate: this.framesDroppedLate,
204
+ framesDroppedOverflow: this.framesDroppedOverflow,
205
+ queueDepth: this.queue.length,
206
+ };
207
+ }
208
+
209
+ destroy(): void {
210
+ this.destroyed = true;
211
+ if (this.rafHandle != null) cancelAnimationFrame(this.rafHandle);
212
+ this.flush();
213
+ this.canvas.remove();
214
+ this.target.style.visibility = "";
215
+ }
216
+ }