avbridge 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +120 -0
- package/LICENSE +21 -0
- package/README.md +415 -0
- package/dist/avi-M5B4SHRM.cjs +164 -0
- package/dist/avi-M5B4SHRM.cjs.map +1 -0
- package/dist/avi-POCGZ4JX.js +162 -0
- package/dist/avi-POCGZ4JX.js.map +1 -0
- package/dist/chunk-5ISVAODK.js +80 -0
- package/dist/chunk-5ISVAODK.js.map +1 -0
- package/dist/chunk-F7YS2XOA.cjs +2966 -0
- package/dist/chunk-F7YS2XOA.cjs.map +1 -0
- package/dist/chunk-FKM7QBZU.js +2957 -0
- package/dist/chunk-FKM7QBZU.js.map +1 -0
- package/dist/chunk-J5MCMN3S.js +27 -0
- package/dist/chunk-J5MCMN3S.js.map +1 -0
- package/dist/chunk-L4NPOJ36.cjs +180 -0
- package/dist/chunk-L4NPOJ36.cjs.map +1 -0
- package/dist/chunk-NZU7W256.cjs +29 -0
- package/dist/chunk-NZU7W256.cjs.map +1 -0
- package/dist/chunk-PQTZS7OA.js +147 -0
- package/dist/chunk-PQTZS7OA.js.map +1 -0
- package/dist/chunk-WD2ZNQA7.js +177 -0
- package/dist/chunk-WD2ZNQA7.js.map +1 -0
- package/dist/chunk-Y5FYF5KG.cjs +153 -0
- package/dist/chunk-Y5FYF5KG.cjs.map +1 -0
- package/dist/chunk-Z2FJ5TJC.cjs +82 -0
- package/dist/chunk-Z2FJ5TJC.cjs.map +1 -0
- package/dist/element.cjs +433 -0
- package/dist/element.cjs.map +1 -0
- package/dist/element.d.cts +158 -0
- package/dist/element.d.ts +158 -0
- package/dist/element.js +431 -0
- package/dist/element.js.map +1 -0
- package/dist/index.cjs +576 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +80 -0
- package/dist/index.d.ts +80 -0
- package/dist/index.js +554 -0
- package/dist/index.js.map +1 -0
- package/dist/libav-http-reader-FPYDBMYK.cjs +16 -0
- package/dist/libav-http-reader-FPYDBMYK.cjs.map +1 -0
- package/dist/libav-http-reader-NQJVY273.js +3 -0
- package/dist/libav-http-reader-NQJVY273.js.map +1 -0
- package/dist/libav-import-2JURFHEW.js +8 -0
- package/dist/libav-import-2JURFHEW.js.map +1 -0
- package/dist/libav-import-GST2AMPL.cjs +30 -0
- package/dist/libav-import-GST2AMPL.cjs.map +1 -0
- package/dist/libav-loader-KA2MAWLM.js +3 -0
- package/dist/libav-loader-KA2MAWLM.js.map +1 -0
- package/dist/libav-loader-ZHOERPHW.cjs +12 -0
- package/dist/libav-loader-ZHOERPHW.cjs.map +1 -0
- package/dist/player-BBwbCkdL.d.cts +365 -0
- package/dist/player-BBwbCkdL.d.ts +365 -0
- package/dist/source-SC6ZEQYR.cjs +28 -0
- package/dist/source-SC6ZEQYR.cjs.map +1 -0
- package/dist/source-ZFS4H7J3.js +3 -0
- package/dist/source-ZFS4H7J3.js.map +1 -0
- package/dist/variant-routing-GOHB2RZN.cjs +12 -0
- package/dist/variant-routing-GOHB2RZN.cjs.map +1 -0
- package/dist/variant-routing-JOBWXYKD.js +3 -0
- package/dist/variant-routing-JOBWXYKD.js.map +1 -0
- package/package.json +95 -0
- package/src/classify/index.ts +1 -0
- package/src/classify/rules.ts +214 -0
- package/src/convert/index.ts +2 -0
- package/src/convert/remux.ts +522 -0
- package/src/convert/transcode.ts +329 -0
- package/src/diagnostics.ts +99 -0
- package/src/element/avbridge-player.ts +576 -0
- package/src/element.ts +19 -0
- package/src/events.ts +71 -0
- package/src/index.ts +42 -0
- package/src/libav-stubs.d.ts +24 -0
- package/src/player.ts +455 -0
- package/src/plugins/builtin.ts +37 -0
- package/src/plugins/registry.ts +32 -0
- package/src/probe/avi.ts +242 -0
- package/src/probe/index.ts +59 -0
- package/src/probe/mediabunny.ts +194 -0
- package/src/strategies/fallback/audio-output.ts +293 -0
- package/src/strategies/fallback/clock.ts +7 -0
- package/src/strategies/fallback/decoder.ts +660 -0
- package/src/strategies/fallback/index.ts +170 -0
- package/src/strategies/fallback/libav-import.ts +27 -0
- package/src/strategies/fallback/libav-loader.ts +190 -0
- package/src/strategies/fallback/variant-routing.ts +43 -0
- package/src/strategies/fallback/video-renderer.ts +216 -0
- package/src/strategies/hybrid/decoder.ts +641 -0
- package/src/strategies/hybrid/index.ts +139 -0
- package/src/strategies/native.ts +107 -0
- package/src/strategies/remux/annexb.ts +112 -0
- package/src/strategies/remux/index.ts +79 -0
- package/src/strategies/remux/mse.ts +234 -0
- package/src/strategies/remux/pipeline.ts +254 -0
- package/src/subtitles/index.ts +91 -0
- package/src/subtitles/render.ts +62 -0
- package/src/subtitles/srt.ts +62 -0
- package/src/subtitles/vtt.ts +5 -0
- package/src/types-shim.d.ts +3 -0
- package/src/types.ts +360 -0
- package/src/util/codec-strings.ts +86 -0
- package/src/util/libav-http-reader.ts +315 -0
- package/src/util/source.ts +274 -0
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
import type { MediaContext, PlaybackSession } from "../../types.js";
|
|
2
|
+
import { VideoRenderer } from "./video-renderer.js";
|
|
3
|
+
import { AudioOutput } from "./audio-output.js";
|
|
4
|
+
import { startDecoder, type DecoderHandles } from "./decoder.js";
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Fallback strategy session.
|
|
8
|
+
*
|
|
9
|
+
* Owns the orchestration between the libav decoder, the audio scheduler,
|
|
10
|
+
* and the canvas renderer. Three things make this non-trivial:
|
|
11
|
+
*
|
|
12
|
+
* 1. **Cold-start ready gate.** When `play()` is called, we wait until the
|
|
13
|
+
* audio scheduler has buffered enough audio (≥ 300 ms) AND the renderer
|
|
14
|
+
* has at least one decoded video frame, before actually telling the
|
|
15
|
+
* audio context to start. Without this gate, audio and the wall clock
|
|
16
|
+
* race ahead of the still-warming-up software decoder, and every video
|
|
17
|
+
* frame lands "in the past" and gets dropped.
|
|
18
|
+
*
|
|
19
|
+
* 2. **Pause / resume.** The audio context is suspended on pause and
|
|
20
|
+
* resumed on play. The media-time anchor is preserved across the
|
|
21
|
+
* suspend so the clock is continuous.
|
|
22
|
+
*
|
|
23
|
+
* 3. **Seek.** Pauses the audio scheduler, asks the decoder to cancel its
|
|
24
|
+
* current pump and `av_seek_frame` to the target, resets the audio
|
|
25
|
+
* output's media-time anchor to the seek target, flushes the renderer
|
|
26
|
+
* queue, then re-enters the ready gate. If we were playing before the
|
|
27
|
+
* seek, we automatically resume once the buffer fills.
|
|
28
|
+
*
|
|
29
|
+
* The unified player API on top of this just sees `play() / pause() /
|
|
30
|
+
* seek(t)` — none of the buffering choreography leaks out.
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
const READY_AUDIO_BUFFER_SECONDS = 0.3;
|
|
34
|
+
const READY_TIMEOUT_SECONDS = 10;
|
|
35
|
+
|
|
36
|
+
export async function createFallbackSession(
|
|
37
|
+
ctx: MediaContext,
|
|
38
|
+
target: HTMLVideoElement,
|
|
39
|
+
): Promise<PlaybackSession> {
|
|
40
|
+
// Normalize the source so URL inputs go through the libav HTTP block
|
|
41
|
+
// reader instead of being buffered into memory.
|
|
42
|
+
const { normalizeSource } = await import("../../util/source.js");
|
|
43
|
+
const source = await normalizeSource(ctx.source);
|
|
44
|
+
|
|
45
|
+
const fps = ctx.videoTracks[0]?.fps ?? 30;
|
|
46
|
+
const audio = new AudioOutput();
|
|
47
|
+
const renderer = new VideoRenderer(target, audio, fps);
|
|
48
|
+
|
|
49
|
+
let handles: DecoderHandles;
|
|
50
|
+
try {
|
|
51
|
+
handles = await startDecoder({
|
|
52
|
+
source,
|
|
53
|
+
filename: ctx.name ?? "input.bin",
|
|
54
|
+
context: ctx,
|
|
55
|
+
renderer,
|
|
56
|
+
audio,
|
|
57
|
+
});
|
|
58
|
+
} catch (err) {
|
|
59
|
+
audio.destroy();
|
|
60
|
+
renderer.destroy();
|
|
61
|
+
throw err;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// Patch the <video> element so the unified player layer (which polls
|
|
65
|
+
// `target.currentTime` for `timeupdate` events and lets users assign to
|
|
66
|
+
// it for seeks) gets the right values from the fallback strategy.
|
|
67
|
+
Object.defineProperty(target, "currentTime", {
|
|
68
|
+
configurable: true,
|
|
69
|
+
get: () => audio.now(),
|
|
70
|
+
set: (v: number) => {
|
|
71
|
+
// Fire-and-forget — the user is expected to await player.seek() if
|
|
72
|
+
// they want to know when the seek completes.
|
|
73
|
+
void doSeek(v);
|
|
74
|
+
},
|
|
75
|
+
});
|
|
76
|
+
// Mirror duration so the demo's controls can use target.duration too.
|
|
77
|
+
if (ctx.duration && Number.isFinite(ctx.duration)) {
|
|
78
|
+
Object.defineProperty(target, "duration", {
|
|
79
|
+
configurable: true,
|
|
80
|
+
get: () => ctx.duration ?? NaN,
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Wait until the decoder has produced enough buffered output to start
|
|
86
|
+
* playback smoothly. Returns early on timeout so we don't hang forever
|
|
87
|
+
* if the decoder is producing nothing (e.g. immediately past EOF after
|
|
88
|
+
* a seek to the end).
|
|
89
|
+
*/
|
|
90
|
+
async function waitForBuffer(): Promise<void> {
|
|
91
|
+
const start = performance.now();
|
|
92
|
+
while (true) {
|
|
93
|
+
const audioReady = audio.isNoAudio() || audio.bufferAhead() >= READY_AUDIO_BUFFER_SECONDS;
|
|
94
|
+
if (audioReady && renderer.hasFrames()) {
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
if ((performance.now() - start) / 1000 > READY_TIMEOUT_SECONDS) {
|
|
98
|
+
// Give up waiting; play whatever we have.
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
async function doSeek(timeSec: number): Promise<void> {
|
|
106
|
+
const wasPlaying = audio.isPlaying();
|
|
107
|
+
// 1. Stop audio (suspend ctx + capture media time).
|
|
108
|
+
await audio.pause().catch(() => {});
|
|
109
|
+
// 2. Tell the decoder to cancel its pump and seek the demuxer.
|
|
110
|
+
await handles.seek(timeSec).catch((err) =>
|
|
111
|
+
console.warn("[avbridge] decoder seek failed:", err),
|
|
112
|
+
);
|
|
113
|
+
// 3. Reset audio + renderer to the new media time. New samples from
|
|
114
|
+
// the decoder will queue against this anchor.
|
|
115
|
+
await audio.reset(timeSec);
|
|
116
|
+
renderer.flush();
|
|
117
|
+
// 4. If we were playing, wait for the buffer to fill again and then
|
|
118
|
+
// resume. If we were paused, leave it paused at the new position.
|
|
119
|
+
if (wasPlaying) {
|
|
120
|
+
await waitForBuffer();
|
|
121
|
+
await audio.start();
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
return {
|
|
126
|
+
strategy: "fallback",
|
|
127
|
+
|
|
128
|
+
async play() {
|
|
129
|
+
// Either a cold start (very first play() call) or a resume from
|
|
130
|
+
// pause. AudioOutput.start() handles both.
|
|
131
|
+
if (!audio.isPlaying()) {
|
|
132
|
+
await waitForBuffer();
|
|
133
|
+
await audio.start();
|
|
134
|
+
}
|
|
135
|
+
},
|
|
136
|
+
|
|
137
|
+
pause() {
|
|
138
|
+
void audio.pause();
|
|
139
|
+
},
|
|
140
|
+
|
|
141
|
+
async seek(time) {
|
|
142
|
+
await doSeek(time);
|
|
143
|
+
},
|
|
144
|
+
|
|
145
|
+
async setAudioTrack(_id) {
|
|
146
|
+
// Multi-track audio is post-MVP for the fallback strategy.
|
|
147
|
+
},
|
|
148
|
+
|
|
149
|
+
async setSubtitleTrack(_id) {
|
|
150
|
+
// Subtitle overlay support is post-MVP for the fallback strategy.
|
|
151
|
+
},
|
|
152
|
+
|
|
153
|
+
getCurrentTime() {
|
|
154
|
+
return audio.now();
|
|
155
|
+
},
|
|
156
|
+
async destroy() {
|
|
157
|
+
await handles.destroy();
|
|
158
|
+
renderer.destroy();
|
|
159
|
+
audio.destroy();
|
|
160
|
+
try {
|
|
161
|
+
delete (target as unknown as Record<string, unknown>).currentTime;
|
|
162
|
+
delete (target as unknown as Record<string, unknown>).duration;
|
|
163
|
+
} catch { /* ignore */ }
|
|
164
|
+
},
|
|
165
|
+
|
|
166
|
+
getRuntimeStats() {
|
|
167
|
+
return handles.stats();
|
|
168
|
+
},
|
|
169
|
+
};
|
|
170
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Static-import wrapper for the libavjs-webcodecs-bridge optional peer dep.
|
|
3
|
+
*
|
|
4
|
+
* The variant itself is **not** imported here — it's loaded via a runtime
|
|
5
|
+
* dynamic import with `/* @vite-ignore *\/` from `libav-loader.ts`, so the
|
|
6
|
+
* variant's `.mjs` file is never touched by Vite's transform pipeline (which
|
|
7
|
+
* would otherwise pre-bundle it and break the `import.meta.url`-based path
|
|
8
|
+
* resolution it uses to find its sibling .wasm files).
|
|
9
|
+
*
|
|
10
|
+
* The bridge has no such issue — it's pure JS and doesn't reference sibling
|
|
11
|
+
* binaries — so a normal static import is fine here.
|
|
12
|
+
*
|
|
13
|
+
* TypeScript resolves `libavjs-webcodecs-bridge` via the `paths` mapping in
|
|
14
|
+
* tsconfig.json which redirects to `src/libav-stubs.d.ts`, sidestepping the
|
|
15
|
+
* polyfill source files that don't typecheck under TS 5.7.
|
|
16
|
+
*/
|
|
17
|
+
import * as bridge from "libavjs-webcodecs-bridge";
|
|
18
|
+
|
|
19
|
+
export const libavBridge: BridgeModule = bridge as unknown as BridgeModule;
|
|
20
|
+
|
|
21
|
+
export interface BridgeModule {
|
|
22
|
+
videoStreamToConfig(libav: unknown, stream: unknown): Promise<VideoDecoderConfig | null>;
|
|
23
|
+
audioStreamToConfig(libav: unknown, stream: unknown): Promise<AudioDecoderConfig | null>;
|
|
24
|
+
packetToEncodedVideoChunk(pkt: unknown, stream: unknown): EncodedVideoChunk;
|
|
25
|
+
packetToEncodedAudioChunk(pkt: unknown, stream: unknown): EncodedAudioChunk;
|
|
26
|
+
libavFrameToVideoFrame?(frame: unknown, stream: unknown): VideoFrame | null;
|
|
27
|
+
}
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lazy libav.js loader supporting multiple variants.
|
|
3
|
+
*
|
|
4
|
+
* avbridge recognises three libav variants:
|
|
5
|
+
*
|
|
6
|
+
* - **webcodecs** — npm `@libav.js/variant-webcodecs`, ~5 MB. Modern formats
|
|
7
|
+
* only (mp4/mkv/webm/ogg/wav/...) — designed to bridge to WebCodecs.
|
|
8
|
+
*
|
|
9
|
+
* - **default** — npm `@libav.js/variant-default`, ~12 MB. Audio-only build
|
|
10
|
+
* (Opus, FLAC, WAV) despite the name. Useful for audio fallback.
|
|
11
|
+
*
|
|
12
|
+
* - **avbridge** — a custom build produced by `scripts/build-libav.sh` and
|
|
13
|
+
* landing in `vendor/libav/`. Includes the AVI/ASF/FLV/MKV demuxers plus
|
|
14
|
+
* the legacy decoders (WMV3, MPEG-4 Part 2, MS-MPEG4 v1/2/3, VC-1, MPEG-1/2,
|
|
15
|
+
* AC-3/E-AC-3, WMAv1/v2/Pro). This is the only variant that can read AVI;
|
|
16
|
+
* the npm variants are intentionally minimal and ship none of the legacy
|
|
17
|
+
* demuxers.
|
|
18
|
+
*
|
|
19
|
+
* Variant resolution always goes through a runtime URL + `/* @vite-ignore *\/`
|
|
20
|
+
* dynamic import. Static imports trigger Vite's optimized-deps pipeline,
|
|
21
|
+
* which rewrites `import.meta.url` away from the real `dist/` directory and
|
|
22
|
+
* breaks libav's sibling-binary loading.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
export type LibavVariant = "webcodecs" | "default" | "avbridge";
|
|
26
|
+
|
|
27
|
+
export interface LoadLibavOptions {
|
|
28
|
+
/**
|
|
29
|
+
* Force threading on/off for this load. If unspecified, defaults to
|
|
30
|
+
* "true if `crossOriginIsolated`, otherwise false". Some libav.js code
|
|
31
|
+
* paths (notably the cross-thread reader-device protocol used during
|
|
32
|
+
* `avformat_find_stream_info` for AVI) are unreliable in threaded mode,
|
|
33
|
+
* so probing forces this to `false` while decode keeps it default.
|
|
34
|
+
*/
|
|
35
|
+
threads?: boolean;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// Cache key includes both variant and threading mode so probe and decode
|
|
39
|
+
// can run different libav instances of the same variant.
|
|
40
|
+
const cache: Map<string, Promise<LibavInstance>> = new Map();
|
|
41
|
+
|
|
42
|
+
function cacheKey(variant: LibavVariant, threads: boolean): string {
|
|
43
|
+
return `${variant}:${threads ? "thr" : "wasm"}`;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Load (and cache) a libav.js variant. Pass `"webcodecs"` for the small
|
|
48
|
+
* default; pass `"default"` for the audio fallback; pass `"avbridge"` for the
|
|
49
|
+
* custom build that supports AVI/WMV/legacy codecs.
|
|
50
|
+
*/
|
|
51
|
+
export function loadLibav(
|
|
52
|
+
variant: LibavVariant = "webcodecs",
|
|
53
|
+
opts: LoadLibavOptions = {},
|
|
54
|
+
): Promise<LibavInstance> {
|
|
55
|
+
// Threading is OFF by default. The threaded libav.js variant is too
|
|
56
|
+
// fragile in practice for our usage:
|
|
57
|
+
// - Probe (`avformat_find_stream_info` for AVI) throws an `undefined`
|
|
58
|
+
// exception out of `ff_init_demuxer_file`, apparently due to the
|
|
59
|
+
// cross-thread reader-device protocol racing with the main thread.
|
|
60
|
+
// - Decode hits a `TypeError: Cannot read properties of undefined
|
|
61
|
+
// (reading 'apply')` inside libav.js's own worker message handler
|
|
62
|
+
// within seconds of starting — a bug in libav.js's threaded message
|
|
63
|
+
// dispatch that we can't fix from outside.
|
|
64
|
+
//
|
|
65
|
+
// Performance work for the fallback strategy needs to come from elsewhere
|
|
66
|
+
// (WASM SIMD, OffscreenCanvas, larger decode batches) instead of libav's
|
|
67
|
+
// pthreads. Threading can still be force-enabled with
|
|
68
|
+
// `globalThis.AVBRIDGE_LIBAV_THREADS = true` for testing if libav.js fixes
|
|
69
|
+
// those bugs in a future release.
|
|
70
|
+
const env = globalThis as { AVBRIDGE_LIBAV_THREADS?: boolean };
|
|
71
|
+
const wantThreads =
|
|
72
|
+
opts.threads !== undefined
|
|
73
|
+
? opts.threads
|
|
74
|
+
: env.AVBRIDGE_LIBAV_THREADS === true;
|
|
75
|
+
|
|
76
|
+
const key = cacheKey(variant, wantThreads);
|
|
77
|
+
let entry = cache.get(key);
|
|
78
|
+
if (!entry) {
|
|
79
|
+
entry = loadVariant(variant, wantThreads);
|
|
80
|
+
cache.set(key, entry);
|
|
81
|
+
}
|
|
82
|
+
return entry;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async function loadVariant(
|
|
86
|
+
variant: LibavVariant,
|
|
87
|
+
wantThreads: boolean,
|
|
88
|
+
): Promise<LibavInstance> {
|
|
89
|
+
const key = cacheKey(variant, wantThreads);
|
|
90
|
+
const base = `${libavBaseUrl()}/${variant}`;
|
|
91
|
+
// The custom variant is named `libav-avbridge.mjs`; the npm variants follow
|
|
92
|
+
// the same convention (`libav-webcodecs.mjs`, `libav-default.mjs`).
|
|
93
|
+
const variantUrl = `${base}/libav-${variant}.mjs`;
|
|
94
|
+
|
|
95
|
+
let mod: LoadedVariant;
|
|
96
|
+
try {
|
|
97
|
+
// @ts-ignore runtime URL
|
|
98
|
+
const imported: unknown = await import(/* @vite-ignore */ variantUrl);
|
|
99
|
+
if (!imported || typeof (imported as { LibAV?: unknown }).LibAV !== "function") {
|
|
100
|
+
throw new Error(`module at ${variantUrl} did not export LibAV`);
|
|
101
|
+
}
|
|
102
|
+
mod = imported as LoadedVariant;
|
|
103
|
+
} catch (err) {
|
|
104
|
+
cache.delete(key);
|
|
105
|
+
const hint =
|
|
106
|
+
variant === "avbridge"
|
|
107
|
+
? `The "avbridge" variant is a custom local build. Run \`./scripts/build-libav.sh\` ` +
|
|
108
|
+
`to produce it (requires Emscripten; ~15-30 min the first time), then ` +
|
|
109
|
+
`\`npm run predemo\` to copy it into the demo asset path.`
|
|
110
|
+
: `Make sure the variant files are present (run \`npm run predemo\` or copy ` +
|
|
111
|
+
`node_modules/@libav.js/variant-${variant}/dist/* into the URL space).`;
|
|
112
|
+
throw new Error(
|
|
113
|
+
`failed to load libav.js "${variant}" variant from ${variantUrl}. ${hint} ` +
|
|
114
|
+
`Original error: ${(err as Error).message || String(err)}`,
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
try {
|
|
119
|
+
const inst = (await mod.LibAV(buildOpts(base, wantThreads))) as LibavInstance;
|
|
120
|
+
await silenceLibavLogs(inst);
|
|
121
|
+
return inst;
|
|
122
|
+
} catch (err) {
|
|
123
|
+
cache.delete(key);
|
|
124
|
+
throw chain(`LibAV() factory failed for "${variant}" variant (threads=${wantThreads})`, err);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Lower libav's internal log level so the console doesn't get flooded with
|
|
130
|
+
* `[mp3 @ ...] Header missing` and `Video uses a non-standard and wasteful
|
|
131
|
+
* way to store B-frames` warnings on every legacy file. We still get any
|
|
132
|
+
* actual JS-level errors via the normal Error path; this only affects
|
|
133
|
+
* libav's own ffmpeg log channel.
|
|
134
|
+
*
|
|
135
|
+
* AV_LOG_QUIET = -8 (no output at all). If you want to keep fatal errors,
|
|
136
|
+
* use AV_LOG_FATAL = 8 instead.
|
|
137
|
+
*/
|
|
138
|
+
async function silenceLibavLogs(inst: LibavInstance): Promise<void> {
|
|
139
|
+
try {
|
|
140
|
+
const setLevel = (inst as { av_log_set_level?: (n: number) => Promise<void> })
|
|
141
|
+
.av_log_set_level;
|
|
142
|
+
if (typeof setLevel === "function") {
|
|
143
|
+
const quiet = (inst as { AV_LOG_QUIET?: number }).AV_LOG_QUIET ?? -8;
|
|
144
|
+
await setLevel(quiet);
|
|
145
|
+
}
|
|
146
|
+
} catch {
|
|
147
|
+
/* not fatal — verbose logs are noise, not an error */
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function buildOpts(base: string, wantThreads: boolean): Record<string, unknown> {
|
|
152
|
+
// The wantThreads decision is made by `loadLibav()` so callers (probe,
|
|
153
|
+
// decoder) can override per-load. Decode wants pthreads for speed; probe
|
|
154
|
+
// forces them off because libav.js's cross-thread reader-device protocol
|
|
155
|
+
// is unreliable mid-`avformat_find_stream_info` for some AVI files.
|
|
156
|
+
return {
|
|
157
|
+
base,
|
|
158
|
+
nothreads: !wantThreads,
|
|
159
|
+
yesthreads: wantThreads,
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function libavBaseUrl(): string {
|
|
164
|
+
const override =
|
|
165
|
+
typeof globalThis !== "undefined"
|
|
166
|
+
? (globalThis as { AVBRIDGE_LIBAV_BASE?: string }).AVBRIDGE_LIBAV_BASE
|
|
167
|
+
: undefined;
|
|
168
|
+
if (override) return override;
|
|
169
|
+
if (typeof location !== "undefined" && location.protocol.startsWith("http")) {
|
|
170
|
+
return `${location.origin}/libav`;
|
|
171
|
+
}
|
|
172
|
+
return "/libav";
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function chain(message: string, err: unknown): Error {
|
|
176
|
+
const inner = err instanceof Error ? err.message : String(err);
|
|
177
|
+
// eslint-disable-next-line no-console
|
|
178
|
+
console.error(`[avbridge] ${message}:`, err);
|
|
179
|
+
return new Error(`${message}: ${inner || "(no message — see browser console)"}`);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
interface LoadedVariant {
|
|
183
|
+
LibAV: (opts?: Record<string, unknown>) => Promise<Record<string, unknown>>;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** Loose structural type — the AVI probe and the fallback decoder add fields. */
|
|
187
|
+
export type LibavInstance = Record<string, unknown> & {
|
|
188
|
+
mkreadaheadfile(name: string, blob: Blob): Promise<void>;
|
|
189
|
+
unlinkreadaheadfile(name: string): Promise<void>;
|
|
190
|
+
};
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { MediaContext, AudioCodec, VideoCodec } from "../../types.js";
|
|
2
|
+
import type { LibavVariant } from "./libav-loader.js";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Decide which libav.js variant to load for a given media context.
|
|
6
|
+
*
|
|
7
|
+
* - **webcodecs** (~5 MB, npm) — modern formats only, designed for the
|
|
8
|
+
* WebCodecs bridge. Used when the codec is browser-supported and we just
|
|
9
|
+
* need libav.js for demuxing or as a parser source.
|
|
10
|
+
*
|
|
11
|
+
* - **avbridge** (custom build, vendor/libav/) — has the AVI/ASF/FLV demuxers
|
|
12
|
+
* and the legacy decoders (WMV3, MPEG-4 Part 2, VC-1, MS-MPEG4 v1/2/3,
|
|
13
|
+
* AC-3, WMA*). Required for any of those formats; the npm variants ship
|
|
14
|
+
* none of them.
|
|
15
|
+
*
|
|
16
|
+
* Rule: pick "avbridge" if either the container or any codec is one only the
|
|
17
|
+
* custom build can handle. Otherwise pick "webcodecs".
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
const LEGACY_CONTAINERS = new Set(["avi", "asf", "flv"]);
|
|
21
|
+
|
|
22
|
+
const LEGACY_VIDEO_CODECS = new Set<VideoCodec>([
|
|
23
|
+
"wmv3",
|
|
24
|
+
"vc1",
|
|
25
|
+
"mpeg4", // MPEG-4 Part 2 / DivX / Xvid
|
|
26
|
+
"rv40",
|
|
27
|
+
"mpeg2",
|
|
28
|
+
"mpeg1",
|
|
29
|
+
"theora",
|
|
30
|
+
]);
|
|
31
|
+
|
|
32
|
+
const LEGACY_AUDIO_CODECS = new Set<AudioCodec>(["wmav2", "wmapro", "ac3", "eac3"]);
|
|
33
|
+
|
|
34
|
+
export function pickLibavVariant(ctx: MediaContext): LibavVariant {
|
|
35
|
+
if (LEGACY_CONTAINERS.has(ctx.container)) return "avbridge";
|
|
36
|
+
for (const v of ctx.videoTracks) {
|
|
37
|
+
if (LEGACY_VIDEO_CODECS.has(v.codec)) return "avbridge";
|
|
38
|
+
}
|
|
39
|
+
for (const a of ctx.audioTracks) {
|
|
40
|
+
if (LEGACY_AUDIO_CODECS.has(a.codec)) return "avbridge";
|
|
41
|
+
}
|
|
42
|
+
return "webcodecs";
|
|
43
|
+
}
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import type { ClockSource } from "./audio-output.js";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Renders decoded `VideoFrame`s into a 2D canvas overlaid on the user's
|
|
5
|
+
* `<video>` element. The fallback strategy never assigns a src to the video,
|
|
6
|
+
* so we hide it and put the canvas in its place visually.
|
|
7
|
+
*
|
|
8
|
+
* The renderer has two modes:
|
|
9
|
+
*
|
|
10
|
+
* 1. **Pre-roll** — `clock.isPlaying()` is false. The very first decoded
|
|
11
|
+
* frame is painted as a "poster" so the user sees something while audio
|
|
12
|
+
* buffers; subsequent frames stay queued without being dropped.
|
|
13
|
+
*
|
|
14
|
+
* 2. **Synced** — `clock.isPlaying()` is true. On each rAF tick, find the
|
|
15
|
+
* latest frame whose timestamp ≤ `clock.now() + lookahead` and paint it.
|
|
16
|
+
* Drop any older frames as "late."
|
|
17
|
+
*
|
|
18
|
+
* The pre-roll behavior is what fixes the cold-start "first minute is all
|
|
19
|
+
* dropped" problem: without it, the wall clock raced ahead while the
|
|
20
|
+
* decoder was still warming up, and every frame was already in the past by
|
|
21
|
+
* the time it landed in the queue.
|
|
22
|
+
*/
|
|
23
|
+
export class VideoRenderer {
|
|
24
|
+
private canvas: HTMLCanvasElement;
|
|
25
|
+
private ctx: CanvasRenderingContext2D;
|
|
26
|
+
private queue: VideoFrame[] = [];
|
|
27
|
+
private rafHandle: number | null = null;
|
|
28
|
+
private destroyed = false;
|
|
29
|
+
|
|
30
|
+
private framesPainted = 0;
|
|
31
|
+
private framesDroppedLate = 0;
|
|
32
|
+
private framesDroppedOverflow = 0;
|
|
33
|
+
private prerolled = false;
|
|
34
|
+
/** Wall-clock time of the last paint, in ms (performance.now()). */
|
|
35
|
+
private lastPaintWall = 0;
|
|
36
|
+
/** Minimum ms between paints — paces video at roughly source fps. */
|
|
37
|
+
private paintIntervalMs: number;
|
|
38
|
+
|
|
39
|
+
/** Resolves once the first decoded frame has been enqueued. */
|
|
40
|
+
readonly firstFrameReady: Promise<void>;
|
|
41
|
+
private resolveFirstFrame!: () => void;
|
|
42
|
+
|
|
43
|
+
constructor(
|
|
44
|
+
private readonly target: HTMLVideoElement,
|
|
45
|
+
private readonly clock: ClockSource,
|
|
46
|
+
fps = 30,
|
|
47
|
+
) {
|
|
48
|
+
this.paintIntervalMs = Math.max(1, 1000 / fps);
|
|
49
|
+
this.firstFrameReady = new Promise<void>((resolve) => {
|
|
50
|
+
this.resolveFirstFrame = resolve;
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
this.canvas = document.createElement("canvas");
|
|
54
|
+
this.canvas.style.cssText =
|
|
55
|
+
"position:absolute;left:0;top:0;width:100%;height:100%;background:black;";
|
|
56
|
+
const parent = target.parentElement;
|
|
57
|
+
if (parent && getComputedStyle(parent).position === "static") {
|
|
58
|
+
parent.style.position = "relative";
|
|
59
|
+
}
|
|
60
|
+
parent?.insertBefore(this.canvas, target);
|
|
61
|
+
target.style.visibility = "hidden";
|
|
62
|
+
|
|
63
|
+
const ctx = this.canvas.getContext("2d");
|
|
64
|
+
if (!ctx) throw new Error("video renderer: failed to acquire 2D context");
|
|
65
|
+
this.ctx = ctx;
|
|
66
|
+
|
|
67
|
+
this.tick = this.tick.bind(this);
|
|
68
|
+
this.rafHandle = requestAnimationFrame(this.tick);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** True once at least one frame has been enqueued. */
|
|
72
|
+
hasFrames(): boolean {
|
|
73
|
+
return this.queue.length > 0 || this.framesPainted > 0;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Current depth of the frame queue. Used by the decoder for backpressure. */
|
|
77
|
+
queueDepth(): number {
|
|
78
|
+
return this.queue.length;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Soft cap for decoder backpressure. The decoder pump throttles when
|
|
83
|
+
* `queueDepth() >= queueHighWater`. Set high enough that normal decode
|
|
84
|
+
* bursts don't trigger the renderer's overflow-drop loop (which runs at
|
|
85
|
+
* every paint), but low enough that the decoder doesn't run unboundedly
|
|
86
|
+
* ahead. The hard cap in `enqueue()` is 64.
|
|
87
|
+
*/
|
|
88
|
+
readonly queueHighWater = 30;
|
|
89
|
+
|
|
90
|
+
enqueue(frame: VideoFrame): void {
|
|
91
|
+
if (this.destroyed) {
|
|
92
|
+
frame.close();
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
this.queue.push(frame);
|
|
96
|
+
if (this.queue.length === 1 && this.framesPainted === 0) {
|
|
97
|
+
this.resolveFirstFrame();
|
|
98
|
+
}
|
|
99
|
+
// Hard cap. Should rarely trigger because the decoder backs off at
|
|
100
|
+
// queueHighWater (30) and the drift correction trims gently. This is
|
|
101
|
+
// the last-resort defense against runaway producers.
|
|
102
|
+
while (this.queue.length > 60) {
|
|
103
|
+
this.queue.shift()?.close();
|
|
104
|
+
this.framesDroppedOverflow++;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
private tick(): void {
|
|
109
|
+
if (this.destroyed) return;
|
|
110
|
+
this.rafHandle = requestAnimationFrame(this.tick);
|
|
111
|
+
|
|
112
|
+
if (this.queue.length === 0) return;
|
|
113
|
+
|
|
114
|
+
const playing = this.clock.isPlaying();
|
|
115
|
+
|
|
116
|
+
// Pre-roll: paint the very first frame as a poster while audio buffers.
|
|
117
|
+
if (!playing) {
|
|
118
|
+
if (!this.prerolled) {
|
|
119
|
+
const head = this.queue.shift()!;
|
|
120
|
+
this.paint(head);
|
|
121
|
+
head.close();
|
|
122
|
+
this.prerolled = true;
|
|
123
|
+
this.lastPaintWall = performance.now();
|
|
124
|
+
}
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// Wall-clock-paced painting with coarse A/V drift correction.
|
|
129
|
+
//
|
|
130
|
+
// Base policy: paint one frame every `paintIntervalMs` of wall time,
|
|
131
|
+
// regardless of the frame's synthetic timestamp. This avoids the old
|
|
132
|
+
// per-frame audio-gate that caused massive overflow during decode bursts.
|
|
133
|
+
//
|
|
134
|
+
// Drift correction (runs every ~1 sec):
|
|
135
|
+
// - Video > 150 ms behind audio → drop one frame (catch up)
|
|
136
|
+
// - Video > 150 ms ahead of audio → skip one paint (let audio catch up)
|
|
137
|
+
//
|
|
138
|
+
// This keeps long-run sync robust even for legacy AVI/DivX with messy
|
|
139
|
+
// timestamps, packed B-frames, and odd frame durations. The correction
|
|
140
|
+
// is deliberately gentle (one frame at a time) so it doesn't cause
|
|
141
|
+
// visible stuttering.
|
|
142
|
+
const wallNow = performance.now();
|
|
143
|
+
if (wallNow - this.lastPaintWall < this.paintIntervalMs - 2) return;
|
|
144
|
+
|
|
145
|
+
if (this.queue.length === 0) return;
|
|
146
|
+
|
|
147
|
+
// Coarse drift correction: compare the head frame's timestamp to
|
|
148
|
+
// audio.now() every ~1 sec (every 30 frames at 30fps). The frame ts
|
|
149
|
+
// and audio.now() are both in seconds of media time. Drift beyond
|
|
150
|
+
// 150ms triggers gentle correction — one frame per check, not a burst.
|
|
151
|
+
if (this.framesPainted > 0 && this.framesPainted % 30 === 0) {
|
|
152
|
+
const audioNowUs = this.clock.now() * 1_000_000;
|
|
153
|
+
const headTs = this.queue[0].timestamp ?? 0;
|
|
154
|
+
const driftUs = headTs - audioNowUs;
|
|
155
|
+
|
|
156
|
+
if (driftUs < -150_000) {
|
|
157
|
+
// Video behind audio by > 150ms — drop one frame to catch up.
|
|
158
|
+
this.queue.shift()?.close();
|
|
159
|
+
this.framesDroppedLate++;
|
|
160
|
+
if (this.queue.length === 0) return;
|
|
161
|
+
} else if (driftUs > 150_000) {
|
|
162
|
+
// Video ahead of audio by > 150ms — skip this paint cycle.
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
const frame = this.queue.shift()!;
|
|
168
|
+
this.paint(frame);
|
|
169
|
+
frame.close();
|
|
170
|
+
this.lastPaintWall = wallNow;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
private paint(frame: VideoFrame): void {
|
|
174
|
+
if (
|
|
175
|
+
this.canvas.width !== frame.displayWidth ||
|
|
176
|
+
this.canvas.height !== frame.displayHeight
|
|
177
|
+
) {
|
|
178
|
+
this.canvas.width = frame.displayWidth;
|
|
179
|
+
this.canvas.height = frame.displayHeight;
|
|
180
|
+
}
|
|
181
|
+
try {
|
|
182
|
+
this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
|
|
183
|
+
this.framesPainted++;
|
|
184
|
+
} catch (err) {
|
|
185
|
+
// Log only once so a structurally broken frame format doesn't spam
|
|
186
|
+
// the console at 60 Hz, but we still find out about it.
|
|
187
|
+
if (this.framesPainted === 0 && this.framesDroppedLate === 0) {
|
|
188
|
+
// eslint-disable-next-line no-console
|
|
189
|
+
console.warn("[avbridge] canvas drawImage failed:", err);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** Discard all queued frames. Used by seek to drop stale buffers. */
|
|
195
|
+
flush(): void {
|
|
196
|
+
while (this.queue.length > 0) this.queue.shift()?.close();
|
|
197
|
+
this.prerolled = false;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
stats(): Record<string, unknown> {
|
|
201
|
+
return {
|
|
202
|
+
framesPainted: this.framesPainted,
|
|
203
|
+
framesDroppedLate: this.framesDroppedLate,
|
|
204
|
+
framesDroppedOverflow: this.framesDroppedOverflow,
|
|
205
|
+
queueDepth: this.queue.length,
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
destroy(): void {
|
|
210
|
+
this.destroyed = true;
|
|
211
|
+
if (this.rafHandle != null) cancelAnimationFrame(this.rafHandle);
|
|
212
|
+
this.flush();
|
|
213
|
+
this.canvas.remove();
|
|
214
|
+
this.target.style.visibility = "";
|
|
215
|
+
}
|
|
216
|
+
}
|