@kenzuya/mediabunny 1.26.0 → 1.28.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bundles/{mediabunny.mjs → mediabunny.js} +21963 -21390
- package/dist/bundles/mediabunny.min.js +490 -0
- package/dist/modules/shared/mp3-misc.d.ts.map +1 -1
- package/dist/modules/src/adts/adts-demuxer.d.ts +6 -6
- package/dist/modules/src/adts/adts-demuxer.d.ts.map +1 -1
- package/dist/modules/src/adts/adts-muxer.d.ts +4 -4
- package/dist/modules/src/adts/adts-muxer.d.ts.map +1 -1
- package/dist/modules/src/adts/adts-reader.d.ts +1 -1
- package/dist/modules/src/adts/adts-reader.d.ts.map +1 -1
- package/dist/modules/src/avi/avi-demuxer.d.ts +44 -0
- package/dist/modules/src/avi/avi-demuxer.d.ts.map +1 -0
- package/dist/modules/src/avi/avi-misc.d.ts +88 -0
- package/dist/modules/src/avi/avi-misc.d.ts.map +1 -0
- package/dist/modules/src/avi/avi-muxer.d.ts +45 -0
- package/dist/modules/src/avi/avi-muxer.d.ts.map +1 -0
- package/dist/modules/src/avi/riff-writer.d.ts +26 -0
- package/dist/modules/src/avi/riff-writer.d.ts.map +1 -0
- package/dist/modules/src/codec-data.d.ts +8 -3
- package/dist/modules/src/codec-data.d.ts.map +1 -1
- package/dist/modules/src/codec.d.ts +10 -10
- package/dist/modules/src/codec.d.ts.map +1 -1
- package/dist/modules/src/conversion.d.ts +33 -16
- package/dist/modules/src/conversion.d.ts.map +1 -1
- package/dist/modules/src/custom-coder.d.ts +8 -8
- package/dist/modules/src/custom-coder.d.ts.map +1 -1
- package/dist/modules/src/demuxer.d.ts +3 -3
- package/dist/modules/src/demuxer.d.ts.map +1 -1
- package/dist/modules/src/encode.d.ts +8 -8
- package/dist/modules/src/encode.d.ts.map +1 -1
- package/dist/modules/src/flac/flac-demuxer.d.ts +7 -7
- package/dist/modules/src/flac/flac-demuxer.d.ts.map +1 -1
- package/dist/modules/src/flac/flac-misc.d.ts +3 -3
- package/dist/modules/src/flac/flac-misc.d.ts.map +1 -1
- package/dist/modules/src/flac/flac-muxer.d.ts +5 -5
- package/dist/modules/src/flac/flac-muxer.d.ts.map +1 -1
- package/dist/modules/src/id3.d.ts +3 -3
- package/dist/modules/src/id3.d.ts.map +1 -1
- package/dist/modules/src/index.d.ts +20 -20
- package/dist/modules/src/index.d.ts.map +1 -1
- package/dist/modules/src/input-format.d.ts +22 -0
- package/dist/modules/src/input-format.d.ts.map +1 -1
- package/dist/modules/src/input-track.d.ts +8 -8
- package/dist/modules/src/input-track.d.ts.map +1 -1
- package/dist/modules/src/input.d.ts +12 -12
- package/dist/modules/src/isobmff/isobmff-boxes.d.ts +2 -2
- package/dist/modules/src/isobmff/isobmff-boxes.d.ts.map +1 -1
- package/dist/modules/src/isobmff/isobmff-demuxer.d.ts +12 -12
- package/dist/modules/src/isobmff/isobmff-demuxer.d.ts.map +1 -1
- package/dist/modules/src/isobmff/isobmff-misc.d.ts.map +1 -1
- package/dist/modules/src/isobmff/isobmff-muxer.d.ts +11 -11
- package/dist/modules/src/isobmff/isobmff-muxer.d.ts.map +1 -1
- package/dist/modules/src/isobmff/isobmff-reader.d.ts +2 -2
- package/dist/modules/src/isobmff/isobmff-reader.d.ts.map +1 -1
- package/dist/modules/src/matroska/ebml.d.ts +3 -3
- package/dist/modules/src/matroska/ebml.d.ts.map +1 -1
- package/dist/modules/src/matroska/matroska-demuxer.d.ts +13 -13
- package/dist/modules/src/matroska/matroska-demuxer.d.ts.map +1 -1
- package/dist/modules/src/matroska/matroska-input.d.ts +33 -0
- package/dist/modules/src/matroska/matroska-input.d.ts.map +1 -0
- package/dist/modules/src/matroska/matroska-misc.d.ts.map +1 -1
- package/dist/modules/src/matroska/matroska-muxer.d.ts +5 -5
- package/dist/modules/src/matroska/matroska-muxer.d.ts.map +1 -1
- package/dist/modules/src/media-sink.d.ts +5 -5
- package/dist/modules/src/media-sink.d.ts.map +1 -1
- package/dist/modules/src/media-source.d.ts +22 -4
- package/dist/modules/src/media-source.d.ts.map +1 -1
- package/dist/modules/src/metadata.d.ts +2 -2
- package/dist/modules/src/metadata.d.ts.map +1 -1
- package/dist/modules/src/misc.d.ts +5 -4
- package/dist/modules/src/misc.d.ts.map +1 -1
- package/dist/modules/src/mp3/mp3-demuxer.d.ts +7 -7
- package/dist/modules/src/mp3/mp3-demuxer.d.ts.map +1 -1
- package/dist/modules/src/mp3/mp3-muxer.d.ts +4 -4
- package/dist/modules/src/mp3/mp3-muxer.d.ts.map +1 -1
- package/dist/modules/src/mp3/mp3-reader.d.ts +2 -2
- package/dist/modules/src/mp3/mp3-reader.d.ts.map +1 -1
- package/dist/modules/src/mp3/mp3-writer.d.ts +1 -1
- package/dist/modules/src/mp3/mp3-writer.d.ts.map +1 -1
- package/dist/modules/src/muxer.d.ts +4 -4
- package/dist/modules/src/muxer.d.ts.map +1 -1
- package/dist/modules/src/ogg/ogg-demuxer.d.ts +7 -7
- package/dist/modules/src/ogg/ogg-demuxer.d.ts.map +1 -1
- package/dist/modules/src/ogg/ogg-misc.d.ts +1 -1
- package/dist/modules/src/ogg/ogg-misc.d.ts.map +1 -1
- package/dist/modules/src/ogg/ogg-muxer.d.ts +5 -5
- package/dist/modules/src/ogg/ogg-muxer.d.ts.map +1 -1
- package/dist/modules/src/ogg/ogg-reader.d.ts +1 -1
- package/dist/modules/src/ogg/ogg-reader.d.ts.map +1 -1
- package/dist/modules/src/output-format.d.ts +51 -6
- package/dist/modules/src/output-format.d.ts.map +1 -1
- package/dist/modules/src/output.d.ts +13 -13
- package/dist/modules/src/output.d.ts.map +1 -1
- package/dist/modules/src/packet.d.ts +1 -1
- package/dist/modules/src/packet.d.ts.map +1 -1
- package/dist/modules/src/pcm.d.ts.map +1 -1
- package/dist/modules/src/reader.d.ts +2 -2
- package/dist/modules/src/reader.d.ts.map +1 -1
- package/dist/modules/src/sample.d.ts +57 -15
- package/dist/modules/src/sample.d.ts.map +1 -1
- package/dist/modules/src/source.d.ts +3 -3
- package/dist/modules/src/source.d.ts.map +1 -1
- package/dist/modules/src/subtitles.d.ts +1 -1
- package/dist/modules/src/subtitles.d.ts.map +1 -1
- package/dist/modules/src/target.d.ts +2 -2
- package/dist/modules/src/target.d.ts.map +1 -1
- package/dist/modules/src/tsconfig.tsbuildinfo +1 -1
- package/dist/modules/src/wave/riff-writer.d.ts +1 -1
- package/dist/modules/src/wave/riff-writer.d.ts.map +1 -1
- package/dist/modules/src/wave/wave-demuxer.d.ts +6 -6
- package/dist/modules/src/wave/wave-demuxer.d.ts.map +1 -1
- package/dist/modules/src/wave/wave-muxer.d.ts +4 -4
- package/dist/modules/src/wave/wave-muxer.d.ts.map +1 -1
- package/dist/modules/src/writer.d.ts +1 -1
- package/dist/modules/src/writer.d.ts.map +1 -1
- package/dist/packages/eac3/eac3.wasm +0 -0
- package/dist/packages/eac3/mediabunny-eac3.js +1058 -0
- package/dist/packages/eac3/mediabunny-eac3.min.js +44 -0
- package/dist/packages/mp3-encoder/mediabunny-mp3-encoder.js +694 -0
- package/dist/packages/mp3-encoder/mediabunny-mp3-encoder.min.js +58 -0
- package/dist/packages/mpeg4/mediabunny-mpeg4.js +1198 -0
- package/dist/packages/mpeg4/mediabunny-mpeg4.min.js +44 -0
- package/dist/packages/mpeg4/xvid.wasm +0 -0
- package/package.json +18 -57
- package/dist/bundles/mediabunny.cjs +0 -26140
- package/dist/bundles/mediabunny.min.cjs +0 -147
- package/dist/bundles/mediabunny.min.mjs +0 -146
- package/dist/mediabunny.d.ts +0 -3319
- package/dist/modules/shared/mp3-misc.js +0 -147
- package/dist/modules/src/adts/adts-demuxer.js +0 -239
- package/dist/modules/src/adts/adts-muxer.js +0 -80
- package/dist/modules/src/adts/adts-reader.js +0 -63
- package/dist/modules/src/codec-data.js +0 -1730
- package/dist/modules/src/codec.js +0 -869
- package/dist/modules/src/conversion.js +0 -1459
- package/dist/modules/src/custom-coder.js +0 -117
- package/dist/modules/src/demuxer.js +0 -12
- package/dist/modules/src/encode.js +0 -442
- package/dist/modules/src/flac/flac-demuxer.js +0 -504
- package/dist/modules/src/flac/flac-misc.js +0 -135
- package/dist/modules/src/flac/flac-muxer.js +0 -222
- package/dist/modules/src/id3.js +0 -848
- package/dist/modules/src/index.js +0 -28
- package/dist/modules/src/input-format.js +0 -480
- package/dist/modules/src/input-track.js +0 -372
- package/dist/modules/src/input.js +0 -188
- package/dist/modules/src/isobmff/isobmff-boxes.js +0 -1480
- package/dist/modules/src/isobmff/isobmff-demuxer.js +0 -2618
- package/dist/modules/src/isobmff/isobmff-misc.js +0 -20
- package/dist/modules/src/isobmff/isobmff-muxer.js +0 -966
- package/dist/modules/src/isobmff/isobmff-reader.js +0 -72
- package/dist/modules/src/matroska/ebml.js +0 -653
- package/dist/modules/src/matroska/matroska-demuxer.js +0 -2133
- package/dist/modules/src/matroska/matroska-misc.js +0 -20
- package/dist/modules/src/matroska/matroska-muxer.js +0 -1017
- package/dist/modules/src/media-sink.js +0 -1736
- package/dist/modules/src/media-source.js +0 -1825
- package/dist/modules/src/metadata.js +0 -193
- package/dist/modules/src/misc.js +0 -623
- package/dist/modules/src/mp3/mp3-demuxer.js +0 -285
- package/dist/modules/src/mp3/mp3-muxer.js +0 -123
- package/dist/modules/src/mp3/mp3-reader.js +0 -26
- package/dist/modules/src/mp3/mp3-writer.js +0 -78
- package/dist/modules/src/muxer.js +0 -50
- package/dist/modules/src/node.d.ts +0 -9
- package/dist/modules/src/node.d.ts.map +0 -1
- package/dist/modules/src/node.js +0 -9
- package/dist/modules/src/ogg/ogg-demuxer.js +0 -763
- package/dist/modules/src/ogg/ogg-misc.js +0 -78
- package/dist/modules/src/ogg/ogg-muxer.js +0 -353
- package/dist/modules/src/ogg/ogg-reader.js +0 -65
- package/dist/modules/src/output-format.js +0 -527
- package/dist/modules/src/output.js +0 -300
- package/dist/modules/src/packet.js +0 -182
- package/dist/modules/src/pcm.js +0 -85
- package/dist/modules/src/reader.js +0 -236
- package/dist/modules/src/sample.js +0 -1056
- package/dist/modules/src/source.js +0 -1182
- package/dist/modules/src/subtitles.js +0 -575
- package/dist/modules/src/target.js +0 -140
- package/dist/modules/src/wave/riff-writer.js +0 -30
- package/dist/modules/src/wave/wave-demuxer.js +0 -447
- package/dist/modules/src/wave/wave-muxer.js +0 -318
- package/dist/modules/src/writer.js +0 -370
- package/src/adts/adts-demuxer.ts +0 -331
- package/src/adts/adts-muxer.ts +0 -111
- package/src/adts/adts-reader.ts +0 -85
- package/src/codec-data.ts +0 -2078
- package/src/codec.ts +0 -1092
- package/src/conversion.ts +0 -2112
- package/src/custom-coder.ts +0 -197
- package/src/demuxer.ts +0 -24
- package/src/encode.ts +0 -739
- package/src/flac/flac-demuxer.ts +0 -730
- package/src/flac/flac-misc.ts +0 -164
- package/src/flac/flac-muxer.ts +0 -320
- package/src/id3.ts +0 -925
- package/src/index.ts +0 -221
- package/src/input-format.ts +0 -541
- package/src/input-track.ts +0 -529
- package/src/input.ts +0 -235
- package/src/isobmff/isobmff-boxes.ts +0 -1719
- package/src/isobmff/isobmff-demuxer.ts +0 -3190
- package/src/isobmff/isobmff-misc.ts +0 -29
- package/src/isobmff/isobmff-muxer.ts +0 -1348
- package/src/isobmff/isobmff-reader.ts +0 -91
- package/src/matroska/ebml.ts +0 -730
- package/src/matroska/matroska-demuxer.ts +0 -2481
- package/src/matroska/matroska-misc.ts +0 -29
- package/src/matroska/matroska-muxer.ts +0 -1276
- package/src/media-sink.ts +0 -2179
- package/src/media-source.ts +0 -2243
- package/src/metadata.ts +0 -320
- package/src/misc.ts +0 -798
- package/src/mp3/mp3-demuxer.ts +0 -383
- package/src/mp3/mp3-muxer.ts +0 -166
- package/src/mp3/mp3-reader.ts +0 -34
- package/src/mp3/mp3-writer.ts +0 -120
- package/src/muxer.ts +0 -88
- package/src/node.ts +0 -11
- package/src/ogg/ogg-demuxer.ts +0 -1053
- package/src/ogg/ogg-misc.ts +0 -116
- package/src/ogg/ogg-muxer.ts +0 -497
- package/src/ogg/ogg-reader.ts +0 -93
- package/src/output-format.ts +0 -945
- package/src/output.ts +0 -488
- package/src/packet.ts +0 -263
- package/src/pcm.ts +0 -112
- package/src/reader.ts +0 -323
- package/src/sample.ts +0 -1461
- package/src/source.ts +0 -1688
- package/src/subtitles.ts +0 -711
- package/src/target.ts +0 -204
- package/src/tsconfig.json +0 -16
- package/src/wave/riff-writer.ts +0 -36
- package/src/wave/wave-demuxer.ts +0 -529
- package/src/wave/wave-muxer.ts +0 -371
- package/src/writer.ts +0 -490
package/src/media-sink.ts
DELETED
|
@@ -1,2179 +0,0 @@
|
|
|
1
|
-
/*!
|
|
2
|
-
* Copyright (c) 2025-present, Vanilagy and contributors
|
|
3
|
-
*
|
|
4
|
-
* This Source Code Form is subject to the terms of the Mozilla Public
|
|
5
|
-
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
6
|
-
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
7
|
-
*/
|
|
8
|
-
|
|
9
|
-
import { parsePcmCodec, PCM_AUDIO_CODECS, PcmAudioCodec, VideoCodec, AudioCodec } from './codec';
|
|
10
|
-
import {
|
|
11
|
-
deserializeAvcDecoderConfigurationRecord,
|
|
12
|
-
determineVideoPacketType,
|
|
13
|
-
extractHevcNalUnits,
|
|
14
|
-
extractNalUnitTypeForHevc,
|
|
15
|
-
HevcNalUnitType,
|
|
16
|
-
parseAvcSps,
|
|
17
|
-
} from './codec-data';
|
|
18
|
-
import { CustomVideoDecoder, customVideoDecoders, CustomAudioDecoder, customAudioDecoders } from './custom-coder';
|
|
19
|
-
import { InputDisposedError } from './input';
|
|
20
|
-
import { InputAudioTrack, InputTrack, InputVideoTrack } from './input-track';
|
|
21
|
-
import {
|
|
22
|
-
AnyIterable,
|
|
23
|
-
assert,
|
|
24
|
-
assertNever,
|
|
25
|
-
CallSerializer,
|
|
26
|
-
getInt24,
|
|
27
|
-
getUint24,
|
|
28
|
-
insertSorted,
|
|
29
|
-
isChromium,
|
|
30
|
-
isFirefox,
|
|
31
|
-
isNumber,
|
|
32
|
-
isWebKit,
|
|
33
|
-
last,
|
|
34
|
-
mapAsyncGenerator,
|
|
35
|
-
promiseWithResolvers,
|
|
36
|
-
Rotation,
|
|
37
|
-
toAsyncIterator,
|
|
38
|
-
toDataView,
|
|
39
|
-
toUint8Array,
|
|
40
|
-
validateAnyIterable,
|
|
41
|
-
} from './misc';
|
|
42
|
-
import { EncodedPacket } from './packet';
|
|
43
|
-
import { fromAlaw, fromUlaw } from './pcm';
|
|
44
|
-
import { AudioSample, clampCropRectangle, CropRectangle, validateCropRectangle, VideoSample } from './sample';
|
|
45
|
-
|
|
46
|
-
/**
|
|
47
|
-
* Additional options for controlling packet retrieval.
|
|
48
|
-
* @group Media sinks
|
|
49
|
-
* @public
|
|
50
|
-
*/
|
|
51
|
-
export type PacketRetrievalOptions = {
|
|
52
|
-
/**
|
|
53
|
-
* When set to `true`, only packet metadata (like timestamp) will be retrieved - the actual packet data will not
|
|
54
|
-
* be loaded.
|
|
55
|
-
*/
|
|
56
|
-
metadataOnly?: boolean;
|
|
57
|
-
|
|
58
|
-
/**
|
|
59
|
-
* When set to true, key packets will be verified upon retrieval by looking into the packet's bitstream.
|
|
60
|
-
* If not enabled, the packet types will be determined solely by what's stored in the containing file and may be
|
|
61
|
-
* incorrect, potentially leading to decoder errors. Since determining a packet's actual type requires looking into
|
|
62
|
-
* its data, this option cannot be enabled together with `metadataOnly`.
|
|
63
|
-
*/
|
|
64
|
-
verifyKeyPackets?: boolean;
|
|
65
|
-
};
|
|
66
|
-
|
|
67
|
-
const validatePacketRetrievalOptions = (options: PacketRetrievalOptions) => {
|
|
68
|
-
if (!options || typeof options !== 'object') {
|
|
69
|
-
throw new TypeError('options must be an object.');
|
|
70
|
-
}
|
|
71
|
-
if (options.metadataOnly !== undefined && typeof options.metadataOnly !== 'boolean') {
|
|
72
|
-
throw new TypeError('options.metadataOnly, when defined, must be a boolean.');
|
|
73
|
-
}
|
|
74
|
-
if (options.verifyKeyPackets !== undefined && typeof options.verifyKeyPackets !== 'boolean') {
|
|
75
|
-
throw new TypeError('options.verifyKeyPackets, when defined, must be a boolean.');
|
|
76
|
-
}
|
|
77
|
-
if (options.verifyKeyPackets && options.metadataOnly) {
|
|
78
|
-
throw new TypeError('options.verifyKeyPackets and options.metadataOnly cannot be enabled together.');
|
|
79
|
-
}
|
|
80
|
-
};
|
|
81
|
-
|
|
82
|
-
const validateTimestamp = (timestamp: number) => {
|
|
83
|
-
if (!isNumber(timestamp)) {
|
|
84
|
-
throw new TypeError('timestamp must be a number.'); // It can be non-finite, that's fine
|
|
85
|
-
}
|
|
86
|
-
};
|
|
87
|
-
|
|
88
|
-
const maybeFixPacketType = (
|
|
89
|
-
track: InputTrack,
|
|
90
|
-
promise: Promise<EncodedPacket | null>,
|
|
91
|
-
options: PacketRetrievalOptions,
|
|
92
|
-
) => {
|
|
93
|
-
if (options.verifyKeyPackets) {
|
|
94
|
-
return promise.then(async (packet) => {
|
|
95
|
-
if (!packet || packet.type === 'delta') {
|
|
96
|
-
return packet;
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
const determinedType = await track.determinePacketType(packet);
|
|
100
|
-
if (determinedType) {
|
|
101
|
-
// @ts-expect-error Technically readonly
|
|
102
|
-
packet.type = determinedType;
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
return packet;
|
|
106
|
-
});
|
|
107
|
-
} else {
|
|
108
|
-
return promise;
|
|
109
|
-
}
|
|
110
|
-
};
|
|
111
|
-
|
|
112
|
-
/**
|
|
113
|
-
* Sink for retrieving encoded packets from an input track.
|
|
114
|
-
* @group Media sinks
|
|
115
|
-
* @public
|
|
116
|
-
*/
|
|
117
|
-
export class EncodedPacketSink {
|
|
118
|
-
/** @internal */
|
|
119
|
-
_track: InputTrack;
|
|
120
|
-
|
|
121
|
-
/** Creates a new {@link EncodedPacketSink} for the given {@link InputTrack}. */
|
|
122
|
-
constructor(track: InputTrack) {
|
|
123
|
-
if (!(track instanceof InputTrack)) {
|
|
124
|
-
throw new TypeError('track must be an InputTrack.');
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
this._track = track;
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
/**
|
|
131
|
-
* Retrieves the track's first packet (in decode order), or null if it has no packets. The first packet is very
|
|
132
|
-
* likely to be a key packet.
|
|
133
|
-
*/
|
|
134
|
-
getFirstPacket(options: PacketRetrievalOptions = {}) {
|
|
135
|
-
validatePacketRetrievalOptions(options);
|
|
136
|
-
|
|
137
|
-
if (this._track.input._disposed) {
|
|
138
|
-
throw new InputDisposedError();
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
return maybeFixPacketType(this._track, this._track._backing.getFirstPacket(options), options);
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
/**
|
|
145
|
-
* Retrieves the packet corresponding to the given timestamp, in seconds. More specifically, returns the last packet
|
|
146
|
-
* (in presentation order) with a start timestamp less than or equal to the given timestamp. This method can be
|
|
147
|
-
* used to retrieve a track's last packet using `getPacket(Infinity)`. The method returns null if the timestamp
|
|
148
|
-
* is before the first packet in the track.
|
|
149
|
-
*
|
|
150
|
-
* @param timestamp - The timestamp used for retrieval, in seconds.
|
|
151
|
-
*/
|
|
152
|
-
getPacket(timestamp: number, options: PacketRetrievalOptions = {}) {
|
|
153
|
-
validateTimestamp(timestamp);
|
|
154
|
-
validatePacketRetrievalOptions(options);
|
|
155
|
-
|
|
156
|
-
if (this._track.input._disposed) {
|
|
157
|
-
throw new InputDisposedError();
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
return maybeFixPacketType(this._track, this._track._backing.getPacket(timestamp, options), options);
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
/**
|
|
164
|
-
* Retrieves the packet following the given packet (in decode order), or null if the given packet is the
|
|
165
|
-
* last packet.
|
|
166
|
-
*/
|
|
167
|
-
getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions = {}) {
|
|
168
|
-
if (!(packet instanceof EncodedPacket)) {
|
|
169
|
-
throw new TypeError('packet must be an EncodedPacket.');
|
|
170
|
-
}
|
|
171
|
-
validatePacketRetrievalOptions(options);
|
|
172
|
-
|
|
173
|
-
if (this._track.input._disposed) {
|
|
174
|
-
throw new InputDisposedError();
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
return maybeFixPacketType(this._track, this._track._backing.getNextPacket(packet, options), options);
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
/**
|
|
181
|
-
* Retrieves the key packet corresponding to the given timestamp, in seconds. More specifically, returns the last
|
|
182
|
-
* key packet (in presentation order) with a start timestamp less than or equal to the given timestamp. A key packet
|
|
183
|
-
* is a packet that doesn't require previous packets to be decoded. This method can be used to retrieve a track's
|
|
184
|
-
* last key packet using `getKeyPacket(Infinity)`. The method returns null if the timestamp is before the first
|
|
185
|
-
* key packet in the track.
|
|
186
|
-
*
|
|
187
|
-
* To ensure that the returned packet is guaranteed to be a real key frame, enable `options.verifyKeyPackets`.
|
|
188
|
-
*
|
|
189
|
-
* @param timestamp - The timestamp used for retrieval, in seconds.
|
|
190
|
-
*/
|
|
191
|
-
async getKeyPacket(timestamp: number, options: PacketRetrievalOptions = {}): Promise<EncodedPacket | null> {
|
|
192
|
-
validateTimestamp(timestamp);
|
|
193
|
-
validatePacketRetrievalOptions(options);
|
|
194
|
-
|
|
195
|
-
if (this._track.input._disposed) {
|
|
196
|
-
throw new InputDisposedError();
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
if (!options.verifyKeyPackets) {
|
|
200
|
-
return this._track._backing.getKeyPacket(timestamp, options);
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
const packet = await this._track._backing.getKeyPacket(timestamp, options);
|
|
204
|
-
if (!packet || packet.type === 'delta') {
|
|
205
|
-
return packet;
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
const determinedType = await this._track.determinePacketType(packet);
|
|
209
|
-
if (determinedType === 'delta') {
|
|
210
|
-
// Try returning the previous key packet (in hopes that it's actually a key packet)
|
|
211
|
-
return this.getKeyPacket(packet.timestamp - 1 / this._track.timeResolution, options);
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
return packet;
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
/**
|
|
218
|
-
* Retrieves the key packet following the given packet (in decode order), or null if the given packet is the last
|
|
219
|
-
* key packet.
|
|
220
|
-
*
|
|
221
|
-
* To ensure that the returned packet is guaranteed to be a real key frame, enable `options.verifyKeyPackets`.
|
|
222
|
-
*/
|
|
223
|
-
async getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions = {}): Promise<EncodedPacket | null> {
|
|
224
|
-
if (!(packet instanceof EncodedPacket)) {
|
|
225
|
-
throw new TypeError('packet must be an EncodedPacket.');
|
|
226
|
-
}
|
|
227
|
-
validatePacketRetrievalOptions(options);
|
|
228
|
-
|
|
229
|
-
if (this._track.input._disposed) {
|
|
230
|
-
throw new InputDisposedError();
|
|
231
|
-
}
|
|
232
|
-
|
|
233
|
-
if (!options.verifyKeyPackets) {
|
|
234
|
-
return this._track._backing.getNextKeyPacket(packet, options);
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
const nextPacket = await this._track._backing.getNextKeyPacket(packet, options);
|
|
238
|
-
if (!nextPacket || nextPacket.type === 'delta') {
|
|
239
|
-
return nextPacket;
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
const determinedType = await this._track.determinePacketType(nextPacket);
|
|
243
|
-
if (determinedType === 'delta') {
|
|
244
|
-
// Try returning the next key packet (in hopes that it's actually a key packet)
|
|
245
|
-
return this.getNextKeyPacket(nextPacket, options);
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
return nextPacket;
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
/**
|
|
252
|
-
* Creates an async iterator that yields the packets in this track in decode order. To enable fast iteration, this
|
|
253
|
-
* method will intelligently preload packets based on the speed of the consumer.
|
|
254
|
-
*
|
|
255
|
-
* @param startPacket - (optional) The packet from which iteration should begin. This packet will also be yielded.
|
|
256
|
-
* @param endTimestamp - (optional) The timestamp at which iteration should end. This packet will _not_ be yielded.
|
|
257
|
-
*/
|
|
258
|
-
packets(
|
|
259
|
-
startPacket?: EncodedPacket,
|
|
260
|
-
endPacket?: EncodedPacket,
|
|
261
|
-
options: PacketRetrievalOptions = {},
|
|
262
|
-
): AsyncGenerator<EncodedPacket, void, unknown> {
|
|
263
|
-
if (startPacket !== undefined && !(startPacket instanceof EncodedPacket)) {
|
|
264
|
-
throw new TypeError('startPacket must be an EncodedPacket.');
|
|
265
|
-
}
|
|
266
|
-
if (startPacket !== undefined && startPacket.isMetadataOnly && !options?.metadataOnly) {
|
|
267
|
-
throw new TypeError('startPacket can only be metadata-only if options.metadataOnly is enabled.');
|
|
268
|
-
}
|
|
269
|
-
if (endPacket !== undefined && !(endPacket instanceof EncodedPacket)) {
|
|
270
|
-
throw new TypeError('endPacket must be an EncodedPacket.');
|
|
271
|
-
}
|
|
272
|
-
validatePacketRetrievalOptions(options);
|
|
273
|
-
|
|
274
|
-
if (this._track.input._disposed) {
|
|
275
|
-
throw new InputDisposedError();
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
const packetQueue: EncodedPacket[] = [];
|
|
279
|
-
|
|
280
|
-
let { promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers();
|
|
281
|
-
let { promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers();
|
|
282
|
-
let ended = false;
|
|
283
|
-
let terminated = false;
|
|
284
|
-
|
|
285
|
-
// This stores errors that are "out of band" in the sense that they didn't occur in the normal flow of this
|
|
286
|
-
// method but instead in a different context. This error should not go unnoticed and must be bubbled up to
|
|
287
|
-
// the consumer.
|
|
288
|
-
let outOfBandError = null as Error | null;
|
|
289
|
-
|
|
290
|
-
const timestamps: number[] = [];
|
|
291
|
-
// The queue should always be big enough to hold 1 second worth of packets
|
|
292
|
-
const maxQueueSize = () => Math.max(2, timestamps.length);
|
|
293
|
-
|
|
294
|
-
// The following is the "pump" process that keeps pumping packets into the queue
|
|
295
|
-
(async () => {
|
|
296
|
-
let packet = startPacket ?? await this.getFirstPacket(options);
|
|
297
|
-
|
|
298
|
-
while (packet && !terminated && !this._track.input._disposed) {
|
|
299
|
-
if (endPacket && packet.sequenceNumber >= endPacket?.sequenceNumber) {
|
|
300
|
-
break;
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
if (packetQueue.length > maxQueueSize()) {
|
|
304
|
-
({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers());
|
|
305
|
-
await queueDequeue;
|
|
306
|
-
continue;
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
packetQueue.push(packet);
|
|
310
|
-
|
|
311
|
-
onQueueNotEmpty();
|
|
312
|
-
({ promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers());
|
|
313
|
-
|
|
314
|
-
packet = await this.getNextPacket(packet, options);
|
|
315
|
-
}
|
|
316
|
-
|
|
317
|
-
ended = true;
|
|
318
|
-
onQueueNotEmpty();
|
|
319
|
-
})().catch((error: Error) => {
|
|
320
|
-
if (!outOfBandError) {
|
|
321
|
-
outOfBandError = error;
|
|
322
|
-
onQueueNotEmpty();
|
|
323
|
-
}
|
|
324
|
-
});
|
|
325
|
-
|
|
326
|
-
const track = this._track;
|
|
327
|
-
|
|
328
|
-
return {
|
|
329
|
-
async next() {
|
|
330
|
-
while (true) {
|
|
331
|
-
if (track.input._disposed) {
|
|
332
|
-
throw new InputDisposedError();
|
|
333
|
-
} else if (terminated) {
|
|
334
|
-
return { value: undefined, done: true };
|
|
335
|
-
} else if (outOfBandError) {
|
|
336
|
-
throw outOfBandError;
|
|
337
|
-
} else if (packetQueue.length > 0) {
|
|
338
|
-
const value = packetQueue.shift()!;
|
|
339
|
-
const now = performance.now();
|
|
340
|
-
timestamps.push(now);
|
|
341
|
-
|
|
342
|
-
while (timestamps.length > 0 && now - timestamps[0]! >= 1000) {
|
|
343
|
-
timestamps.shift();
|
|
344
|
-
}
|
|
345
|
-
|
|
346
|
-
onQueueDequeue();
|
|
347
|
-
|
|
348
|
-
return { value, done: false };
|
|
349
|
-
} else if (ended) {
|
|
350
|
-
return { value: undefined, done: true };
|
|
351
|
-
} else {
|
|
352
|
-
await queueNotEmpty;
|
|
353
|
-
}
|
|
354
|
-
}
|
|
355
|
-
},
|
|
356
|
-
async return() {
|
|
357
|
-
terminated = true;
|
|
358
|
-
onQueueDequeue();
|
|
359
|
-
onQueueNotEmpty();
|
|
360
|
-
|
|
361
|
-
return { value: undefined, done: true };
|
|
362
|
-
},
|
|
363
|
-
async throw(error) {
|
|
364
|
-
throw error;
|
|
365
|
-
},
|
|
366
|
-
[Symbol.asyncIterator]() {
|
|
367
|
-
return this;
|
|
368
|
-
},
|
|
369
|
-
};
|
|
370
|
-
}
|
|
371
|
-
}
|
|
372
|
-
|
|
373
|
-
abstract class DecoderWrapper<
|
|
374
|
-
MediaSample extends VideoSample | AudioSample,
|
|
375
|
-
> {
|
|
376
|
-
constructor(
|
|
377
|
-
public onSample: (sample: MediaSample) => unknown,
|
|
378
|
-
public onError: (error: Error) => unknown,
|
|
379
|
-
) {}
|
|
380
|
-
|
|
381
|
-
abstract getDecodeQueueSize(): number;
|
|
382
|
-
abstract decode(packet: EncodedPacket): void;
|
|
383
|
-
abstract flush(): Promise<void>;
|
|
384
|
-
abstract close(): void;
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
/**
|
|
388
|
-
* Base class for decoded media sample sinks.
|
|
389
|
-
* @group Media sinks
|
|
390
|
-
* @public
|
|
391
|
-
*/
|
|
392
|
-
export abstract class BaseMediaSampleSink<
|
|
393
|
-
MediaSample extends VideoSample | AudioSample,
|
|
394
|
-
> {
|
|
395
|
-
/** @internal */
|
|
396
|
-
abstract _track: InputTrack;
|
|
397
|
-
|
|
398
|
-
/** @internal */
|
|
399
|
-
abstract _createDecoder(
|
|
400
|
-
onSample: (sample: MediaSample) => unknown,
|
|
401
|
-
onError: (error: Error) => unknown
|
|
402
|
-
): Promise<DecoderWrapper<MediaSample>>;
|
|
403
|
-
/** @internal */
|
|
404
|
-
abstract _createPacketSink(): EncodedPacketSink;
|
|
405
|
-
|
|
406
|
-
/** @internal */
|
|
407
|
-
protected mediaSamplesInRange(
|
|
408
|
-
startTimestamp = 0,
|
|
409
|
-
endTimestamp = Infinity,
|
|
410
|
-
): AsyncGenerator<MediaSample, void, unknown> {
|
|
411
|
-
validateTimestamp(startTimestamp);
|
|
412
|
-
validateTimestamp(endTimestamp);
|
|
413
|
-
|
|
414
|
-
const sampleQueue: MediaSample[] = [];
|
|
415
|
-
let firstSampleQueued = false;
|
|
416
|
-
let lastSample: MediaSample | null = null;
|
|
417
|
-
let { promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers();
|
|
418
|
-
let { promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers();
|
|
419
|
-
let decoderIsFlushed = false;
|
|
420
|
-
let ended = false;
|
|
421
|
-
let terminated = false;
|
|
422
|
-
|
|
423
|
-
// This stores errors that are "out of band" in the sense that they didn't occur in the normal flow of this
|
|
424
|
-
// method but instead in a different context. This error should not go unnoticed and must be bubbled up to
|
|
425
|
-
// the consumer.
|
|
426
|
-
let outOfBandError = null as Error | null;
|
|
427
|
-
|
|
428
|
-
// The following is the "pump" process that keeps pumping packets into the decoder
|
|
429
|
-
(async () => {
|
|
430
|
-
const decoderError = new Error();
|
|
431
|
-
const decoder = await this._createDecoder((sample) => {
|
|
432
|
-
onQueueDequeue();
|
|
433
|
-
if (sample.timestamp >= endTimestamp) {
|
|
434
|
-
ended = true;
|
|
435
|
-
}
|
|
436
|
-
|
|
437
|
-
if (ended) {
|
|
438
|
-
sample.close();
|
|
439
|
-
return;
|
|
440
|
-
}
|
|
441
|
-
|
|
442
|
-
if (lastSample) {
|
|
443
|
-
if (sample.timestamp > startTimestamp) {
|
|
444
|
-
// We don't know ahead of time what the first first is. This is because the first first is the
|
|
445
|
-
// last first whose timestamp is less than or equal to the start timestamp. Therefore we need to
|
|
446
|
-
// wait for the first first after the start timestamp, and then we'll know that the previous
|
|
447
|
-
// first was the first first.
|
|
448
|
-
sampleQueue.push(lastSample);
|
|
449
|
-
firstSampleQueued = true;
|
|
450
|
-
} else {
|
|
451
|
-
lastSample.close();
|
|
452
|
-
}
|
|
453
|
-
}
|
|
454
|
-
|
|
455
|
-
if (sample.timestamp >= startTimestamp) {
|
|
456
|
-
sampleQueue.push(sample);
|
|
457
|
-
firstSampleQueued = true;
|
|
458
|
-
}
|
|
459
|
-
|
|
460
|
-
lastSample = firstSampleQueued ? null : sample;
|
|
461
|
-
|
|
462
|
-
if (sampleQueue.length > 0) {
|
|
463
|
-
onQueueNotEmpty();
|
|
464
|
-
({ promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers());
|
|
465
|
-
}
|
|
466
|
-
}, (error) => {
|
|
467
|
-
if (!outOfBandError) {
|
|
468
|
-
error.stack = decoderError.stack; // Provide a more useful stack trace
|
|
469
|
-
outOfBandError = error;
|
|
470
|
-
onQueueNotEmpty();
|
|
471
|
-
}
|
|
472
|
-
});
|
|
473
|
-
|
|
474
|
-
const packetSink = this._createPacketSink();
|
|
475
|
-
const keyPacket = await packetSink.getKeyPacket(startTimestamp, { verifyKeyPackets: true })
|
|
476
|
-
?? await packetSink.getFirstPacket();
|
|
477
|
-
if (!keyPacket) {
|
|
478
|
-
return;
|
|
479
|
-
}
|
|
480
|
-
|
|
481
|
-
let currentPacket: EncodedPacket | null = keyPacket;
|
|
482
|
-
|
|
483
|
-
let endPacket: EncodedPacket | undefined = undefined;
|
|
484
|
-
if (endTimestamp < Infinity) {
|
|
485
|
-
// When an end timestamp is set, we cannot simply use that for the packet iterator due to out-of-order
|
|
486
|
-
// frames (B-frames). Instead, we'll need to keep decoding packets until we get a frame that exceeds
|
|
487
|
-
// this end time. However, we can still put a bound on it: Since key frames are by definition never
|
|
488
|
-
// out of order, we can stop at the first key frame after the end timestamp.
|
|
489
|
-
const packet = await packetSink.getPacket(endTimestamp);
|
|
490
|
-
const keyPacket = !packet
|
|
491
|
-
? null
|
|
492
|
-
: packet.type === 'key' && packet.timestamp === endTimestamp
|
|
493
|
-
? packet
|
|
494
|
-
: await packetSink.getNextKeyPacket(packet, { verifyKeyPackets: true });
|
|
495
|
-
|
|
496
|
-
if (keyPacket) {
|
|
497
|
-
endPacket = keyPacket;
|
|
498
|
-
}
|
|
499
|
-
}
|
|
500
|
-
|
|
501
|
-
const packets = packetSink.packets(keyPacket, endPacket);
|
|
502
|
-
await packets.next(); // Skip the start packet as we already have it
|
|
503
|
-
|
|
504
|
-
while (currentPacket && !ended && !this._track.input._disposed) {
|
|
505
|
-
const maxQueueSize = computeMaxQueueSize(sampleQueue.length);
|
|
506
|
-
if (sampleQueue.length + decoder.getDecodeQueueSize() > maxQueueSize) {
|
|
507
|
-
({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers());
|
|
508
|
-
await queueDequeue;
|
|
509
|
-
continue;
|
|
510
|
-
}
|
|
511
|
-
|
|
512
|
-
decoder.decode(currentPacket);
|
|
513
|
-
|
|
514
|
-
const packetResult = await packets.next();
|
|
515
|
-
if (packetResult.done) {
|
|
516
|
-
break;
|
|
517
|
-
}
|
|
518
|
-
|
|
519
|
-
currentPacket = packetResult.value;
|
|
520
|
-
}
|
|
521
|
-
|
|
522
|
-
await packets.return();
|
|
523
|
-
|
|
524
|
-
if (!terminated && !this._track.input._disposed) {
|
|
525
|
-
await decoder.flush();
|
|
526
|
-
}
|
|
527
|
-
decoder.close();
|
|
528
|
-
|
|
529
|
-
if (!firstSampleQueued && lastSample) {
|
|
530
|
-
sampleQueue.push(lastSample);
|
|
531
|
-
}
|
|
532
|
-
|
|
533
|
-
decoderIsFlushed = true;
|
|
534
|
-
onQueueNotEmpty(); // To unstuck the generator
|
|
535
|
-
})().catch((error: Error) => {
|
|
536
|
-
if (!outOfBandError) {
|
|
537
|
-
outOfBandError = error;
|
|
538
|
-
onQueueNotEmpty();
|
|
539
|
-
}
|
|
540
|
-
});
|
|
541
|
-
|
|
542
|
-
const track = this._track;
|
|
543
|
-
const closeSamples = () => {
|
|
544
|
-
lastSample?.close();
|
|
545
|
-
for (const sample of sampleQueue) {
|
|
546
|
-
sample.close();
|
|
547
|
-
}
|
|
548
|
-
};
|
|
549
|
-
|
|
550
|
-
return {
|
|
551
|
-
async next() {
|
|
552
|
-
while (true) {
|
|
553
|
-
if (track.input._disposed) {
|
|
554
|
-
closeSamples();
|
|
555
|
-
throw new InputDisposedError();
|
|
556
|
-
} else if (terminated) {
|
|
557
|
-
return { value: undefined, done: true };
|
|
558
|
-
} else if (outOfBandError) {
|
|
559
|
-
closeSamples();
|
|
560
|
-
throw outOfBandError;
|
|
561
|
-
} else if (sampleQueue.length > 0) {
|
|
562
|
-
const value = sampleQueue.shift()!;
|
|
563
|
-
onQueueDequeue();
|
|
564
|
-
return { value, done: false };
|
|
565
|
-
} else if (!decoderIsFlushed) {
|
|
566
|
-
await queueNotEmpty;
|
|
567
|
-
} else {
|
|
568
|
-
return { value: undefined, done: true };
|
|
569
|
-
}
|
|
570
|
-
}
|
|
571
|
-
},
|
|
572
|
-
async return() {
|
|
573
|
-
terminated = true;
|
|
574
|
-
ended = true;
|
|
575
|
-
onQueueDequeue();
|
|
576
|
-
onQueueNotEmpty();
|
|
577
|
-
closeSamples();
|
|
578
|
-
|
|
579
|
-
return { value: undefined, done: true };
|
|
580
|
-
},
|
|
581
|
-
async throw(error) {
|
|
582
|
-
throw error;
|
|
583
|
-
},
|
|
584
|
-
[Symbol.asyncIterator]() {
|
|
585
|
-
return this;
|
|
586
|
-
},
|
|
587
|
-
};
|
|
588
|
-
}
|
|
589
|
-
|
|
590
|
-
/** @internal */
|
|
591
|
-
protected mediaSamplesAtTimestamps(
|
|
592
|
-
timestamps: AnyIterable<number>,
|
|
593
|
-
): AsyncGenerator<MediaSample | null, void, unknown> {
|
|
594
|
-
validateAnyIterable(timestamps);
|
|
595
|
-
const timestampIterator = toAsyncIterator(timestamps);
|
|
596
|
-
const timestampsOfInterest: number[] = [];
|
|
597
|
-
|
|
598
|
-
const sampleQueue: (MediaSample | null)[] = [];
|
|
599
|
-
let { promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers();
|
|
600
|
-
let { promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers();
|
|
601
|
-
let decoderIsFlushed = false;
|
|
602
|
-
let terminated = false;
|
|
603
|
-
|
|
604
|
-
// This stores errors that are "out of band" in the sense that they didn't occur in the normal flow of this
|
|
605
|
-
// method but instead in a different context. This error should not go unnoticed and must be bubbled up to
|
|
606
|
-
// the consumer.
|
|
607
|
-
let outOfBandError = null as Error | null;
|
|
608
|
-
|
|
609
|
-
const pushToQueue = (sample: MediaSample | null) => {
|
|
610
|
-
sampleQueue.push(sample);
|
|
611
|
-
onQueueNotEmpty();
|
|
612
|
-
({ promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers());
|
|
613
|
-
};
|
|
614
|
-
|
|
615
|
-
// The following is the "pump" process that keeps pumping packets into the decoder
|
|
616
|
-
(async () => {
|
|
617
|
-
const decoderError = new Error();
|
|
618
|
-
const decoder = await this._createDecoder((sample) => {
|
|
619
|
-
onQueueDequeue();
|
|
620
|
-
|
|
621
|
-
if (terminated) {
|
|
622
|
-
sample.close();
|
|
623
|
-
return;
|
|
624
|
-
}
|
|
625
|
-
|
|
626
|
-
let sampleUses = 0;
|
|
627
|
-
while (
|
|
628
|
-
timestampsOfInterest.length > 0
|
|
629
|
-
&& sample.timestamp - timestampsOfInterest[0]! > -1e-10 // Give it a little epsilon
|
|
630
|
-
) {
|
|
631
|
-
sampleUses++;
|
|
632
|
-
timestampsOfInterest.shift();
|
|
633
|
-
}
|
|
634
|
-
|
|
635
|
-
if (sampleUses > 0) {
|
|
636
|
-
for (let i = 0; i < sampleUses; i++) {
|
|
637
|
-
// Clone the sample if we need to emit it multiple times
|
|
638
|
-
pushToQueue((i < sampleUses - 1 ? sample.clone() : sample) as MediaSample);
|
|
639
|
-
}
|
|
640
|
-
} else {
|
|
641
|
-
sample.close();
|
|
642
|
-
}
|
|
643
|
-
}, (error) => {
|
|
644
|
-
if (!outOfBandError) {
|
|
645
|
-
error.stack = decoderError.stack; // Provide a more useful stack trace
|
|
646
|
-
outOfBandError = error;
|
|
647
|
-
onQueueNotEmpty();
|
|
648
|
-
}
|
|
649
|
-
});
|
|
650
|
-
|
|
651
|
-
const packetSink = this._createPacketSink();
|
|
652
|
-
let lastPacket: EncodedPacket | null = null;
|
|
653
|
-
let lastKeyPacket: EncodedPacket | null = null;
|
|
654
|
-
|
|
655
|
-
// The end sequence number (inclusive) in the next batch of packets that will be decoded. The batch starts
|
|
656
|
-
// at the last key frame and goes until this sequence number.
|
|
657
|
-
let maxSequenceNumber = -1;
|
|
658
|
-
|
|
659
|
-
const decodePackets = async () => {
|
|
660
|
-
assert(lastKeyPacket);
|
|
661
|
-
|
|
662
|
-
// Start at the current key packet
|
|
663
|
-
let currentPacket = lastKeyPacket;
|
|
664
|
-
decoder.decode(currentPacket);
|
|
665
|
-
|
|
666
|
-
while (currentPacket.sequenceNumber < maxSequenceNumber) {
|
|
667
|
-
const maxQueueSize = computeMaxQueueSize(sampleQueue.length);
|
|
668
|
-
while (sampleQueue.length + decoder.getDecodeQueueSize() > maxQueueSize && !terminated) {
|
|
669
|
-
({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers());
|
|
670
|
-
await queueDequeue;
|
|
671
|
-
}
|
|
672
|
-
|
|
673
|
-
if (terminated) {
|
|
674
|
-
break;
|
|
675
|
-
}
|
|
676
|
-
|
|
677
|
-
const nextPacket = await packetSink.getNextPacket(currentPacket);
|
|
678
|
-
assert(nextPacket);
|
|
679
|
-
|
|
680
|
-
decoder.decode(nextPacket);
|
|
681
|
-
currentPacket = nextPacket;
|
|
682
|
-
}
|
|
683
|
-
|
|
684
|
-
maxSequenceNumber = -1;
|
|
685
|
-
};
|
|
686
|
-
|
|
687
|
-
const flushDecoder = async () => {
|
|
688
|
-
await decoder.flush();
|
|
689
|
-
|
|
690
|
-
// We don't expect this list to have any elements in it anymore, but in case it does, let's emit
|
|
691
|
-
// nulls for every remaining element, then clear it.
|
|
692
|
-
for (let i = 0; i < timestampsOfInterest.length; i++) {
|
|
693
|
-
pushToQueue(null);
|
|
694
|
-
}
|
|
695
|
-
timestampsOfInterest.length = 0;
|
|
696
|
-
};
|
|
697
|
-
|
|
698
|
-
for await (const timestamp of timestampIterator) {
|
|
699
|
-
validateTimestamp(timestamp);
|
|
700
|
-
|
|
701
|
-
if (terminated || this._track.input._disposed) {
|
|
702
|
-
break;
|
|
703
|
-
}
|
|
704
|
-
|
|
705
|
-
const targetPacket = await packetSink.getPacket(timestamp);
|
|
706
|
-
const keyPacket = targetPacket && await packetSink.getKeyPacket(timestamp, { verifyKeyPackets: true });
|
|
707
|
-
|
|
708
|
-
if (!keyPacket) {
|
|
709
|
-
if (maxSequenceNumber !== -1) {
|
|
710
|
-
await decodePackets();
|
|
711
|
-
await flushDecoder();
|
|
712
|
-
}
|
|
713
|
-
|
|
714
|
-
pushToQueue(null);
|
|
715
|
-
lastPacket = null;
|
|
716
|
-
continue;
|
|
717
|
-
}
|
|
718
|
-
|
|
719
|
-
// Check if the key packet has changed or if we're going back in time
|
|
720
|
-
if (
|
|
721
|
-
lastPacket
|
|
722
|
-
&& (
|
|
723
|
-
keyPacket.sequenceNumber !== lastKeyPacket!.sequenceNumber
|
|
724
|
-
|| targetPacket.timestamp < lastPacket.timestamp
|
|
725
|
-
)
|
|
726
|
-
) {
|
|
727
|
-
await decodePackets();
|
|
728
|
-
await flushDecoder(); // Always flush here, improves decoder compatibility
|
|
729
|
-
}
|
|
730
|
-
|
|
731
|
-
timestampsOfInterest.push(targetPacket.timestamp);
|
|
732
|
-
maxSequenceNumber = Math.max(targetPacket.sequenceNumber, maxSequenceNumber);
|
|
733
|
-
|
|
734
|
-
lastPacket = targetPacket;
|
|
735
|
-
lastKeyPacket = keyPacket;
|
|
736
|
-
}
|
|
737
|
-
|
|
738
|
-
if (!terminated && !this._track.input._disposed) {
|
|
739
|
-
if (maxSequenceNumber !== -1) {
|
|
740
|
-
// We still need to decode packets
|
|
741
|
-
await decodePackets();
|
|
742
|
-
}
|
|
743
|
-
|
|
744
|
-
await flushDecoder();
|
|
745
|
-
}
|
|
746
|
-
decoder.close();
|
|
747
|
-
|
|
748
|
-
decoderIsFlushed = true;
|
|
749
|
-
onQueueNotEmpty(); // To unstuck the generator
|
|
750
|
-
})().catch((error: Error) => {
|
|
751
|
-
if (!outOfBandError) {
|
|
752
|
-
outOfBandError = error;
|
|
753
|
-
onQueueNotEmpty();
|
|
754
|
-
}
|
|
755
|
-
});
|
|
756
|
-
|
|
757
|
-
const track = this._track;
|
|
758
|
-
const closeSamples = () => {
|
|
759
|
-
for (const sample of sampleQueue) {
|
|
760
|
-
sample?.close();
|
|
761
|
-
}
|
|
762
|
-
};
|
|
763
|
-
|
|
764
|
-
return {
|
|
765
|
-
async next() {
|
|
766
|
-
while (true) {
|
|
767
|
-
if (track.input._disposed) {
|
|
768
|
-
closeSamples();
|
|
769
|
-
throw new InputDisposedError();
|
|
770
|
-
} else if (terminated) {
|
|
771
|
-
return { value: undefined, done: true };
|
|
772
|
-
} else if (outOfBandError) {
|
|
773
|
-
closeSamples();
|
|
774
|
-
throw outOfBandError;
|
|
775
|
-
} else if (sampleQueue.length > 0) {
|
|
776
|
-
const value = sampleQueue.shift();
|
|
777
|
-
assert(value !== undefined);
|
|
778
|
-
onQueueDequeue();
|
|
779
|
-
return { value, done: false };
|
|
780
|
-
} else if (!decoderIsFlushed) {
|
|
781
|
-
await queueNotEmpty;
|
|
782
|
-
} else {
|
|
783
|
-
return { value: undefined, done: true };
|
|
784
|
-
}
|
|
785
|
-
}
|
|
786
|
-
},
|
|
787
|
-
async return() {
|
|
788
|
-
terminated = true;
|
|
789
|
-
onQueueDequeue();
|
|
790
|
-
onQueueNotEmpty();
|
|
791
|
-
closeSamples();
|
|
792
|
-
|
|
793
|
-
return { value: undefined, done: true };
|
|
794
|
-
},
|
|
795
|
-
async throw(error) {
|
|
796
|
-
throw error;
|
|
797
|
-
},
|
|
798
|
-
[Symbol.asyncIterator]() {
|
|
799
|
-
return this;
|
|
800
|
-
},
|
|
801
|
-
};
|
|
802
|
-
}
|
|
803
|
-
}
|
|
804
|
-
|
|
805
|
-
const computeMaxQueueSize = (decodedSampleQueueSize: number) => {
|
|
806
|
-
// If we have decoded samples lying around, limit the total queue size to a small value (decoded samples can use up
|
|
807
|
-
// a lot of memory). If not, we're fine with a much bigger queue of encoded packets waiting to be decoded. In fact,
|
|
808
|
-
// some decoders only start flushing out decoded chunks when the packet queue is large enough.
|
|
809
|
-
return decodedSampleQueueSize === 0 ? 40 : 8;
|
|
810
|
-
};
|
|
811
|
-
|
|
812
|
-
class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
|
|
813
|
-
decoder: VideoDecoder | null = null;
|
|
814
|
-
|
|
815
|
-
customDecoder: CustomVideoDecoder | null = null;
|
|
816
|
-
customDecoderCallSerializer = new CallSerializer();
|
|
817
|
-
customDecoderQueueSize = 0;
|
|
818
|
-
|
|
819
|
-
inputTimestamps: number[] = []; // Timestamps input into the decoder, sorted.
|
|
820
|
-
sampleQueue: VideoSample[] = []; // Safari-specific thing, check usage.
|
|
821
|
-
currentPacketIndex = 0;
|
|
822
|
-
raslSkipped = false; // For HEVC stuff
|
|
823
|
-
|
|
824
|
-
// Alpha stuff
|
|
825
|
-
alphaDecoder: VideoDecoder | null = null;
|
|
826
|
-
alphaHadKeyframe = false;
|
|
827
|
-
colorQueue: VideoFrame[] = [];
|
|
828
|
-
alphaQueue: (VideoFrame | null)[] = [];
|
|
829
|
-
merger: ColorAlphaMerger | null = null;
|
|
830
|
-
mergerCreationFailed = false;
|
|
831
|
-
decodedAlphaChunkCount = 0;
|
|
832
|
-
alphaDecoderQueueSize = 0;
|
|
833
|
-
/** Each value is the number of decoded alpha chunks at which a null alpha frame should be added. */
|
|
834
|
-
nullAlphaFrameQueue: number[] = [];
|
|
835
|
-
currentAlphaPacketIndex = 0;
|
|
836
|
-
alphaRaslSkipped = false; // For HEVC stuff
|
|
837
|
-
|
|
838
|
-
constructor(
|
|
839
|
-
onSample: (sample: VideoSample) => unknown,
|
|
840
|
-
onError: (error: Error) => unknown,
|
|
841
|
-
public codec: VideoCodec,
|
|
842
|
-
public decoderConfig: VideoDecoderConfig,
|
|
843
|
-
public rotation: Rotation,
|
|
844
|
-
public timeResolution: number,
|
|
845
|
-
) {
|
|
846
|
-
super(onSample, onError);
|
|
847
|
-
|
|
848
|
-
const MatchingCustomDecoder = customVideoDecoders.find(x => x.supports(codec, decoderConfig));
|
|
849
|
-
if (MatchingCustomDecoder) {
|
|
850
|
-
// @ts-expect-error "Can't create instance of abstract class 🤓"
|
|
851
|
-
this.customDecoder = new MatchingCustomDecoder() as CustomVideoDecoder;
|
|
852
|
-
// @ts-expect-error It's technically readonly
|
|
853
|
-
this.customDecoder.codec = codec;
|
|
854
|
-
// @ts-expect-error It's technically readonly
|
|
855
|
-
this.customDecoder.config = decoderConfig;
|
|
856
|
-
// @ts-expect-error It's technically readonly
|
|
857
|
-
this.customDecoder.onSample = (sample) => {
|
|
858
|
-
if (!(sample instanceof VideoSample)) {
|
|
859
|
-
throw new TypeError('The argument passed to onSample must be a VideoSample.');
|
|
860
|
-
}
|
|
861
|
-
|
|
862
|
-
this.finalizeAndEmitSample(sample);
|
|
863
|
-
};
|
|
864
|
-
|
|
865
|
-
void this.customDecoderCallSerializer.call(() => this.customDecoder!.init());
|
|
866
|
-
} else {
|
|
867
|
-
const colorHandler = (frame: VideoFrame) => {
|
|
868
|
-
if (this.alphaQueue.length > 0) {
|
|
869
|
-
// Even when no alpha data is present (most of the time), there will be nulls in this queue
|
|
870
|
-
const alphaFrame = this.alphaQueue.shift();
|
|
871
|
-
assert(alphaFrame !== undefined);
|
|
872
|
-
|
|
873
|
-
this.mergeAlpha(frame, alphaFrame);
|
|
874
|
-
} else {
|
|
875
|
-
this.colorQueue.push(frame);
|
|
876
|
-
}
|
|
877
|
-
};
|
|
878
|
-
|
|
879
|
-
if (codec === 'avc' && this.decoderConfig.description && isChromium()) {
|
|
880
|
-
// Chromium has/had a bug with playing interlaced AVC (https://issues.chromium.org/issues/456919096)
|
|
881
|
-
// which can be worked around by requesting that software decoding be used. So, here we peek into the
|
|
882
|
-
// AVC description, if present, and switch to software decoding if we find interlaced content.
|
|
883
|
-
const record = deserializeAvcDecoderConfigurationRecord(toUint8Array(this.decoderConfig.description));
|
|
884
|
-
if (record && record.sequenceParameterSets.length > 0) {
|
|
885
|
-
const sps = parseAvcSps(record.sequenceParameterSets[0]!);
|
|
886
|
-
if (sps && sps.frameMbsOnlyFlag === 0) {
|
|
887
|
-
this.decoderConfig = {
|
|
888
|
-
...this.decoderConfig,
|
|
889
|
-
hardwareAcceleration: 'prefer-software',
|
|
890
|
-
};
|
|
891
|
-
}
|
|
892
|
-
}
|
|
893
|
-
}
|
|
894
|
-
|
|
895
|
-
this.decoder = new VideoDecoder({
|
|
896
|
-
output: (frame) => {
|
|
897
|
-
try {
|
|
898
|
-
colorHandler(frame);
|
|
899
|
-
} catch (error) {
|
|
900
|
-
this.onError(error as Error);
|
|
901
|
-
}
|
|
902
|
-
},
|
|
903
|
-
error: onError,
|
|
904
|
-
});
|
|
905
|
-
this.decoder.configure(this.decoderConfig);
|
|
906
|
-
}
|
|
907
|
-
}
|
|
908
|
-
|
|
909
|
-
getDecodeQueueSize() {
|
|
910
|
-
if (this.customDecoder) {
|
|
911
|
-
return this.customDecoderQueueSize;
|
|
912
|
-
} else {
|
|
913
|
-
assert(this.decoder);
|
|
914
|
-
|
|
915
|
-
return Math.max(
|
|
916
|
-
this.decoder.decodeQueueSize,
|
|
917
|
-
this.alphaDecoder?.decodeQueueSize ?? 0,
|
|
918
|
-
);
|
|
919
|
-
}
|
|
920
|
-
}
|
|
921
|
-
|
|
922
|
-
decode(packet: EncodedPacket) {
|
|
923
|
-
if (this.codec === 'hevc' && this.currentPacketIndex > 0 && !this.raslSkipped) {
|
|
924
|
-
if (this.hasHevcRaslPicture(packet.data)) {
|
|
925
|
-
return; // Drop
|
|
926
|
-
}
|
|
927
|
-
|
|
928
|
-
this.raslSkipped = true;
|
|
929
|
-
}
|
|
930
|
-
|
|
931
|
-
this.currentPacketIndex++;
|
|
932
|
-
|
|
933
|
-
if (this.customDecoder) {
|
|
934
|
-
this.customDecoderQueueSize++;
|
|
935
|
-
void this.customDecoderCallSerializer
|
|
936
|
-
.call(() => this.customDecoder!.decode(packet))
|
|
937
|
-
.then(() => this.customDecoderQueueSize--);
|
|
938
|
-
} else {
|
|
939
|
-
assert(this.decoder);
|
|
940
|
-
|
|
941
|
-
if (!isWebKit()) {
|
|
942
|
-
insertSorted(this.inputTimestamps, packet.timestamp, x => x);
|
|
943
|
-
}
|
|
944
|
-
|
|
945
|
-
this.decoder.decode(packet.toEncodedVideoChunk());
|
|
946
|
-
this.decodeAlphaData(packet);
|
|
947
|
-
}
|
|
948
|
-
}
|
|
949
|
-
|
|
950
|
-
decodeAlphaData(packet: EncodedPacket) {
|
|
951
|
-
if (!packet.sideData.alpha || this.mergerCreationFailed) {
|
|
952
|
-
// No alpha side data in the packet, most common case
|
|
953
|
-
this.pushNullAlphaFrame();
|
|
954
|
-
return;
|
|
955
|
-
}
|
|
956
|
-
|
|
957
|
-
if (!this.merger) {
|
|
958
|
-
try {
|
|
959
|
-
this.merger = new ColorAlphaMerger();
|
|
960
|
-
} catch (error) {
|
|
961
|
-
console.error('Due to an error, only color data will be decoded.', error);
|
|
962
|
-
|
|
963
|
-
this.mergerCreationFailed = true;
|
|
964
|
-
this.decodeAlphaData(packet); // Go again
|
|
965
|
-
|
|
966
|
-
return;
|
|
967
|
-
}
|
|
968
|
-
}
|
|
969
|
-
|
|
970
|
-
// Check if we need to set up the alpha decoder
|
|
971
|
-
if (!this.alphaDecoder) {
|
|
972
|
-
const alphaHandler = (frame: VideoFrame) => {
|
|
973
|
-
this.alphaDecoderQueueSize--;
|
|
974
|
-
|
|
975
|
-
if (this.colorQueue.length > 0) {
|
|
976
|
-
const colorFrame = this.colorQueue.shift();
|
|
977
|
-
assert(colorFrame !== undefined);
|
|
978
|
-
|
|
979
|
-
this.mergeAlpha(colorFrame, frame);
|
|
980
|
-
} else {
|
|
981
|
-
this.alphaQueue.push(frame);
|
|
982
|
-
}
|
|
983
|
-
|
|
984
|
-
// Check if any null frames have been queued for this point
|
|
985
|
-
this.decodedAlphaChunkCount++;
|
|
986
|
-
while (
|
|
987
|
-
this.nullAlphaFrameQueue.length > 0
|
|
988
|
-
&& this.nullAlphaFrameQueue[0] === this.decodedAlphaChunkCount
|
|
989
|
-
) {
|
|
990
|
-
this.nullAlphaFrameQueue.shift();
|
|
991
|
-
|
|
992
|
-
if (this.colorQueue.length > 0) {
|
|
993
|
-
const colorFrame = this.colorQueue.shift();
|
|
994
|
-
assert(colorFrame !== undefined);
|
|
995
|
-
|
|
996
|
-
this.mergeAlpha(colorFrame, null);
|
|
997
|
-
} else {
|
|
998
|
-
this.alphaQueue.push(null);
|
|
999
|
-
}
|
|
1000
|
-
}
|
|
1001
|
-
};
|
|
1002
|
-
|
|
1003
|
-
this.alphaDecoder = new VideoDecoder({
|
|
1004
|
-
output: (frame) => {
|
|
1005
|
-
try {
|
|
1006
|
-
alphaHandler(frame);
|
|
1007
|
-
} catch (error) {
|
|
1008
|
-
this.onError(error as Error);
|
|
1009
|
-
}
|
|
1010
|
-
},
|
|
1011
|
-
error: this.onError,
|
|
1012
|
-
});
|
|
1013
|
-
this.alphaDecoder.configure(this.decoderConfig);
|
|
1014
|
-
}
|
|
1015
|
-
|
|
1016
|
-
const type = determineVideoPacketType(this.codec, this.decoderConfig, packet.sideData.alpha);
|
|
1017
|
-
|
|
1018
|
-
// Alpha packets might follow a different key frame rhythm than the main packets. Therefore, before we start
|
|
1019
|
-
// decoding, we must first find a packet that's actually a key frame. Until then, we treat the image as opaque.
|
|
1020
|
-
if (!this.alphaHadKeyframe) {
|
|
1021
|
-
this.alphaHadKeyframe = type === 'key';
|
|
1022
|
-
}
|
|
1023
|
-
|
|
1024
|
-
if (this.alphaHadKeyframe) {
|
|
1025
|
-
// Same RASL skipping logic as for color, unlikely to be hit (since who uses HEVC with separate alpha??) but
|
|
1026
|
-
// here for symmetry.
|
|
1027
|
-
if (this.codec === 'hevc' && this.currentAlphaPacketIndex > 0 && !this.alphaRaslSkipped) {
|
|
1028
|
-
if (this.hasHevcRaslPicture(packet.sideData.alpha)) {
|
|
1029
|
-
this.pushNullAlphaFrame();
|
|
1030
|
-
return;
|
|
1031
|
-
}
|
|
1032
|
-
|
|
1033
|
-
this.alphaRaslSkipped = true;
|
|
1034
|
-
}
|
|
1035
|
-
|
|
1036
|
-
this.currentAlphaPacketIndex++;
|
|
1037
|
-
this.alphaDecoder.decode(packet.alphaToEncodedVideoChunk(type ?? packet.type));
|
|
1038
|
-
this.alphaDecoderQueueSize++;
|
|
1039
|
-
} else {
|
|
1040
|
-
this.pushNullAlphaFrame();
|
|
1041
|
-
}
|
|
1042
|
-
}
|
|
1043
|
-
|
|
1044
|
-
pushNullAlphaFrame() {
|
|
1045
|
-
if (this.alphaDecoderQueueSize === 0) {
|
|
1046
|
-
// Easy
|
|
1047
|
-
this.alphaQueue.push(null);
|
|
1048
|
-
} else {
|
|
1049
|
-
// There are still alpha chunks being decoded, so pushing `null` immediately would result in out-of-order
|
|
1050
|
-
// data and be incorrect. Instead, we need to enqueue a "null frame" for when the current decoder workload
|
|
1051
|
-
// has finished.
|
|
1052
|
-
this.nullAlphaFrameQueue.push(this.decodedAlphaChunkCount + this.alphaDecoderQueueSize);
|
|
1053
|
-
}
|
|
1054
|
-
}
|
|
1055
|
-
|
|
1056
|
-
/**
|
|
1057
|
-
* If we're using HEVC, we need to make sure to skip any RASL slices that follow a non-IDR key frame such as
|
|
1058
|
-
* CRA_NUT. This is because RASL slices cannot be decoded without data before the CRA_NUT. Browsers behave
|
|
1059
|
-
* differently here: Chromium drops the packets, Safari throws a decoder error. Either way, it's not good
|
|
1060
|
-
* and causes bugs upstream. So, let's take the dropping into our own hands.
|
|
1061
|
-
*/
|
|
1062
|
-
hasHevcRaslPicture(packetData: Uint8Array) {
|
|
1063
|
-
const nalUnits = extractHevcNalUnits(packetData, this.decoderConfig);
|
|
1064
|
-
return nalUnits.some((x) => {
|
|
1065
|
-
const type = extractNalUnitTypeForHevc(x);
|
|
1066
|
-
return type === HevcNalUnitType.RASL_N || type === HevcNalUnitType.RASL_R;
|
|
1067
|
-
});
|
|
1068
|
-
}
|
|
1069
|
-
|
|
1070
|
-
/** Handler for the WebCodecs VideoDecoder for ironing out browser differences. */
|
|
1071
|
-
sampleHandler(sample: VideoSample) {
|
|
1072
|
-
if (isWebKit()) {
|
|
1073
|
-
// For correct B-frame handling, we don't just hand over the frames directly but instead add them to
|
|
1074
|
-
// a queue, because we want to ensure frames are emitted in presentation order. We flush the queue
|
|
1075
|
-
// each time we receive a frame with a timestamp larger than the highest we've seen so far, as we
|
|
1076
|
-
// can sure that is not a B-frame. Typically, WebCodecs automatically guarantees that frames are
|
|
1077
|
-
// emitted in presentation order, but Safari doesn't always follow this rule.
|
|
1078
|
-
if (this.sampleQueue.length > 0 && (sample.timestamp >= last(this.sampleQueue)!.timestamp)) {
|
|
1079
|
-
for (const sample of this.sampleQueue) {
|
|
1080
|
-
this.finalizeAndEmitSample(sample);
|
|
1081
|
-
}
|
|
1082
|
-
|
|
1083
|
-
this.sampleQueue.length = 0;
|
|
1084
|
-
}
|
|
1085
|
-
|
|
1086
|
-
insertSorted(this.sampleQueue, sample, x => x.timestamp);
|
|
1087
|
-
} else {
|
|
1088
|
-
// Assign it the next earliest timestamp from the input. We do this because browsers, by spec, are
|
|
1089
|
-
// required to emit decoded frames in presentation order *while* retaining the timestamp of their
|
|
1090
|
-
// originating EncodedVideoChunk. For files with B-frames but no out-of-order timestamps (like a
|
|
1091
|
-
// missing ctts box, for example), this causes a mismatch. We therefore fix the timestamps and
|
|
1092
|
-
// ensure they are sorted by doing this.
|
|
1093
|
-
const timestamp = this.inputTimestamps.shift();
|
|
1094
|
-
|
|
1095
|
-
// There's no way we'd have more decoded frames than encoded packets we passed in. Actually, the
|
|
1096
|
-
// correspondence should be 1:1.
|
|
1097
|
-
assert(timestamp !== undefined);
|
|
1098
|
-
|
|
1099
|
-
sample.setTimestamp(timestamp);
|
|
1100
|
-
this.finalizeAndEmitSample(sample);
|
|
1101
|
-
}
|
|
1102
|
-
}
|
|
1103
|
-
|
|
1104
|
-
finalizeAndEmitSample(sample: VideoSample) {
|
|
1105
|
-
// Round the timestamps to the time resolution
|
|
1106
|
-
sample.setTimestamp(Math.round(sample.timestamp * this.timeResolution) / this.timeResolution);
|
|
1107
|
-
sample.setDuration(Math.round(sample.duration * this.timeResolution) / this.timeResolution);
|
|
1108
|
-
sample.setRotation(this.rotation);
|
|
1109
|
-
|
|
1110
|
-
this.onSample(sample);
|
|
1111
|
-
}
|
|
1112
|
-
|
|
1113
|
-
mergeAlpha(color: VideoFrame, alpha: VideoFrame | null) {
|
|
1114
|
-
if (!alpha) {
|
|
1115
|
-
// Nothing needs to be merged
|
|
1116
|
-
const finalSample = new VideoSample(color);
|
|
1117
|
-
this.sampleHandler(finalSample);
|
|
1118
|
-
|
|
1119
|
-
return;
|
|
1120
|
-
}
|
|
1121
|
-
|
|
1122
|
-
assert(this.merger);
|
|
1123
|
-
|
|
1124
|
-
this.merger.update(color, alpha);
|
|
1125
|
-
color.close();
|
|
1126
|
-
alpha.close();
|
|
1127
|
-
|
|
1128
|
-
const finalFrame = new VideoFrame(this.merger.canvas, {
|
|
1129
|
-
timestamp: color.timestamp,
|
|
1130
|
-
duration: color.duration ?? undefined,
|
|
1131
|
-
});
|
|
1132
|
-
|
|
1133
|
-
const finalSample = new VideoSample(finalFrame);
|
|
1134
|
-
this.sampleHandler(finalSample);
|
|
1135
|
-
}
|
|
1136
|
-
|
|
1137
|
-
async flush() {
|
|
1138
|
-
if (this.customDecoder) {
|
|
1139
|
-
await this.customDecoderCallSerializer.call(() => this.customDecoder!.flush());
|
|
1140
|
-
} else {
|
|
1141
|
-
assert(this.decoder);
|
|
1142
|
-
await Promise.all([
|
|
1143
|
-
this.decoder.flush(),
|
|
1144
|
-
this.alphaDecoder?.flush(),
|
|
1145
|
-
]);
|
|
1146
|
-
|
|
1147
|
-
this.colorQueue.forEach(x => x.close());
|
|
1148
|
-
this.colorQueue.length = 0;
|
|
1149
|
-
this.alphaQueue.forEach(x => x?.close());
|
|
1150
|
-
this.alphaQueue.length = 0;
|
|
1151
|
-
|
|
1152
|
-
this.alphaHadKeyframe = false;
|
|
1153
|
-
this.decodedAlphaChunkCount = 0;
|
|
1154
|
-
this.alphaDecoderQueueSize = 0;
|
|
1155
|
-
this.nullAlphaFrameQueue.length = 0;
|
|
1156
|
-
this.currentAlphaPacketIndex = 0;
|
|
1157
|
-
this.alphaRaslSkipped = false;
|
|
1158
|
-
}
|
|
1159
|
-
|
|
1160
|
-
if (isWebKit()) {
|
|
1161
|
-
for (const sample of this.sampleQueue) {
|
|
1162
|
-
this.finalizeAndEmitSample(sample);
|
|
1163
|
-
}
|
|
1164
|
-
|
|
1165
|
-
this.sampleQueue.length = 0;
|
|
1166
|
-
}
|
|
1167
|
-
|
|
1168
|
-
this.currentPacketIndex = 0;
|
|
1169
|
-
this.raslSkipped = false;
|
|
1170
|
-
}
|
|
1171
|
-
|
|
1172
|
-
close() {
|
|
1173
|
-
if (this.customDecoder) {
|
|
1174
|
-
void this.customDecoderCallSerializer.call(() => this.customDecoder!.close());
|
|
1175
|
-
} else {
|
|
1176
|
-
assert(this.decoder);
|
|
1177
|
-
this.decoder.close();
|
|
1178
|
-
this.alphaDecoder?.close();
|
|
1179
|
-
|
|
1180
|
-
this.colorQueue.forEach(x => x.close());
|
|
1181
|
-
this.colorQueue.length = 0;
|
|
1182
|
-
this.alphaQueue.forEach(x => x?.close());
|
|
1183
|
-
this.alphaQueue.length = 0;
|
|
1184
|
-
|
|
1185
|
-
this.merger?.close();
|
|
1186
|
-
}
|
|
1187
|
-
|
|
1188
|
-
for (const sample of this.sampleQueue) {
|
|
1189
|
-
sample.close();
|
|
1190
|
-
}
|
|
1191
|
-
this.sampleQueue.length = 0;
|
|
1192
|
-
}
|
|
1193
|
-
}
|
|
1194
|
-
|
|
1195
|
-
/** Utility class that merges together color and alpha information using simple WebGL 2 shaders. */
|
|
1196
|
-
class ColorAlphaMerger {
|
|
1197
|
-
canvas: OffscreenCanvas | HTMLCanvasElement;
|
|
1198
|
-
private gl: WebGL2RenderingContext;
|
|
1199
|
-
private program: WebGLProgram;
|
|
1200
|
-
private vao: WebGLVertexArrayObject;
|
|
1201
|
-
private colorTexture: WebGLTexture;
|
|
1202
|
-
private alphaTexture: WebGLTexture;
|
|
1203
|
-
|
|
1204
|
-
constructor() {
|
|
1205
|
-
// Canvas will be resized later
|
|
1206
|
-
if (typeof OffscreenCanvas !== 'undefined') {
|
|
1207
|
-
// Prefer OffscreenCanvas for Worker environments
|
|
1208
|
-
this.canvas = new OffscreenCanvas(300, 150);
|
|
1209
|
-
} else {
|
|
1210
|
-
this.canvas = document.createElement('canvas');
|
|
1211
|
-
}
|
|
1212
|
-
|
|
1213
|
-
const gl = this.canvas.getContext('webgl2', {
|
|
1214
|
-
premultipliedAlpha: false,
|
|
1215
|
-
}) as unknown as WebGL2RenderingContext | null; // Casting because of some TypeScript weirdness
|
|
1216
|
-
if (!gl) {
|
|
1217
|
-
throw new Error('Couldn\'t acquire WebGL 2 context.');
|
|
1218
|
-
}
|
|
1219
|
-
|
|
1220
|
-
this.gl = gl;
|
|
1221
|
-
this.program = this.createProgram();
|
|
1222
|
-
this.vao = this.createVAO();
|
|
1223
|
-
this.colorTexture = this.createTexture();
|
|
1224
|
-
this.alphaTexture = this.createTexture();
|
|
1225
|
-
|
|
1226
|
-
this.gl.useProgram(this.program);
|
|
1227
|
-
this.gl.uniform1i(this.gl.getUniformLocation(this.program, 'u_colorTexture'), 0);
|
|
1228
|
-
this.gl.uniform1i(this.gl.getUniformLocation(this.program, 'u_alphaTexture'), 1);
|
|
1229
|
-
}
|
|
1230
|
-
|
|
1231
|
-
private createProgram(): WebGLProgram {
|
|
1232
|
-
const vertexShader = this.createShader(this.gl.VERTEX_SHADER, `#version 300 es
|
|
1233
|
-
in vec2 a_position;
|
|
1234
|
-
in vec2 a_texCoord;
|
|
1235
|
-
out vec2 v_texCoord;
|
|
1236
|
-
|
|
1237
|
-
void main() {
|
|
1238
|
-
gl_Position = vec4(a_position, 0.0, 1.0);
|
|
1239
|
-
v_texCoord = a_texCoord;
|
|
1240
|
-
}
|
|
1241
|
-
`);
|
|
1242
|
-
|
|
1243
|
-
const fragmentShader = this.createShader(this.gl.FRAGMENT_SHADER, `#version 300 es
|
|
1244
|
-
precision highp float;
|
|
1245
|
-
|
|
1246
|
-
uniform sampler2D u_colorTexture;
|
|
1247
|
-
uniform sampler2D u_alphaTexture;
|
|
1248
|
-
in vec2 v_texCoord;
|
|
1249
|
-
out vec4 fragColor;
|
|
1250
|
-
|
|
1251
|
-
void main() {
|
|
1252
|
-
vec3 color = texture(u_colorTexture, v_texCoord).rgb;
|
|
1253
|
-
float alpha = texture(u_alphaTexture, v_texCoord).r;
|
|
1254
|
-
fragColor = vec4(color, alpha);
|
|
1255
|
-
}
|
|
1256
|
-
`);
|
|
1257
|
-
|
|
1258
|
-
const program = this.gl.createProgram();
|
|
1259
|
-
this.gl.attachShader(program, vertexShader);
|
|
1260
|
-
this.gl.attachShader(program, fragmentShader);
|
|
1261
|
-
this.gl.linkProgram(program);
|
|
1262
|
-
|
|
1263
|
-
return program;
|
|
1264
|
-
}
|
|
1265
|
-
|
|
1266
|
-
private createShader(type: number, source: string): WebGLShader {
|
|
1267
|
-
const shader = this.gl.createShader(type)!;
|
|
1268
|
-
this.gl.shaderSource(shader, source);
|
|
1269
|
-
this.gl.compileShader(shader);
|
|
1270
|
-
return shader;
|
|
1271
|
-
}
|
|
1272
|
-
|
|
1273
|
-
private createVAO(): WebGLVertexArrayObject {
|
|
1274
|
-
const vao = this.gl.createVertexArray();
|
|
1275
|
-
this.gl.bindVertexArray(vao);
|
|
1276
|
-
|
|
1277
|
-
const vertices = new Float32Array([
|
|
1278
|
-
-1, -1, 0, 1,
|
|
1279
|
-
1, -1, 1, 1,
|
|
1280
|
-
-1, 1, 0, 0,
|
|
1281
|
-
1, 1, 1, 0,
|
|
1282
|
-
]);
|
|
1283
|
-
|
|
1284
|
-
const buffer = this.gl.createBuffer();
|
|
1285
|
-
this.gl.bindBuffer(this.gl.ARRAY_BUFFER, buffer);
|
|
1286
|
-
this.gl.bufferData(this.gl.ARRAY_BUFFER, vertices, this.gl.STATIC_DRAW);
|
|
1287
|
-
|
|
1288
|
-
const positionLocation = this.gl.getAttribLocation(this.program, 'a_position');
|
|
1289
|
-
const texCoordLocation = this.gl.getAttribLocation(this.program, 'a_texCoord');
|
|
1290
|
-
|
|
1291
|
-
this.gl.enableVertexAttribArray(positionLocation);
|
|
1292
|
-
this.gl.vertexAttribPointer(positionLocation, 2, this.gl.FLOAT, false, 16, 0);
|
|
1293
|
-
|
|
1294
|
-
this.gl.enableVertexAttribArray(texCoordLocation);
|
|
1295
|
-
this.gl.vertexAttribPointer(texCoordLocation, 2, this.gl.FLOAT, false, 16, 8);
|
|
1296
|
-
|
|
1297
|
-
return vao;
|
|
1298
|
-
}
|
|
1299
|
-
|
|
1300
|
-
private createTexture(): WebGLTexture {
|
|
1301
|
-
const texture = this.gl.createTexture();
|
|
1302
|
-
|
|
1303
|
-
this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
|
|
1304
|
-
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
|
|
1305
|
-
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
|
|
1306
|
-
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR);
|
|
1307
|
-
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.LINEAR);
|
|
1308
|
-
|
|
1309
|
-
return texture;
|
|
1310
|
-
}
|
|
1311
|
-
|
|
1312
|
-
update(color: VideoFrame, alpha: VideoFrame): void {
|
|
1313
|
-
if (color.displayWidth !== this.canvas.width || color.displayHeight !== this.canvas.height) {
|
|
1314
|
-
this.canvas.width = color.displayWidth;
|
|
1315
|
-
this.canvas.height = color.displayHeight;
|
|
1316
|
-
}
|
|
1317
|
-
|
|
1318
|
-
this.gl.activeTexture(this.gl.TEXTURE0);
|
|
1319
|
-
this.gl.bindTexture(this.gl.TEXTURE_2D, this.colorTexture);
|
|
1320
|
-
this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, color);
|
|
1321
|
-
|
|
1322
|
-
this.gl.activeTexture(this.gl.TEXTURE1);
|
|
1323
|
-
this.gl.bindTexture(this.gl.TEXTURE_2D, this.alphaTexture);
|
|
1324
|
-
this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, alpha);
|
|
1325
|
-
|
|
1326
|
-
this.gl.viewport(0, 0, this.canvas.width, this.canvas.height);
|
|
1327
|
-
this.gl.clear(this.gl.COLOR_BUFFER_BIT);
|
|
1328
|
-
|
|
1329
|
-
this.gl.bindVertexArray(this.vao);
|
|
1330
|
-
this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4);
|
|
1331
|
-
}
|
|
1332
|
-
|
|
1333
|
-
close() {
|
|
1334
|
-
this.gl.getExtension('WEBGL_lose_context')?.loseContext();
|
|
1335
|
-
this.gl = null as unknown as WebGL2RenderingContext;
|
|
1336
|
-
}
|
|
1337
|
-
}
|
|
1338
|
-
|
|
1339
|
-
/**
|
|
1340
|
-
* A sink that retrieves decoded video samples (video frames) from a video track.
|
|
1341
|
-
* @group Media sinks
|
|
1342
|
-
* @public
|
|
1343
|
-
*/
|
|
1344
|
-
export class VideoSampleSink extends BaseMediaSampleSink<VideoSample> {
|
|
1345
|
-
/** @internal */
|
|
1346
|
-
_track: InputVideoTrack;
|
|
1347
|
-
|
|
1348
|
-
/** Creates a new {@link VideoSampleSink} for the given {@link InputVideoTrack}. */
|
|
1349
|
-
constructor(videoTrack: InputVideoTrack) {
|
|
1350
|
-
if (!(videoTrack instanceof InputVideoTrack)) {
|
|
1351
|
-
throw new TypeError('videoTrack must be an InputVideoTrack.');
|
|
1352
|
-
}
|
|
1353
|
-
|
|
1354
|
-
super();
|
|
1355
|
-
|
|
1356
|
-
this._track = videoTrack;
|
|
1357
|
-
}
|
|
1358
|
-
|
|
1359
|
-
/** @internal */
|
|
1360
|
-
async _createDecoder(
|
|
1361
|
-
onSample: (sample: VideoSample) => unknown,
|
|
1362
|
-
onError: (error: Error) => unknown,
|
|
1363
|
-
) {
|
|
1364
|
-
if (!(await this._track.canDecode())) {
|
|
1365
|
-
throw new Error(
|
|
1366
|
-
'This video track cannot be decoded by this browser. Make sure to check decodability before using'
|
|
1367
|
-
+ ' a track.',
|
|
1368
|
-
);
|
|
1369
|
-
}
|
|
1370
|
-
|
|
1371
|
-
const codec = this._track.codec;
|
|
1372
|
-
const rotation = this._track.rotation;
|
|
1373
|
-
const decoderConfig = await this._track.getDecoderConfig();
|
|
1374
|
-
const timeResolution = this._track.timeResolution;
|
|
1375
|
-
assert(codec && decoderConfig);
|
|
1376
|
-
|
|
1377
|
-
return new VideoDecoderWrapper(onSample, onError, codec, decoderConfig, rotation, timeResolution);
|
|
1378
|
-
}
|
|
1379
|
-
|
|
1380
|
-
/** @internal */
|
|
1381
|
-
_createPacketSink() {
|
|
1382
|
-
return new EncodedPacketSink(this._track);
|
|
1383
|
-
}
|
|
1384
|
-
|
|
1385
|
-
/**
|
|
1386
|
-
* Retrieves the video sample (frame) corresponding to the given timestamp, in seconds. More specifically, returns
|
|
1387
|
-
* the last video sample (in presentation order) with a start timestamp less than or equal to the given timestamp.
|
|
1388
|
-
* Returns null if the timestamp is before the track's first timestamp.
|
|
1389
|
-
*
|
|
1390
|
-
* @param timestamp - The timestamp used for retrieval, in seconds.
|
|
1391
|
-
*/
|
|
1392
|
-
async getSample(timestamp: number) {
|
|
1393
|
-
validateTimestamp(timestamp);
|
|
1394
|
-
|
|
1395
|
-
for await (const sample of this.mediaSamplesAtTimestamps([timestamp])) {
|
|
1396
|
-
return sample;
|
|
1397
|
-
}
|
|
1398
|
-
throw new Error('Internal error: Iterator returned nothing.');
|
|
1399
|
-
}
|
|
1400
|
-
|
|
1401
|
-
/**
|
|
1402
|
-
* Creates an async iterator that yields the video samples (frames) of this track in presentation order. This method
|
|
1403
|
-
* will intelligently pre-decode a few frames ahead to enable fast iteration.
|
|
1404
|
-
*
|
|
1405
|
-
* @param startTimestamp - The timestamp in seconds at which to start yielding samples (inclusive).
|
|
1406
|
-
* @param endTimestamp - The timestamp in seconds at which to stop yielding samples (exclusive).
|
|
1407
|
-
*/
|
|
1408
|
-
samples(startTimestamp = 0, endTimestamp = Infinity) {
|
|
1409
|
-
return this.mediaSamplesInRange(startTimestamp, endTimestamp);
|
|
1410
|
-
}
|
|
1411
|
-
|
|
1412
|
-
/**
|
|
1413
|
-
* Creates an async iterator that yields a video sample (frame) for each timestamp in the argument. This method
|
|
1414
|
-
* uses an optimized decoding pipeline if these timestamps are monotonically sorted, decoding each packet at most
|
|
1415
|
-
* once, and is therefore more efficient than manually getting the sample for every timestamp. The iterator may
|
|
1416
|
-
* yield null if no frame is available for a given timestamp.
|
|
1417
|
-
*
|
|
1418
|
-
* @param timestamps - An iterable or async iterable of timestamps in seconds.
|
|
1419
|
-
*/
|
|
1420
|
-
samplesAtTimestamps(timestamps: AnyIterable<number>) {
|
|
1421
|
-
return this.mediaSamplesAtTimestamps(timestamps);
|
|
1422
|
-
}
|
|
1423
|
-
}
|
|
1424
|
-
|
|
1425
|
-
/**
|
|
1426
|
-
* A canvas with additional timing information (timestamp & duration).
|
|
1427
|
-
* @group Media sinks
|
|
1428
|
-
* @public
|
|
1429
|
-
*/
|
|
1430
|
-
export type WrappedCanvas = {
|
|
1431
|
-
/** A canvas element or offscreen canvas. */
|
|
1432
|
-
canvas: HTMLCanvasElement | OffscreenCanvas;
|
|
1433
|
-
/** The timestamp of the corresponding video sample, in seconds. */
|
|
1434
|
-
timestamp: number;
|
|
1435
|
-
/** The duration of the corresponding video sample, in seconds. */
|
|
1436
|
-
duration: number;
|
|
1437
|
-
};
|
|
1438
|
-
|
|
1439
|
-
/**
|
|
1440
|
-
* Options for constructing a CanvasSink.
|
|
1441
|
-
* @group Media sinks
|
|
1442
|
-
* @public
|
|
1443
|
-
*/
|
|
1444
|
-
export type CanvasSinkOptions = {
|
|
1445
|
-
/**
|
|
1446
|
-
* Whether the output canvases should have transparency instead of a black background. Defaults to `false`. Set
|
|
1447
|
-
* this to `true` when using this sink to read transparent videos.
|
|
1448
|
-
*/
|
|
1449
|
-
alpha?: boolean;
|
|
1450
|
-
/**
|
|
1451
|
-
* The width of the output canvas in pixels, defaulting to the display width of the video track. If height is not
|
|
1452
|
-
* set, it will be deduced automatically based on aspect ratio.
|
|
1453
|
-
*/
|
|
1454
|
-
width?: number;
|
|
1455
|
-
/**
|
|
1456
|
-
* The height of the output canvas in pixels, defaulting to the display height of the video track. If width is not
|
|
1457
|
-
* set, it will be deduced automatically based on aspect ratio.
|
|
1458
|
-
*/
|
|
1459
|
-
height?: number;
|
|
1460
|
-
/**
|
|
1461
|
-
* The fitting algorithm in case both width and height are set.
|
|
1462
|
-
*
|
|
1463
|
-
* - `'fill'` will stretch the image to fill the entire box, potentially altering aspect ratio.
|
|
1464
|
-
* - `'contain'` will contain the entire image within the box while preserving aspect ratio. This may lead to
|
|
1465
|
-
* letterboxing.
|
|
1466
|
-
* - `'cover'` will scale the image until the entire box is filled, while preserving aspect ratio.
|
|
1467
|
-
*/
|
|
1468
|
-
fit?: 'fill' | 'contain' | 'cover';
|
|
1469
|
-
/**
|
|
1470
|
-
* The clockwise rotation by which to rotate the raw video frame. Defaults to the rotation set in the file metadata.
|
|
1471
|
-
* Rotation is applied before resizing.
|
|
1472
|
-
*/
|
|
1473
|
-
rotation?: Rotation;
|
|
1474
|
-
/**
|
|
1475
|
-
* Specifies the rectangular region of the input video to crop to. The crop region will automatically be clamped to
|
|
1476
|
-
* the dimensions of the input video track. Cropping is performed after rotation but before resizing.
|
|
1477
|
-
*/
|
|
1478
|
-
crop?: CropRectangle;
|
|
1479
|
-
/**
|
|
1480
|
-
* When set, specifies the number of canvases in the pool. These canvases will be reused in a ring buffer /
|
|
1481
|
-
* round-robin type fashion. This keeps the amount of allocated VRAM constant and relieves the browser from
|
|
1482
|
-
* constantly allocating/deallocating canvases. A pool size of 0 or `undefined` disables the pool and means a new
|
|
1483
|
-
* canvas is created each time.
|
|
1484
|
-
*/
|
|
1485
|
-
poolSize?: number;
|
|
1486
|
-
};
|
|
1487
|
-
|
|
1488
|
-
/**
|
|
1489
|
-
* A sink that renders video samples (frames) of the given video track to canvases. This is often more useful than
|
|
1490
|
-
* directly retrieving frames, as it comes with common preprocessing steps such as resizing or applying rotation
|
|
1491
|
-
* metadata.
|
|
1492
|
-
*
|
|
1493
|
-
* This sink will yield `HTMLCanvasElement`s when in a DOM context, and `OffscreenCanvas`es otherwise.
|
|
1494
|
-
*
|
|
1495
|
-
* @group Media sinks
|
|
1496
|
-
* @public
|
|
1497
|
-
*/
|
|
1498
|
-
export class CanvasSink {
|
|
1499
|
-
/** @internal */
|
|
1500
|
-
_videoTrack: InputVideoTrack;
|
|
1501
|
-
/** @internal */
|
|
1502
|
-
_alpha: boolean;
|
|
1503
|
-
/** @internal */
|
|
1504
|
-
_width: number;
|
|
1505
|
-
/** @internal */
|
|
1506
|
-
_height: number;
|
|
1507
|
-
/** @internal */
|
|
1508
|
-
_fit: 'fill' | 'contain' | 'cover';
|
|
1509
|
-
/** @internal */
|
|
1510
|
-
_rotation: Rotation;
|
|
1511
|
-
/** @internal */
|
|
1512
|
-
_crop?: { left: number; top: number; width: number; height: number };
|
|
1513
|
-
/** @internal */
|
|
1514
|
-
_videoSampleSink: VideoSampleSink;
|
|
1515
|
-
/** @internal */
|
|
1516
|
-
_canvasPool: (HTMLCanvasElement | OffscreenCanvas | null)[];
|
|
1517
|
-
/** @internal */
|
|
1518
|
-
_nextCanvasIndex = 0;
|
|
1519
|
-
|
|
1520
|
-
/** Creates a new {@link CanvasSink} for the given {@link InputVideoTrack}. */
|
|
1521
|
-
constructor(videoTrack: InputVideoTrack, options: CanvasSinkOptions = {}) {
|
|
1522
|
-
if (!(videoTrack instanceof InputVideoTrack)) {
|
|
1523
|
-
throw new TypeError('videoTrack must be an InputVideoTrack.');
|
|
1524
|
-
}
|
|
1525
|
-
if (options && typeof options !== 'object') {
|
|
1526
|
-
throw new TypeError('options must be an object.');
|
|
1527
|
-
}
|
|
1528
|
-
if (options.alpha !== undefined && typeof options.alpha !== 'boolean') {
|
|
1529
|
-
throw new TypeError('options.alpha, when provided, must be a boolean.');
|
|
1530
|
-
}
|
|
1531
|
-
if (options.width !== undefined && (!Number.isInteger(options.width) || options.width <= 0)) {
|
|
1532
|
-
throw new TypeError('options.width, when defined, must be a positive integer.');
|
|
1533
|
-
}
|
|
1534
|
-
if (options.height !== undefined && (!Number.isInteger(options.height) || options.height <= 0)) {
|
|
1535
|
-
throw new TypeError('options.height, when defined, must be a positive integer.');
|
|
1536
|
-
}
|
|
1537
|
-
if (options.fit !== undefined && !['fill', 'contain', 'cover'].includes(options.fit)) {
|
|
1538
|
-
throw new TypeError('options.fit, when provided, must be one of "fill", "contain", or "cover".');
|
|
1539
|
-
}
|
|
1540
|
-
if (
|
|
1541
|
-
options.width !== undefined
|
|
1542
|
-
&& options.height !== undefined
|
|
1543
|
-
&& options.fit === undefined
|
|
1544
|
-
) {
|
|
1545
|
-
throw new TypeError(
|
|
1546
|
-
'When both options.width and options.height are provided, options.fit must also be provided.',
|
|
1547
|
-
);
|
|
1548
|
-
}
|
|
1549
|
-
if (options.rotation !== undefined && ![0, 90, 180, 270].includes(options.rotation)) {
|
|
1550
|
-
throw new TypeError('options.rotation, when provided, must be 0, 90, 180 or 270.');
|
|
1551
|
-
}
|
|
1552
|
-
if (options.crop !== undefined) {
|
|
1553
|
-
validateCropRectangle(options.crop, 'options.');
|
|
1554
|
-
}
|
|
1555
|
-
if (
|
|
1556
|
-
options.poolSize !== undefined
|
|
1557
|
-
&& (typeof options.poolSize !== 'number' || !Number.isInteger(options.poolSize) || options.poolSize < 0)
|
|
1558
|
-
) {
|
|
1559
|
-
throw new TypeError('poolSize must be a non-negative integer.');
|
|
1560
|
-
}
|
|
1561
|
-
|
|
1562
|
-
const rotation = options.rotation ?? videoTrack.rotation;
|
|
1563
|
-
|
|
1564
|
-
const [rotatedWidth, rotatedHeight] = rotation % 180 === 0
|
|
1565
|
-
? [videoTrack.codedWidth, videoTrack.codedHeight]
|
|
1566
|
-
: [videoTrack.codedHeight, videoTrack.codedWidth];
|
|
1567
|
-
|
|
1568
|
-
const crop = options.crop;
|
|
1569
|
-
if (crop) {
|
|
1570
|
-
clampCropRectangle(crop, rotatedWidth, rotatedHeight);
|
|
1571
|
-
}
|
|
1572
|
-
|
|
1573
|
-
let [width, height] = crop
|
|
1574
|
-
? [crop.width, crop.height]
|
|
1575
|
-
: [rotatedWidth, rotatedHeight];
|
|
1576
|
-
const originalAspectRatio = width / height;
|
|
1577
|
-
|
|
1578
|
-
// If width and height aren't defined together, deduce the missing value using the aspect ratio
|
|
1579
|
-
if (options.width !== undefined && options.height === undefined) {
|
|
1580
|
-
width = options.width;
|
|
1581
|
-
height = Math.round(width / originalAspectRatio);
|
|
1582
|
-
} else if (options.width === undefined && options.height !== undefined) {
|
|
1583
|
-
height = options.height;
|
|
1584
|
-
width = Math.round(height * originalAspectRatio);
|
|
1585
|
-
} else if (options.width !== undefined && options.height !== undefined) {
|
|
1586
|
-
width = options.width;
|
|
1587
|
-
height = options.height;
|
|
1588
|
-
}
|
|
1589
|
-
|
|
1590
|
-
this._videoTrack = videoTrack;
|
|
1591
|
-
this._alpha = options.alpha ?? false;
|
|
1592
|
-
this._width = width;
|
|
1593
|
-
this._height = height;
|
|
1594
|
-
this._rotation = rotation;
|
|
1595
|
-
this._crop = crop;
|
|
1596
|
-
this._fit = options.fit ?? 'fill';
|
|
1597
|
-
this._videoSampleSink = new VideoSampleSink(videoTrack);
|
|
1598
|
-
this._canvasPool = Array.from({ length: options.poolSize ?? 0 }, () => null);
|
|
1599
|
-
}
|
|
1600
|
-
|
|
1601
|
-
/** @internal */
|
|
1602
|
-
_videoSampleToWrappedCanvas(sample: VideoSample): WrappedCanvas {
|
|
1603
|
-
let canvas = this._canvasPool[this._nextCanvasIndex];
|
|
1604
|
-
let canvasIsNew = false;
|
|
1605
|
-
|
|
1606
|
-
if (!canvas) {
|
|
1607
|
-
if (typeof document !== 'undefined') {
|
|
1608
|
-
// Prefer an HTMLCanvasElement
|
|
1609
|
-
canvas = document.createElement('canvas');
|
|
1610
|
-
canvas.width = this._width;
|
|
1611
|
-
canvas.height = this._height;
|
|
1612
|
-
} else {
|
|
1613
|
-
canvas = new OffscreenCanvas(this._width, this._height);
|
|
1614
|
-
}
|
|
1615
|
-
|
|
1616
|
-
if (this._canvasPool.length > 0) {
|
|
1617
|
-
this._canvasPool[this._nextCanvasIndex] = canvas;
|
|
1618
|
-
}
|
|
1619
|
-
|
|
1620
|
-
canvasIsNew = true;
|
|
1621
|
-
}
|
|
1622
|
-
|
|
1623
|
-
if (this._canvasPool.length > 0) {
|
|
1624
|
-
this._nextCanvasIndex = (this._nextCanvasIndex + 1) % this._canvasPool.length;
|
|
1625
|
-
}
|
|
1626
|
-
|
|
1627
|
-
const context = canvas.getContext('2d', {
|
|
1628
|
-
alpha: this._alpha || isFirefox(), // Firefox has VideoFrame glitches with opaque canvases
|
|
1629
|
-
}) as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
|
|
1630
|
-
assert(context);
|
|
1631
|
-
|
|
1632
|
-
context.resetTransform();
|
|
1633
|
-
|
|
1634
|
-
if (!canvasIsNew) {
|
|
1635
|
-
if (!this._alpha && isFirefox()) {
|
|
1636
|
-
context.fillStyle = 'black';
|
|
1637
|
-
context.fillRect(0, 0, this._width, this._height);
|
|
1638
|
-
} else {
|
|
1639
|
-
context.clearRect(0, 0, this._width, this._height);
|
|
1640
|
-
}
|
|
1641
|
-
}
|
|
1642
|
-
|
|
1643
|
-
sample.drawWithFit(context, {
|
|
1644
|
-
fit: this._fit,
|
|
1645
|
-
rotation: this._rotation,
|
|
1646
|
-
crop: this._crop,
|
|
1647
|
-
});
|
|
1648
|
-
|
|
1649
|
-
const result = {
|
|
1650
|
-
canvas,
|
|
1651
|
-
timestamp: sample.timestamp,
|
|
1652
|
-
duration: sample.duration,
|
|
1653
|
-
};
|
|
1654
|
-
|
|
1655
|
-
sample.close();
|
|
1656
|
-
return result;
|
|
1657
|
-
}
|
|
1658
|
-
|
|
1659
|
-
/**
|
|
1660
|
-
* Retrieves a canvas with the video frame corresponding to the given timestamp, in seconds. More specifically,
|
|
1661
|
-
* returns the last video frame (in presentation order) with a start timestamp less than or equal to the given
|
|
1662
|
-
* timestamp. Returns null if the timestamp is before the track's first timestamp.
|
|
1663
|
-
*
|
|
1664
|
-
* @param timestamp - The timestamp used for retrieval, in seconds.
|
|
1665
|
-
*/
|
|
1666
|
-
async getCanvas(timestamp: number) {
|
|
1667
|
-
validateTimestamp(timestamp);
|
|
1668
|
-
|
|
1669
|
-
const sample = await this._videoSampleSink.getSample(timestamp);
|
|
1670
|
-
return sample && this._videoSampleToWrappedCanvas(sample);
|
|
1671
|
-
}
|
|
1672
|
-
|
|
1673
|
-
/**
|
|
1674
|
-
* Creates an async iterator that yields canvases with the video frames of this track in presentation order. This
|
|
1675
|
-
* method will intelligently pre-decode a few frames ahead to enable fast iteration.
|
|
1676
|
-
*
|
|
1677
|
-
* @param startTimestamp - The timestamp in seconds at which to start yielding canvases (inclusive).
|
|
1678
|
-
* @param endTimestamp - The timestamp in seconds at which to stop yielding canvases (exclusive).
|
|
1679
|
-
*/
|
|
1680
|
-
canvases(startTimestamp = 0, endTimestamp = Infinity) {
|
|
1681
|
-
return mapAsyncGenerator(
|
|
1682
|
-
this._videoSampleSink.samples(startTimestamp, endTimestamp),
|
|
1683
|
-
sample => this._videoSampleToWrappedCanvas(sample),
|
|
1684
|
-
);
|
|
1685
|
-
}
|
|
1686
|
-
|
|
1687
|
-
/**
|
|
1688
|
-
* Creates an async iterator that yields a canvas for each timestamp in the argument. This method uses an optimized
|
|
1689
|
-
* decoding pipeline if these timestamps are monotonically sorted, decoding each packet at most once, and is
|
|
1690
|
-
* therefore more efficient than manually getting the canvas for every timestamp. The iterator may yield null if
|
|
1691
|
-
* no frame is available for a given timestamp.
|
|
1692
|
-
*
|
|
1693
|
-
* @param timestamps - An iterable or async iterable of timestamps in seconds.
|
|
1694
|
-
*/
|
|
1695
|
-
canvasesAtTimestamps(timestamps: AnyIterable<number>) {
|
|
1696
|
-
return mapAsyncGenerator(
|
|
1697
|
-
this._videoSampleSink.samplesAtTimestamps(timestamps),
|
|
1698
|
-
sample => sample && this._videoSampleToWrappedCanvas(sample),
|
|
1699
|
-
);
|
|
1700
|
-
}
|
|
1701
|
-
}
|
|
1702
|
-
|
|
1703
|
-
class AudioDecoderWrapper extends DecoderWrapper<AudioSample> {
|
|
1704
|
-
decoder: AudioDecoder | null = null;
|
|
1705
|
-
|
|
1706
|
-
customDecoder: CustomAudioDecoder | null = null;
|
|
1707
|
-
customDecoderCallSerializer = new CallSerializer();
|
|
1708
|
-
customDecoderQueueSize = 0;
|
|
1709
|
-
|
|
1710
|
-
// Internal state to accumulate a precise current timestamp based on audio durations, not the (potentially
|
|
1711
|
-
// inaccurate) packet timestamps.
|
|
1712
|
-
currentTimestamp: number | null = null;
|
|
1713
|
-
|
|
1714
|
-
constructor(
|
|
1715
|
-
onSample: (sample: AudioSample) => unknown,
|
|
1716
|
-
onError: (error: Error) => unknown,
|
|
1717
|
-
codec: AudioCodec,
|
|
1718
|
-
decoderConfig: AudioDecoderConfig,
|
|
1719
|
-
) {
|
|
1720
|
-
super(onSample, onError);
|
|
1721
|
-
|
|
1722
|
-
const sampleHandler = (sample: AudioSample) => {
|
|
1723
|
-
if (
|
|
1724
|
-
this.currentTimestamp === null
|
|
1725
|
-
|| Math.abs(sample.timestamp - this.currentTimestamp) >= sample.duration
|
|
1726
|
-
) {
|
|
1727
|
-
// We need to sync with the sample timestamp again
|
|
1728
|
-
this.currentTimestamp = sample.timestamp;
|
|
1729
|
-
}
|
|
1730
|
-
|
|
1731
|
-
const preciseTimestamp = this.currentTimestamp;
|
|
1732
|
-
this.currentTimestamp += sample.duration;
|
|
1733
|
-
|
|
1734
|
-
if (sample.numberOfFrames === 0) {
|
|
1735
|
-
// We skip zero-data (empty) AudioSamples. These are sometimes emitted, for example, by Firefox when it
|
|
1736
|
-
// decodes Vorbis (at the start).
|
|
1737
|
-
sample.close();
|
|
1738
|
-
return;
|
|
1739
|
-
}
|
|
1740
|
-
|
|
1741
|
-
// Round the timestamp to the sample rate
|
|
1742
|
-
const sampleRate = decoderConfig.sampleRate;
|
|
1743
|
-
sample.setTimestamp(Math.round(preciseTimestamp * sampleRate) / sampleRate);
|
|
1744
|
-
|
|
1745
|
-
onSample(sample);
|
|
1746
|
-
};
|
|
1747
|
-
|
|
1748
|
-
const MatchingCustomDecoder = customAudioDecoders.find(x => x.supports(codec, decoderConfig));
|
|
1749
|
-
if (MatchingCustomDecoder) {
|
|
1750
|
-
// @ts-expect-error "Can't create instance of abstract class 🤓"
|
|
1751
|
-
this.customDecoder = new MatchingCustomDecoder() as CustomAudioDecoder;
|
|
1752
|
-
// @ts-expect-error It's technically readonly
|
|
1753
|
-
this.customDecoder.codec = codec;
|
|
1754
|
-
// @ts-expect-error It's technically readonly
|
|
1755
|
-
this.customDecoder.config = decoderConfig;
|
|
1756
|
-
// @ts-expect-error It's technically readonly
|
|
1757
|
-
this.customDecoder.onSample = (sample) => {
|
|
1758
|
-
if (!(sample instanceof AudioSample)) {
|
|
1759
|
-
throw new TypeError('The argument passed to onSample must be an AudioSample.');
|
|
1760
|
-
}
|
|
1761
|
-
|
|
1762
|
-
sampleHandler(sample);
|
|
1763
|
-
};
|
|
1764
|
-
|
|
1765
|
-
void this.customDecoderCallSerializer.call(() => this.customDecoder!.init());
|
|
1766
|
-
} else {
|
|
1767
|
-
this.decoder = new AudioDecoder({
|
|
1768
|
-
output: (data) => {
|
|
1769
|
-
try {
|
|
1770
|
-
sampleHandler(new AudioSample(data));
|
|
1771
|
-
} catch (error) {
|
|
1772
|
-
this.onError(error as Error);
|
|
1773
|
-
}
|
|
1774
|
-
},
|
|
1775
|
-
error: onError,
|
|
1776
|
-
});
|
|
1777
|
-
this.decoder.configure(decoderConfig);
|
|
1778
|
-
}
|
|
1779
|
-
}
|
|
1780
|
-
|
|
1781
|
-
getDecodeQueueSize() {
|
|
1782
|
-
if (this.customDecoder) {
|
|
1783
|
-
return this.customDecoderQueueSize;
|
|
1784
|
-
} else {
|
|
1785
|
-
assert(this.decoder);
|
|
1786
|
-
return this.decoder.decodeQueueSize;
|
|
1787
|
-
}
|
|
1788
|
-
}
|
|
1789
|
-
|
|
1790
|
-
decode(packet: EncodedPacket) {
|
|
1791
|
-
if (this.customDecoder) {
|
|
1792
|
-
this.customDecoderQueueSize++;
|
|
1793
|
-
void this.customDecoderCallSerializer
|
|
1794
|
-
.call(() => this.customDecoder!.decode(packet))
|
|
1795
|
-
.then(() => this.customDecoderQueueSize--);
|
|
1796
|
-
} else {
|
|
1797
|
-
assert(this.decoder);
|
|
1798
|
-
this.decoder.decode(packet.toEncodedAudioChunk());
|
|
1799
|
-
}
|
|
1800
|
-
}
|
|
1801
|
-
|
|
1802
|
-
flush() {
|
|
1803
|
-
if (this.customDecoder) {
|
|
1804
|
-
return this.customDecoderCallSerializer.call(() => this.customDecoder!.flush());
|
|
1805
|
-
} else {
|
|
1806
|
-
assert(this.decoder);
|
|
1807
|
-
return this.decoder.flush();
|
|
1808
|
-
}
|
|
1809
|
-
}
|
|
1810
|
-
|
|
1811
|
-
close() {
|
|
1812
|
-
if (this.customDecoder) {
|
|
1813
|
-
void this.customDecoderCallSerializer.call(() => this.customDecoder!.close());
|
|
1814
|
-
} else {
|
|
1815
|
-
assert(this.decoder);
|
|
1816
|
-
this.decoder.close();
|
|
1817
|
-
}
|
|
1818
|
-
}
|
|
1819
|
-
}
|
|
1820
|
-
|
|
1821
|
-
// There are a lot of PCM variants not natively supported by the browser and by AudioData. Therefore we need a simple
|
|
1822
|
-
// decoder that maps any input PCM format into a PCM format supported by the browser.
|
|
1823
|
-
class PcmAudioDecoderWrapper extends DecoderWrapper<AudioSample> {
|
|
1824
|
-
codec: PcmAudioCodec;
|
|
1825
|
-
|
|
1826
|
-
inputSampleSize: 1 | 2 | 3 | 4 | 8;
|
|
1827
|
-
readInputValue: (view: DataView, byteOffset: number) => number;
|
|
1828
|
-
|
|
1829
|
-
outputSampleSize: 1 | 2 | 4;
|
|
1830
|
-
outputFormat: 'u8' | 's16' | 's32' | 'f32';
|
|
1831
|
-
writeOutputValue: (view: DataView, byteOffset: number, value: number) => void;
|
|
1832
|
-
|
|
1833
|
-
// Internal state to accumulate a precise current timestamp based on audio durations, not the (potentially
|
|
1834
|
-
// inaccurate) packet timestamps.
|
|
1835
|
-
currentTimestamp: number | null = null;
|
|
1836
|
-
|
|
1837
|
-
constructor(
|
|
1838
|
-
onSample: (sample: AudioSample) => unknown,
|
|
1839
|
-
onError: (error: Error) => unknown,
|
|
1840
|
-
public decoderConfig: AudioDecoderConfig,
|
|
1841
|
-
) {
|
|
1842
|
-
super(onSample, onError);
|
|
1843
|
-
|
|
1844
|
-
assert((PCM_AUDIO_CODECS as readonly string[]).includes(decoderConfig.codec));
|
|
1845
|
-
this.codec = decoderConfig.codec as PcmAudioCodec;
|
|
1846
|
-
|
|
1847
|
-
const { dataType, sampleSize, littleEndian } = parsePcmCodec(this.codec);
|
|
1848
|
-
this.inputSampleSize = sampleSize;
|
|
1849
|
-
|
|
1850
|
-
switch (sampleSize) {
|
|
1851
|
-
case 1: {
|
|
1852
|
-
if (dataType === 'unsigned') {
|
|
1853
|
-
this.readInputValue = (view, byteOffset) => view.getUint8(byteOffset) - 2 ** 7;
|
|
1854
|
-
} else if (dataType === 'signed') {
|
|
1855
|
-
this.readInputValue = (view, byteOffset) => view.getInt8(byteOffset);
|
|
1856
|
-
} else if (dataType === 'ulaw') {
|
|
1857
|
-
this.readInputValue = (view, byteOffset) => fromUlaw(view.getUint8(byteOffset));
|
|
1858
|
-
} else if (dataType === 'alaw') {
|
|
1859
|
-
this.readInputValue = (view, byteOffset) => fromAlaw(view.getUint8(byteOffset));
|
|
1860
|
-
} else {
|
|
1861
|
-
assert(false);
|
|
1862
|
-
}
|
|
1863
|
-
}; break;
|
|
1864
|
-
case 2: {
|
|
1865
|
-
if (dataType === 'unsigned') {
|
|
1866
|
-
this.readInputValue = (view, byteOffset) => view.getUint16(byteOffset, littleEndian) - 2 ** 15;
|
|
1867
|
-
} else if (dataType === 'signed') {
|
|
1868
|
-
this.readInputValue = (view, byteOffset) => view.getInt16(byteOffset, littleEndian);
|
|
1869
|
-
} else {
|
|
1870
|
-
assert(false);
|
|
1871
|
-
}
|
|
1872
|
-
}; break;
|
|
1873
|
-
case 3: {
|
|
1874
|
-
if (dataType === 'unsigned') {
|
|
1875
|
-
this.readInputValue = (view, byteOffset) => getUint24(view, byteOffset, littleEndian) - 2 ** 23;
|
|
1876
|
-
} else if (dataType === 'signed') {
|
|
1877
|
-
this.readInputValue = (view, byteOffset) => getInt24(view, byteOffset, littleEndian);
|
|
1878
|
-
} else {
|
|
1879
|
-
assert(false);
|
|
1880
|
-
}
|
|
1881
|
-
}; break;
|
|
1882
|
-
case 4: {
|
|
1883
|
-
if (dataType === 'unsigned') {
|
|
1884
|
-
this.readInputValue = (view, byteOffset) => view.getUint32(byteOffset, littleEndian) - 2 ** 31;
|
|
1885
|
-
} else if (dataType === 'signed') {
|
|
1886
|
-
this.readInputValue = (view, byteOffset) => view.getInt32(byteOffset, littleEndian);
|
|
1887
|
-
} else if (dataType === 'float') {
|
|
1888
|
-
this.readInputValue = (view, byteOffset) => view.getFloat32(byteOffset, littleEndian);
|
|
1889
|
-
} else {
|
|
1890
|
-
assert(false);
|
|
1891
|
-
}
|
|
1892
|
-
}; break;
|
|
1893
|
-
case 8: {
|
|
1894
|
-
if (dataType === 'float') {
|
|
1895
|
-
this.readInputValue = (view, byteOffset) => view.getFloat64(byteOffset, littleEndian);
|
|
1896
|
-
} else {
|
|
1897
|
-
assert(false);
|
|
1898
|
-
}
|
|
1899
|
-
}; break;
|
|
1900
|
-
default: {
|
|
1901
|
-
assertNever(sampleSize);
|
|
1902
|
-
assert(false);
|
|
1903
|
-
};
|
|
1904
|
-
}
|
|
1905
|
-
|
|
1906
|
-
switch (sampleSize) {
|
|
1907
|
-
case 1: {
|
|
1908
|
-
if (dataType === 'ulaw' || dataType === 'alaw') {
|
|
1909
|
-
this.outputSampleSize = 2;
|
|
1910
|
-
this.outputFormat = 's16';
|
|
1911
|
-
this.writeOutputValue = (view, byteOffset, value) => view.setInt16(byteOffset, value, true);
|
|
1912
|
-
} else {
|
|
1913
|
-
this.outputSampleSize = 1;
|
|
1914
|
-
this.outputFormat = 'u8';
|
|
1915
|
-
this.writeOutputValue = (view, byteOffset, value) => view.setUint8(byteOffset, value + 2 ** 7);
|
|
1916
|
-
}
|
|
1917
|
-
}; break;
|
|
1918
|
-
case 2: {
|
|
1919
|
-
this.outputSampleSize = 2;
|
|
1920
|
-
this.outputFormat = 's16';
|
|
1921
|
-
this.writeOutputValue = (view, byteOffset, value) => view.setInt16(byteOffset, value, true);
|
|
1922
|
-
}; break;
|
|
1923
|
-
case 3: {
|
|
1924
|
-
this.outputSampleSize = 4;
|
|
1925
|
-
this.outputFormat = 's32';
|
|
1926
|
-
// From https://www.w3.org/TR/webcodecs:
|
|
1927
|
-
// AudioData containing 24-bit samples SHOULD store those samples in s32 or f32. When samples are
|
|
1928
|
-
// stored in s32, each sample MUST be left-shifted by 8 bits.
|
|
1929
|
-
this.writeOutputValue = (view, byteOffset, value) => view.setInt32(byteOffset, value << 8, true);
|
|
1930
|
-
}; break;
|
|
1931
|
-
case 4: {
|
|
1932
|
-
this.outputSampleSize = 4;
|
|
1933
|
-
|
|
1934
|
-
if (dataType === 'float') {
|
|
1935
|
-
this.outputFormat = 'f32';
|
|
1936
|
-
this.writeOutputValue = (view, byteOffset, value) => view.setFloat32(byteOffset, value, true);
|
|
1937
|
-
} else {
|
|
1938
|
-
this.outputFormat = 's32';
|
|
1939
|
-
this.writeOutputValue = (view, byteOffset, value) => view.setInt32(byteOffset, value, true);
|
|
1940
|
-
}
|
|
1941
|
-
}; break;
|
|
1942
|
-
case 8: {
|
|
1943
|
-
this.outputSampleSize = 4;
|
|
1944
|
-
|
|
1945
|
-
this.outputFormat = 'f32';
|
|
1946
|
-
this.writeOutputValue = (view, byteOffset, value) => view.setFloat32(byteOffset, value, true);
|
|
1947
|
-
}; break;
|
|
1948
|
-
default: {
|
|
1949
|
-
assertNever(sampleSize);
|
|
1950
|
-
assert(false);
|
|
1951
|
-
};
|
|
1952
|
-
};
|
|
1953
|
-
}
|
|
1954
|
-
|
|
1955
|
-
getDecodeQueueSize() {
|
|
1956
|
-
return 0;
|
|
1957
|
-
}
|
|
1958
|
-
|
|
1959
|
-
decode(packet: EncodedPacket) {
|
|
1960
|
-
const inputView = toDataView(packet.data);
|
|
1961
|
-
|
|
1962
|
-
const numberOfFrames = packet.byteLength / this.decoderConfig.numberOfChannels / this.inputSampleSize;
|
|
1963
|
-
|
|
1964
|
-
const outputBufferSize = numberOfFrames * this.decoderConfig.numberOfChannels * this.outputSampleSize;
|
|
1965
|
-
const outputBuffer = new ArrayBuffer(outputBufferSize);
|
|
1966
|
-
const outputView = new DataView(outputBuffer);
|
|
1967
|
-
|
|
1968
|
-
for (let i = 0; i < numberOfFrames * this.decoderConfig.numberOfChannels; i++) {
|
|
1969
|
-
const inputIndex = i * this.inputSampleSize;
|
|
1970
|
-
const outputIndex = i * this.outputSampleSize;
|
|
1971
|
-
|
|
1972
|
-
const value = this.readInputValue(inputView, inputIndex);
|
|
1973
|
-
this.writeOutputValue(outputView, outputIndex, value);
|
|
1974
|
-
}
|
|
1975
|
-
|
|
1976
|
-
const preciseDuration = numberOfFrames / this.decoderConfig.sampleRate;
|
|
1977
|
-
if (this.currentTimestamp === null || Math.abs(packet.timestamp - this.currentTimestamp) >= preciseDuration) {
|
|
1978
|
-
// We need to sync with the packet timestamp again
|
|
1979
|
-
this.currentTimestamp = packet.timestamp;
|
|
1980
|
-
}
|
|
1981
|
-
|
|
1982
|
-
const preciseTimestamp = this.currentTimestamp;
|
|
1983
|
-
this.currentTimestamp += preciseDuration;
|
|
1984
|
-
|
|
1985
|
-
const audioSample = new AudioSample({
|
|
1986
|
-
format: this.outputFormat,
|
|
1987
|
-
data: outputBuffer,
|
|
1988
|
-
numberOfChannels: this.decoderConfig.numberOfChannels,
|
|
1989
|
-
sampleRate: this.decoderConfig.sampleRate,
|
|
1990
|
-
numberOfFrames,
|
|
1991
|
-
timestamp: preciseTimestamp,
|
|
1992
|
-
});
|
|
1993
|
-
|
|
1994
|
-
this.onSample(audioSample);
|
|
1995
|
-
}
|
|
1996
|
-
|
|
1997
|
-
async flush() {
|
|
1998
|
-
// Do nothing
|
|
1999
|
-
}
|
|
2000
|
-
|
|
2001
|
-
close() {
|
|
2002
|
-
// Do nothing
|
|
2003
|
-
}
|
|
2004
|
-
}
|
|
2005
|
-
|
|
2006
|
-
/**
|
|
2007
|
-
* Sink for retrieving decoded audio samples from an audio track.
|
|
2008
|
-
* @group Media sinks
|
|
2009
|
-
* @public
|
|
2010
|
-
*/
|
|
2011
|
-
export class AudioSampleSink extends BaseMediaSampleSink<AudioSample> {
|
|
2012
|
-
/** @internal */
|
|
2013
|
-
_track: InputAudioTrack;
|
|
2014
|
-
|
|
2015
|
-
/** Creates a new {@link AudioSampleSink} for the given {@link InputAudioTrack}. */
|
|
2016
|
-
constructor(audioTrack: InputAudioTrack) {
|
|
2017
|
-
if (!(audioTrack instanceof InputAudioTrack)) {
|
|
2018
|
-
throw new TypeError('audioTrack must be an InputAudioTrack.');
|
|
2019
|
-
}
|
|
2020
|
-
|
|
2021
|
-
super();
|
|
2022
|
-
|
|
2023
|
-
this._track = audioTrack;
|
|
2024
|
-
}
|
|
2025
|
-
|
|
2026
|
-
/** @internal */
|
|
2027
|
-
async _createDecoder(
|
|
2028
|
-
onSample: (sample: AudioSample) => unknown,
|
|
2029
|
-
onError: (error: Error) => unknown,
|
|
2030
|
-
) {
|
|
2031
|
-
if (!(await this._track.canDecode())) {
|
|
2032
|
-
throw new Error(
|
|
2033
|
-
'This audio track cannot be decoded by this browser. Make sure to check decodability before using'
|
|
2034
|
-
+ ' a track.',
|
|
2035
|
-
);
|
|
2036
|
-
}
|
|
2037
|
-
|
|
2038
|
-
const codec = this._track.codec;
|
|
2039
|
-
const decoderConfig = await this._track.getDecoderConfig();
|
|
2040
|
-
assert(codec && decoderConfig);
|
|
2041
|
-
|
|
2042
|
-
if ((PCM_AUDIO_CODECS as readonly string[]).includes(decoderConfig.codec)) {
|
|
2043
|
-
return new PcmAudioDecoderWrapper(onSample, onError, decoderConfig);
|
|
2044
|
-
} else {
|
|
2045
|
-
return new AudioDecoderWrapper(onSample, onError, codec, decoderConfig);
|
|
2046
|
-
}
|
|
2047
|
-
}
|
|
2048
|
-
|
|
2049
|
-
/** @internal */
|
|
2050
|
-
_createPacketSink() {
|
|
2051
|
-
return new EncodedPacketSink(this._track);
|
|
2052
|
-
}
|
|
2053
|
-
|
|
2054
|
-
/**
|
|
2055
|
-
* Retrieves the audio sample corresponding to the given timestamp, in seconds. More specifically, returns
|
|
2056
|
-
* the last audio sample (in presentation order) with a start timestamp less than or equal to the given timestamp.
|
|
2057
|
-
* Returns null if the timestamp is before the track's first timestamp.
|
|
2058
|
-
*
|
|
2059
|
-
* @param timestamp - The timestamp used for retrieval, in seconds.
|
|
2060
|
-
*/
|
|
2061
|
-
async getSample(timestamp: number) {
|
|
2062
|
-
validateTimestamp(timestamp);
|
|
2063
|
-
|
|
2064
|
-
for await (const sample of this.mediaSamplesAtTimestamps([timestamp])) {
|
|
2065
|
-
return sample;
|
|
2066
|
-
}
|
|
2067
|
-
throw new Error('Internal error: Iterator returned nothing.');
|
|
2068
|
-
}
|
|
2069
|
-
|
|
2070
|
-
/**
|
|
2071
|
-
* Creates an async iterator that yields the audio samples of this track in presentation order. This method
|
|
2072
|
-
* will intelligently pre-decode a few samples ahead to enable fast iteration.
|
|
2073
|
-
*
|
|
2074
|
-
* @param startTimestamp - The timestamp in seconds at which to start yielding samples (inclusive).
|
|
2075
|
-
* @param endTimestamp - The timestamp in seconds at which to stop yielding samples (exclusive).
|
|
2076
|
-
*/
|
|
2077
|
-
samples(startTimestamp = 0, endTimestamp = Infinity) {
|
|
2078
|
-
return this.mediaSamplesInRange(startTimestamp, endTimestamp);
|
|
2079
|
-
}
|
|
2080
|
-
|
|
2081
|
-
/**
|
|
2082
|
-
* Creates an async iterator that yields an audio sample for each timestamp in the argument. This method
|
|
2083
|
-
* uses an optimized decoding pipeline if these timestamps are monotonically sorted, decoding each packet at most
|
|
2084
|
-
* once, and is therefore more efficient than manually getting the sample for every timestamp. The iterator may
|
|
2085
|
-
* yield null if no sample is available for a given timestamp.
|
|
2086
|
-
*
|
|
2087
|
-
* @param timestamps - An iterable or async iterable of timestamps in seconds.
|
|
2088
|
-
*/
|
|
2089
|
-
samplesAtTimestamps(timestamps: AnyIterable<number>) {
|
|
2090
|
-
return this.mediaSamplesAtTimestamps(timestamps);
|
|
2091
|
-
}
|
|
2092
|
-
}
|
|
2093
|
-
|
|
2094
|
-
/**
|
|
2095
|
-
* An AudioBuffer with additional timing information (timestamp & duration).
|
|
2096
|
-
* @group Media sinks
|
|
2097
|
-
* @public
|
|
2098
|
-
*/
|
|
2099
|
-
export type WrappedAudioBuffer = {
|
|
2100
|
-
/** An AudioBuffer. */
|
|
2101
|
-
buffer: AudioBuffer;
|
|
2102
|
-
/** The timestamp of the corresponding audio sample, in seconds. */
|
|
2103
|
-
timestamp: number;
|
|
2104
|
-
/** The duration of the corresponding audio sample, in seconds. */
|
|
2105
|
-
duration: number;
|
|
2106
|
-
};
|
|
2107
|
-
|
|
2108
|
-
/**
|
|
2109
|
-
* A sink that retrieves decoded audio samples from an audio track and converts them to `AudioBuffer` instances. This is
|
|
2110
|
-
* often more useful than directly retrieving audio samples, as audio buffers can be directly used with the
|
|
2111
|
-
* Web Audio API.
|
|
2112
|
-
* @group Media sinks
|
|
2113
|
-
* @public
|
|
2114
|
-
*/
|
|
2115
|
-
export class AudioBufferSink {
|
|
2116
|
-
/** @internal */
|
|
2117
|
-
_audioSampleSink: AudioSampleSink;
|
|
2118
|
-
|
|
2119
|
-
/** Creates a new {@link AudioBufferSink} for the given {@link InputAudioTrack}. */
|
|
2120
|
-
constructor(audioTrack: InputAudioTrack) {
|
|
2121
|
-
if (!(audioTrack instanceof InputAudioTrack)) {
|
|
2122
|
-
throw new TypeError('audioTrack must be an InputAudioTrack.');
|
|
2123
|
-
}
|
|
2124
|
-
|
|
2125
|
-
this._audioSampleSink = new AudioSampleSink(audioTrack);
|
|
2126
|
-
}
|
|
2127
|
-
|
|
2128
|
-
/** @internal */
|
|
2129
|
-
_audioSampleToWrappedArrayBuffer(sample: AudioSample): WrappedAudioBuffer {
|
|
2130
|
-
return {
|
|
2131
|
-
buffer: sample.toAudioBuffer(),
|
|
2132
|
-
timestamp: sample.timestamp,
|
|
2133
|
-
duration: sample.duration,
|
|
2134
|
-
};
|
|
2135
|
-
}
|
|
2136
|
-
|
|
2137
|
-
/**
|
|
2138
|
-
* Retrieves the audio buffer corresponding to the given timestamp, in seconds. More specifically, returns
|
|
2139
|
-
* the last audio buffer (in presentation order) with a start timestamp less than or equal to the given timestamp.
|
|
2140
|
-
* Returns null if the timestamp is before the track's first timestamp.
|
|
2141
|
-
*
|
|
2142
|
-
* @param timestamp - The timestamp used for retrieval, in seconds.
|
|
2143
|
-
*/
|
|
2144
|
-
async getBuffer(timestamp: number) {
|
|
2145
|
-
validateTimestamp(timestamp);
|
|
2146
|
-
|
|
2147
|
-
const data = await this._audioSampleSink.getSample(timestamp);
|
|
2148
|
-
return data && this._audioSampleToWrappedArrayBuffer(data);
|
|
2149
|
-
}
|
|
2150
|
-
|
|
2151
|
-
/**
|
|
2152
|
-
* Creates an async iterator that yields audio buffers of this track in presentation order. This method
|
|
2153
|
-
* will intelligently pre-decode a few buffers ahead to enable fast iteration.
|
|
2154
|
-
*
|
|
2155
|
-
* @param startTimestamp - The timestamp in seconds at which to start yielding buffers (inclusive).
|
|
2156
|
-
* @param endTimestamp - The timestamp in seconds at which to stop yielding buffers (exclusive).
|
|
2157
|
-
*/
|
|
2158
|
-
buffers(startTimestamp = 0, endTimestamp = Infinity) {
|
|
2159
|
-
return mapAsyncGenerator(
|
|
2160
|
-
this._audioSampleSink.samples(startTimestamp, endTimestamp),
|
|
2161
|
-
data => this._audioSampleToWrappedArrayBuffer(data),
|
|
2162
|
-
);
|
|
2163
|
-
}
|
|
2164
|
-
|
|
2165
|
-
/**
|
|
2166
|
-
* Creates an async iterator that yields an audio buffer for each timestamp in the argument. This method
|
|
2167
|
-
* uses an optimized decoding pipeline if these timestamps are monotonically sorted, decoding each packet at most
|
|
2168
|
-
* once, and is therefore more efficient than manually getting the buffer for every timestamp. The iterator may
|
|
2169
|
-
* yield null if no buffer is available for a given timestamp.
|
|
2170
|
-
*
|
|
2171
|
-
* @param timestamps - An iterable or async iterable of timestamps in seconds.
|
|
2172
|
-
*/
|
|
2173
|
-
buffersAtTimestamps(timestamps: AnyIterable<number>) {
|
|
2174
|
-
return mapAsyncGenerator(
|
|
2175
|
-
this._audioSampleSink.samplesAtTimestamps(timestamps),
|
|
2176
|
-
data => data && this._audioSampleToWrappedArrayBuffer(data),
|
|
2177
|
-
);
|
|
2178
|
-
}
|
|
2179
|
-
}
|