@kenzuya/mediabunny 1.26.0 → 1.28.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. package/README.md +1 -1
  2. package/dist/bundles/{mediabunny.mjs → mediabunny.js} +21963 -21388
  3. package/dist/bundles/mediabunny.min.js +490 -0
  4. package/dist/modules/shared/mp3-misc.d.ts.map +1 -1
  5. package/dist/modules/src/adts/adts-demuxer.d.ts +6 -6
  6. package/dist/modules/src/adts/adts-demuxer.d.ts.map +1 -1
  7. package/dist/modules/src/adts/adts-muxer.d.ts +4 -4
  8. package/dist/modules/src/adts/adts-muxer.d.ts.map +1 -1
  9. package/dist/modules/src/adts/adts-reader.d.ts +1 -1
  10. package/dist/modules/src/adts/adts-reader.d.ts.map +1 -1
  11. package/dist/modules/src/avi/avi-demuxer.d.ts +44 -0
  12. package/dist/modules/src/avi/avi-demuxer.d.ts.map +1 -0
  13. package/dist/modules/src/avi/avi-misc.d.ts +88 -0
  14. package/dist/modules/src/avi/avi-misc.d.ts.map +1 -0
  15. package/dist/modules/src/avi/avi-muxer.d.ts +45 -0
  16. package/dist/modules/src/avi/avi-muxer.d.ts.map +1 -0
  17. package/dist/modules/src/avi/riff-writer.d.ts +26 -0
  18. package/dist/modules/src/avi/riff-writer.d.ts.map +1 -0
  19. package/dist/modules/src/codec-data.d.ts +8 -3
  20. package/dist/modules/src/codec-data.d.ts.map +1 -1
  21. package/dist/modules/src/codec.d.ts +10 -10
  22. package/dist/modules/src/codec.d.ts.map +1 -1
  23. package/dist/modules/src/conversion.d.ts +33 -16
  24. package/dist/modules/src/conversion.d.ts.map +1 -1
  25. package/dist/modules/src/custom-coder.d.ts +8 -8
  26. package/dist/modules/src/custom-coder.d.ts.map +1 -1
  27. package/dist/modules/src/demuxer.d.ts +3 -3
  28. package/dist/modules/src/demuxer.d.ts.map +1 -1
  29. package/dist/modules/src/encode.d.ts +8 -8
  30. package/dist/modules/src/encode.d.ts.map +1 -1
  31. package/dist/modules/src/flac/flac-demuxer.d.ts +7 -7
  32. package/dist/modules/src/flac/flac-demuxer.d.ts.map +1 -1
  33. package/dist/modules/src/flac/flac-misc.d.ts +3 -3
  34. package/dist/modules/src/flac/flac-misc.d.ts.map +1 -1
  35. package/dist/modules/src/flac/flac-muxer.d.ts +5 -5
  36. package/dist/modules/src/flac/flac-muxer.d.ts.map +1 -1
  37. package/dist/modules/src/id3.d.ts +3 -3
  38. package/dist/modules/src/id3.d.ts.map +1 -1
  39. package/dist/modules/src/index.d.ts +20 -20
  40. package/dist/modules/src/index.d.ts.map +1 -1
  41. package/dist/modules/src/input-format.d.ts +22 -0
  42. package/dist/modules/src/input-format.d.ts.map +1 -1
  43. package/dist/modules/src/input-track.d.ts +8 -8
  44. package/dist/modules/src/input-track.d.ts.map +1 -1
  45. package/dist/modules/src/input.d.ts +12 -12
  46. package/dist/modules/src/isobmff/isobmff-boxes.d.ts +2 -2
  47. package/dist/modules/src/isobmff/isobmff-boxes.d.ts.map +1 -1
  48. package/dist/modules/src/isobmff/isobmff-demuxer.d.ts +12 -12
  49. package/dist/modules/src/isobmff/isobmff-demuxer.d.ts.map +1 -1
  50. package/dist/modules/src/isobmff/isobmff-misc.d.ts.map +1 -1
  51. package/dist/modules/src/isobmff/isobmff-muxer.d.ts +11 -11
  52. package/dist/modules/src/isobmff/isobmff-muxer.d.ts.map +1 -1
  53. package/dist/modules/src/isobmff/isobmff-reader.d.ts +2 -2
  54. package/dist/modules/src/isobmff/isobmff-reader.d.ts.map +1 -1
  55. package/dist/modules/src/matroska/ebml.d.ts +3 -3
  56. package/dist/modules/src/matroska/ebml.d.ts.map +1 -1
  57. package/dist/modules/src/matroska/matroska-demuxer.d.ts +13 -13
  58. package/dist/modules/src/matroska/matroska-demuxer.d.ts.map +1 -1
  59. package/dist/modules/src/matroska/matroska-input.d.ts +33 -0
  60. package/dist/modules/src/matroska/matroska-input.d.ts.map +1 -0
  61. package/dist/modules/src/matroska/matroska-misc.d.ts.map +1 -1
  62. package/dist/modules/src/matroska/matroska-muxer.d.ts +5 -5
  63. package/dist/modules/src/matroska/matroska-muxer.d.ts.map +1 -1
  64. package/dist/modules/src/media-sink.d.ts +5 -5
  65. package/dist/modules/src/media-sink.d.ts.map +1 -1
  66. package/dist/modules/src/media-source.d.ts +22 -4
  67. package/dist/modules/src/media-source.d.ts.map +1 -1
  68. package/dist/modules/src/metadata.d.ts +2 -2
  69. package/dist/modules/src/metadata.d.ts.map +1 -1
  70. package/dist/modules/src/misc.d.ts +5 -4
  71. package/dist/modules/src/misc.d.ts.map +1 -1
  72. package/dist/modules/src/mp3/mp3-demuxer.d.ts +7 -7
  73. package/dist/modules/src/mp3/mp3-demuxer.d.ts.map +1 -1
  74. package/dist/modules/src/mp3/mp3-muxer.d.ts +4 -4
  75. package/dist/modules/src/mp3/mp3-muxer.d.ts.map +1 -1
  76. package/dist/modules/src/mp3/mp3-reader.d.ts +2 -2
  77. package/dist/modules/src/mp3/mp3-reader.d.ts.map +1 -1
  78. package/dist/modules/src/mp3/mp3-writer.d.ts +1 -1
  79. package/dist/modules/src/mp3/mp3-writer.d.ts.map +1 -1
  80. package/dist/modules/src/muxer.d.ts +4 -4
  81. package/dist/modules/src/muxer.d.ts.map +1 -1
  82. package/dist/modules/src/node.d.ts +1 -1
  83. package/dist/modules/src/ogg/ogg-demuxer.d.ts +7 -7
  84. package/dist/modules/src/ogg/ogg-demuxer.d.ts.map +1 -1
  85. package/dist/modules/src/ogg/ogg-misc.d.ts +1 -1
  86. package/dist/modules/src/ogg/ogg-misc.d.ts.map +1 -1
  87. package/dist/modules/src/ogg/ogg-muxer.d.ts +5 -5
  88. package/dist/modules/src/ogg/ogg-muxer.d.ts.map +1 -1
  89. package/dist/modules/src/ogg/ogg-reader.d.ts +1 -1
  90. package/dist/modules/src/ogg/ogg-reader.d.ts.map +1 -1
  91. package/dist/modules/src/output-format.d.ts +51 -6
  92. package/dist/modules/src/output-format.d.ts.map +1 -1
  93. package/dist/modules/src/output.d.ts +13 -13
  94. package/dist/modules/src/output.d.ts.map +1 -1
  95. package/dist/modules/src/packet.d.ts +1 -1
  96. package/dist/modules/src/packet.d.ts.map +1 -1
  97. package/dist/modules/src/pcm.d.ts.map +1 -1
  98. package/dist/modules/src/reader.d.ts +2 -2
  99. package/dist/modules/src/reader.d.ts.map +1 -1
  100. package/dist/modules/src/sample.d.ts +57 -15
  101. package/dist/modules/src/sample.d.ts.map +1 -1
  102. package/dist/modules/src/source.d.ts +3 -3
  103. package/dist/modules/src/source.d.ts.map +1 -1
  104. package/dist/modules/src/subtitles.d.ts +1 -1
  105. package/dist/modules/src/subtitles.d.ts.map +1 -1
  106. package/dist/modules/src/target.d.ts +2 -2
  107. package/dist/modules/src/target.d.ts.map +1 -1
  108. package/dist/modules/src/tsconfig.tsbuildinfo +1 -1
  109. package/dist/modules/src/wave/riff-writer.d.ts +1 -1
  110. package/dist/modules/src/wave/riff-writer.d.ts.map +1 -1
  111. package/dist/modules/src/wave/wave-demuxer.d.ts +6 -6
  112. package/dist/modules/src/wave/wave-demuxer.d.ts.map +1 -1
  113. package/dist/modules/src/wave/wave-muxer.d.ts +4 -4
  114. package/dist/modules/src/wave/wave-muxer.d.ts.map +1 -1
  115. package/dist/modules/src/writer.d.ts +1 -1
  116. package/dist/modules/src/writer.d.ts.map +1 -1
  117. package/dist/packages/eac3/eac3.wasm +0 -0
  118. package/dist/packages/eac3/mediabunny-eac3.js +1058 -0
  119. package/dist/packages/eac3/mediabunny-eac3.min.js +44 -0
  120. package/dist/packages/mp3-encoder/mediabunny-mp3-encoder.js +694 -0
  121. package/dist/packages/mp3-encoder/mediabunny-mp3-encoder.min.js +58 -0
  122. package/dist/packages/mpeg4/mediabunny-mpeg4.js +1198 -0
  123. package/dist/packages/mpeg4/mediabunny-mpeg4.min.js +44 -0
  124. package/dist/packages/mpeg4/xvid.wasm +0 -0
  125. package/package.json +18 -57
  126. package/dist/bundles/mediabunny.cjs +0 -26140
  127. package/dist/bundles/mediabunny.min.cjs +0 -147
  128. package/dist/bundles/mediabunny.min.mjs +0 -146
  129. package/dist/mediabunny.d.ts +0 -3319
  130. package/dist/modules/shared/mp3-misc.js +0 -147
  131. package/dist/modules/src/adts/adts-demuxer.js +0 -239
  132. package/dist/modules/src/adts/adts-muxer.js +0 -80
  133. package/dist/modules/src/adts/adts-reader.js +0 -63
  134. package/dist/modules/src/codec-data.js +0 -1730
  135. package/dist/modules/src/codec.js +0 -869
  136. package/dist/modules/src/conversion.js +0 -1459
  137. package/dist/modules/src/custom-coder.js +0 -117
  138. package/dist/modules/src/demuxer.js +0 -12
  139. package/dist/modules/src/encode.js +0 -442
  140. package/dist/modules/src/flac/flac-demuxer.js +0 -504
  141. package/dist/modules/src/flac/flac-misc.js +0 -135
  142. package/dist/modules/src/flac/flac-muxer.js +0 -222
  143. package/dist/modules/src/id3.js +0 -848
  144. package/dist/modules/src/index.js +0 -28
  145. package/dist/modules/src/input-format.js +0 -480
  146. package/dist/modules/src/input-track.js +0 -372
  147. package/dist/modules/src/input.js +0 -188
  148. package/dist/modules/src/isobmff/isobmff-boxes.js +0 -1480
  149. package/dist/modules/src/isobmff/isobmff-demuxer.js +0 -2618
  150. package/dist/modules/src/isobmff/isobmff-misc.js +0 -20
  151. package/dist/modules/src/isobmff/isobmff-muxer.js +0 -966
  152. package/dist/modules/src/isobmff/isobmff-reader.js +0 -72
  153. package/dist/modules/src/matroska/ebml.js +0 -653
  154. package/dist/modules/src/matroska/matroska-demuxer.js +0 -2133
  155. package/dist/modules/src/matroska/matroska-misc.js +0 -20
  156. package/dist/modules/src/matroska/matroska-muxer.js +0 -1017
  157. package/dist/modules/src/media-sink.js +0 -1736
  158. package/dist/modules/src/media-source.js +0 -1825
  159. package/dist/modules/src/metadata.js +0 -193
  160. package/dist/modules/src/misc.js +0 -623
  161. package/dist/modules/src/mp3/mp3-demuxer.js +0 -285
  162. package/dist/modules/src/mp3/mp3-muxer.js +0 -123
  163. package/dist/modules/src/mp3/mp3-reader.js +0 -26
  164. package/dist/modules/src/mp3/mp3-writer.js +0 -78
  165. package/dist/modules/src/muxer.js +0 -50
  166. package/dist/modules/src/node.js +0 -9
  167. package/dist/modules/src/ogg/ogg-demuxer.js +0 -763
  168. package/dist/modules/src/ogg/ogg-misc.js +0 -78
  169. package/dist/modules/src/ogg/ogg-muxer.js +0 -353
  170. package/dist/modules/src/ogg/ogg-reader.js +0 -65
  171. package/dist/modules/src/output-format.js +0 -527
  172. package/dist/modules/src/output.js +0 -300
  173. package/dist/modules/src/packet.js +0 -182
  174. package/dist/modules/src/pcm.js +0 -85
  175. package/dist/modules/src/reader.js +0 -236
  176. package/dist/modules/src/sample.js +0 -1056
  177. package/dist/modules/src/source.js +0 -1182
  178. package/dist/modules/src/subtitles.js +0 -575
  179. package/dist/modules/src/target.js +0 -140
  180. package/dist/modules/src/wave/riff-writer.js +0 -30
  181. package/dist/modules/src/wave/wave-demuxer.js +0 -447
  182. package/dist/modules/src/wave/wave-muxer.js +0 -318
  183. package/dist/modules/src/writer.js +0 -370
  184. package/src/adts/adts-demuxer.ts +0 -331
  185. package/src/adts/adts-muxer.ts +0 -111
  186. package/src/adts/adts-reader.ts +0 -85
  187. package/src/codec-data.ts +0 -2078
  188. package/src/codec.ts +0 -1092
  189. package/src/conversion.ts +0 -2112
  190. package/src/custom-coder.ts +0 -197
  191. package/src/demuxer.ts +0 -24
  192. package/src/encode.ts +0 -739
  193. package/src/flac/flac-demuxer.ts +0 -730
  194. package/src/flac/flac-misc.ts +0 -164
  195. package/src/flac/flac-muxer.ts +0 -320
  196. package/src/id3.ts +0 -925
  197. package/src/index.ts +0 -221
  198. package/src/input-format.ts +0 -541
  199. package/src/input-track.ts +0 -529
  200. package/src/input.ts +0 -235
  201. package/src/isobmff/isobmff-boxes.ts +0 -1719
  202. package/src/isobmff/isobmff-demuxer.ts +0 -3190
  203. package/src/isobmff/isobmff-misc.ts +0 -29
  204. package/src/isobmff/isobmff-muxer.ts +0 -1348
  205. package/src/isobmff/isobmff-reader.ts +0 -91
  206. package/src/matroska/ebml.ts +0 -730
  207. package/src/matroska/matroska-demuxer.ts +0 -2481
  208. package/src/matroska/matroska-misc.ts +0 -29
  209. package/src/matroska/matroska-muxer.ts +0 -1276
  210. package/src/media-sink.ts +0 -2179
  211. package/src/media-source.ts +0 -2243
  212. package/src/metadata.ts +0 -320
  213. package/src/misc.ts +0 -798
  214. package/src/mp3/mp3-demuxer.ts +0 -383
  215. package/src/mp3/mp3-muxer.ts +0 -166
  216. package/src/mp3/mp3-reader.ts +0 -34
  217. package/src/mp3/mp3-writer.ts +0 -120
  218. package/src/muxer.ts +0 -88
  219. package/src/node.ts +0 -11
  220. package/src/ogg/ogg-demuxer.ts +0 -1053
  221. package/src/ogg/ogg-misc.ts +0 -116
  222. package/src/ogg/ogg-muxer.ts +0 -497
  223. package/src/ogg/ogg-reader.ts +0 -93
  224. package/src/output-format.ts +0 -945
  225. package/src/output.ts +0 -488
  226. package/src/packet.ts +0 -263
  227. package/src/pcm.ts +0 -112
  228. package/src/reader.ts +0 -323
  229. package/src/sample.ts +0 -1461
  230. package/src/source.ts +0 -1688
  231. package/src/subtitles.ts +0 -711
  232. package/src/target.ts +0 -204
  233. package/src/tsconfig.json +0 -16
  234. package/src/wave/riff-writer.ts +0 -36
  235. package/src/wave/wave-demuxer.ts +0 -529
  236. package/src/wave/wave-muxer.ts +0 -371
  237. package/src/writer.ts +0 -490
package/src/media-sink.ts DELETED
@@ -1,2179 +0,0 @@
1
- /*!
2
- * Copyright (c) 2025-present, Vanilagy and contributors
3
- *
4
- * This Source Code Form is subject to the terms of the Mozilla Public
5
- * License, v. 2.0. If a copy of the MPL was not distributed with this
6
- * file, You can obtain one at https://mozilla.org/MPL/2.0/.
7
- */
8
-
9
- import { parsePcmCodec, PCM_AUDIO_CODECS, PcmAudioCodec, VideoCodec, AudioCodec } from './codec';
10
- import {
11
- deserializeAvcDecoderConfigurationRecord,
12
- determineVideoPacketType,
13
- extractHevcNalUnits,
14
- extractNalUnitTypeForHevc,
15
- HevcNalUnitType,
16
- parseAvcSps,
17
- } from './codec-data';
18
- import { CustomVideoDecoder, customVideoDecoders, CustomAudioDecoder, customAudioDecoders } from './custom-coder';
19
- import { InputDisposedError } from './input';
20
- import { InputAudioTrack, InputTrack, InputVideoTrack } from './input-track';
21
- import {
22
- AnyIterable,
23
- assert,
24
- assertNever,
25
- CallSerializer,
26
- getInt24,
27
- getUint24,
28
- insertSorted,
29
- isChromium,
30
- isFirefox,
31
- isNumber,
32
- isWebKit,
33
- last,
34
- mapAsyncGenerator,
35
- promiseWithResolvers,
36
- Rotation,
37
- toAsyncIterator,
38
- toDataView,
39
- toUint8Array,
40
- validateAnyIterable,
41
- } from './misc';
42
- import { EncodedPacket } from './packet';
43
- import { fromAlaw, fromUlaw } from './pcm';
44
- import { AudioSample, clampCropRectangle, CropRectangle, validateCropRectangle, VideoSample } from './sample';
45
-
46
- /**
47
- * Additional options for controlling packet retrieval.
48
- * @group Media sinks
49
- * @public
50
- */
51
- export type PacketRetrievalOptions = {
52
- /**
53
- * When set to `true`, only packet metadata (like timestamp) will be retrieved - the actual packet data will not
54
- * be loaded.
55
- */
56
- metadataOnly?: boolean;
57
-
58
- /**
59
- * When set to true, key packets will be verified upon retrieval by looking into the packet's bitstream.
60
- * If not enabled, the packet types will be determined solely by what's stored in the containing file and may be
61
- * incorrect, potentially leading to decoder errors. Since determining a packet's actual type requires looking into
62
- * its data, this option cannot be enabled together with `metadataOnly`.
63
- */
64
- verifyKeyPackets?: boolean;
65
- };
66
-
67
- const validatePacketRetrievalOptions = (options: PacketRetrievalOptions) => {
68
- if (!options || typeof options !== 'object') {
69
- throw new TypeError('options must be an object.');
70
- }
71
- if (options.metadataOnly !== undefined && typeof options.metadataOnly !== 'boolean') {
72
- throw new TypeError('options.metadataOnly, when defined, must be a boolean.');
73
- }
74
- if (options.verifyKeyPackets !== undefined && typeof options.verifyKeyPackets !== 'boolean') {
75
- throw new TypeError('options.verifyKeyPackets, when defined, must be a boolean.');
76
- }
77
- if (options.verifyKeyPackets && options.metadataOnly) {
78
- throw new TypeError('options.verifyKeyPackets and options.metadataOnly cannot be enabled together.');
79
- }
80
- };
81
-
82
- const validateTimestamp = (timestamp: number) => {
83
- if (!isNumber(timestamp)) {
84
- throw new TypeError('timestamp must be a number.'); // It can be non-finite, that's fine
85
- }
86
- };
87
-
88
- const maybeFixPacketType = (
89
- track: InputTrack,
90
- promise: Promise<EncodedPacket | null>,
91
- options: PacketRetrievalOptions,
92
- ) => {
93
- if (options.verifyKeyPackets) {
94
- return promise.then(async (packet) => {
95
- if (!packet || packet.type === 'delta') {
96
- return packet;
97
- }
98
-
99
- const determinedType = await track.determinePacketType(packet);
100
- if (determinedType) {
101
- // @ts-expect-error Technically readonly
102
- packet.type = determinedType;
103
- }
104
-
105
- return packet;
106
- });
107
- } else {
108
- return promise;
109
- }
110
- };
111
-
112
- /**
113
- * Sink for retrieving encoded packets from an input track.
114
- * @group Media sinks
115
- * @public
116
- */
117
- export class EncodedPacketSink {
118
- /** @internal */
119
- _track: InputTrack;
120
-
121
- /** Creates a new {@link EncodedPacketSink} for the given {@link InputTrack}. */
122
- constructor(track: InputTrack) {
123
- if (!(track instanceof InputTrack)) {
124
- throw new TypeError('track must be an InputTrack.');
125
- }
126
-
127
- this._track = track;
128
- }
129
-
130
- /**
131
- * Retrieves the track's first packet (in decode order), or null if it has no packets. The first packet is very
132
- * likely to be a key packet.
133
- */
134
- getFirstPacket(options: PacketRetrievalOptions = {}) {
135
- validatePacketRetrievalOptions(options);
136
-
137
- if (this._track.input._disposed) {
138
- throw new InputDisposedError();
139
- }
140
-
141
- return maybeFixPacketType(this._track, this._track._backing.getFirstPacket(options), options);
142
- }
143
-
144
- /**
145
- * Retrieves the packet corresponding to the given timestamp, in seconds. More specifically, returns the last packet
146
- * (in presentation order) with a start timestamp less than or equal to the given timestamp. This method can be
147
- * used to retrieve a track's last packet using `getPacket(Infinity)`. The method returns null if the timestamp
148
- * is before the first packet in the track.
149
- *
150
- * @param timestamp - The timestamp used for retrieval, in seconds.
151
- */
152
- getPacket(timestamp: number, options: PacketRetrievalOptions = {}) {
153
- validateTimestamp(timestamp);
154
- validatePacketRetrievalOptions(options);
155
-
156
- if (this._track.input._disposed) {
157
- throw new InputDisposedError();
158
- }
159
-
160
- return maybeFixPacketType(this._track, this._track._backing.getPacket(timestamp, options), options);
161
- }
162
-
163
- /**
164
- * Retrieves the packet following the given packet (in decode order), or null if the given packet is the
165
- * last packet.
166
- */
167
- getNextPacket(packet: EncodedPacket, options: PacketRetrievalOptions = {}) {
168
- if (!(packet instanceof EncodedPacket)) {
169
- throw new TypeError('packet must be an EncodedPacket.');
170
- }
171
- validatePacketRetrievalOptions(options);
172
-
173
- if (this._track.input._disposed) {
174
- throw new InputDisposedError();
175
- }
176
-
177
- return maybeFixPacketType(this._track, this._track._backing.getNextPacket(packet, options), options);
178
- }
179
-
180
- /**
181
- * Retrieves the key packet corresponding to the given timestamp, in seconds. More specifically, returns the last
182
- * key packet (in presentation order) with a start timestamp less than or equal to the given timestamp. A key packet
183
- * is a packet that doesn't require previous packets to be decoded. This method can be used to retrieve a track's
184
- * last key packet using `getKeyPacket(Infinity)`. The method returns null if the timestamp is before the first
185
- * key packet in the track.
186
- *
187
- * To ensure that the returned packet is guaranteed to be a real key frame, enable `options.verifyKeyPackets`.
188
- *
189
- * @param timestamp - The timestamp used for retrieval, in seconds.
190
- */
191
- async getKeyPacket(timestamp: number, options: PacketRetrievalOptions = {}): Promise<EncodedPacket | null> {
192
- validateTimestamp(timestamp);
193
- validatePacketRetrievalOptions(options);
194
-
195
- if (this._track.input._disposed) {
196
- throw new InputDisposedError();
197
- }
198
-
199
- if (!options.verifyKeyPackets) {
200
- return this._track._backing.getKeyPacket(timestamp, options);
201
- }
202
-
203
- const packet = await this._track._backing.getKeyPacket(timestamp, options);
204
- if (!packet || packet.type === 'delta') {
205
- return packet;
206
- }
207
-
208
- const determinedType = await this._track.determinePacketType(packet);
209
- if (determinedType === 'delta') {
210
- // Try returning the previous key packet (in hopes that it's actually a key packet)
211
- return this.getKeyPacket(packet.timestamp - 1 / this._track.timeResolution, options);
212
- }
213
-
214
- return packet;
215
- }
216
-
217
- /**
218
- * Retrieves the key packet following the given packet (in decode order), or null if the given packet is the last
219
- * key packet.
220
- *
221
- * To ensure that the returned packet is guaranteed to be a real key frame, enable `options.verifyKeyPackets`.
222
- */
223
- async getNextKeyPacket(packet: EncodedPacket, options: PacketRetrievalOptions = {}): Promise<EncodedPacket | null> {
224
- if (!(packet instanceof EncodedPacket)) {
225
- throw new TypeError('packet must be an EncodedPacket.');
226
- }
227
- validatePacketRetrievalOptions(options);
228
-
229
- if (this._track.input._disposed) {
230
- throw new InputDisposedError();
231
- }
232
-
233
- if (!options.verifyKeyPackets) {
234
- return this._track._backing.getNextKeyPacket(packet, options);
235
- }
236
-
237
- const nextPacket = await this._track._backing.getNextKeyPacket(packet, options);
238
- if (!nextPacket || nextPacket.type === 'delta') {
239
- return nextPacket;
240
- }
241
-
242
- const determinedType = await this._track.determinePacketType(nextPacket);
243
- if (determinedType === 'delta') {
244
- // Try returning the next key packet (in hopes that it's actually a key packet)
245
- return this.getNextKeyPacket(nextPacket, options);
246
- }
247
-
248
- return nextPacket;
249
- }
250
-
251
- /**
252
- * Creates an async iterator that yields the packets in this track in decode order. To enable fast iteration, this
253
- * method will intelligently preload packets based on the speed of the consumer.
254
- *
255
- * @param startPacket - (optional) The packet from which iteration should begin. This packet will also be yielded.
256
- * @param endTimestamp - (optional) The timestamp at which iteration should end. This packet will _not_ be yielded.
257
- */
258
- packets(
259
- startPacket?: EncodedPacket,
260
- endPacket?: EncodedPacket,
261
- options: PacketRetrievalOptions = {},
262
- ): AsyncGenerator<EncodedPacket, void, unknown> {
263
- if (startPacket !== undefined && !(startPacket instanceof EncodedPacket)) {
264
- throw new TypeError('startPacket must be an EncodedPacket.');
265
- }
266
- if (startPacket !== undefined && startPacket.isMetadataOnly && !options?.metadataOnly) {
267
- throw new TypeError('startPacket can only be metadata-only if options.metadataOnly is enabled.');
268
- }
269
- if (endPacket !== undefined && !(endPacket instanceof EncodedPacket)) {
270
- throw new TypeError('endPacket must be an EncodedPacket.');
271
- }
272
- validatePacketRetrievalOptions(options);
273
-
274
- if (this._track.input._disposed) {
275
- throw new InputDisposedError();
276
- }
277
-
278
- const packetQueue: EncodedPacket[] = [];
279
-
280
- let { promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers();
281
- let { promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers();
282
- let ended = false;
283
- let terminated = false;
284
-
285
- // This stores errors that are "out of band" in the sense that they didn't occur in the normal flow of this
286
- // method but instead in a different context. This error should not go unnoticed and must be bubbled up to
287
- // the consumer.
288
- let outOfBandError = null as Error | null;
289
-
290
- const timestamps: number[] = [];
291
- // The queue should always be big enough to hold 1 second worth of packets
292
- const maxQueueSize = () => Math.max(2, timestamps.length);
293
-
294
- // The following is the "pump" process that keeps pumping packets into the queue
295
- (async () => {
296
- let packet = startPacket ?? await this.getFirstPacket(options);
297
-
298
- while (packet && !terminated && !this._track.input._disposed) {
299
- if (endPacket && packet.sequenceNumber >= endPacket?.sequenceNumber) {
300
- break;
301
- }
302
-
303
- if (packetQueue.length > maxQueueSize()) {
304
- ({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers());
305
- await queueDequeue;
306
- continue;
307
- }
308
-
309
- packetQueue.push(packet);
310
-
311
- onQueueNotEmpty();
312
- ({ promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers());
313
-
314
- packet = await this.getNextPacket(packet, options);
315
- }
316
-
317
- ended = true;
318
- onQueueNotEmpty();
319
- })().catch((error: Error) => {
320
- if (!outOfBandError) {
321
- outOfBandError = error;
322
- onQueueNotEmpty();
323
- }
324
- });
325
-
326
- const track = this._track;
327
-
328
- return {
329
- async next() {
330
- while (true) {
331
- if (track.input._disposed) {
332
- throw new InputDisposedError();
333
- } else if (terminated) {
334
- return { value: undefined, done: true };
335
- } else if (outOfBandError) {
336
- throw outOfBandError;
337
- } else if (packetQueue.length > 0) {
338
- const value = packetQueue.shift()!;
339
- const now = performance.now();
340
- timestamps.push(now);
341
-
342
- while (timestamps.length > 0 && now - timestamps[0]! >= 1000) {
343
- timestamps.shift();
344
- }
345
-
346
- onQueueDequeue();
347
-
348
- return { value, done: false };
349
- } else if (ended) {
350
- return { value: undefined, done: true };
351
- } else {
352
- await queueNotEmpty;
353
- }
354
- }
355
- },
356
- async return() {
357
- terminated = true;
358
- onQueueDequeue();
359
- onQueueNotEmpty();
360
-
361
- return { value: undefined, done: true };
362
- },
363
- async throw(error) {
364
- throw error;
365
- },
366
- [Symbol.asyncIterator]() {
367
- return this;
368
- },
369
- };
370
- }
371
- }
372
-
373
- abstract class DecoderWrapper<
374
- MediaSample extends VideoSample | AudioSample,
375
- > {
376
- constructor(
377
- public onSample: (sample: MediaSample) => unknown,
378
- public onError: (error: Error) => unknown,
379
- ) {}
380
-
381
- abstract getDecodeQueueSize(): number;
382
- abstract decode(packet: EncodedPacket): void;
383
- abstract flush(): Promise<void>;
384
- abstract close(): void;
385
- }
386
-
387
- /**
388
- * Base class for decoded media sample sinks.
389
- * @group Media sinks
390
- * @public
391
- */
392
- export abstract class BaseMediaSampleSink<
393
- MediaSample extends VideoSample | AudioSample,
394
- > {
395
- /** @internal */
396
- abstract _track: InputTrack;
397
-
398
- /** @internal */
399
- abstract _createDecoder(
400
- onSample: (sample: MediaSample) => unknown,
401
- onError: (error: Error) => unknown
402
- ): Promise<DecoderWrapper<MediaSample>>;
403
- /** @internal */
404
- abstract _createPacketSink(): EncodedPacketSink;
405
-
406
- /** @internal */
407
- protected mediaSamplesInRange(
408
- startTimestamp = 0,
409
- endTimestamp = Infinity,
410
- ): AsyncGenerator<MediaSample, void, unknown> {
411
- validateTimestamp(startTimestamp);
412
- validateTimestamp(endTimestamp);
413
-
414
- const sampleQueue: MediaSample[] = [];
415
- let firstSampleQueued = false;
416
- let lastSample: MediaSample | null = null;
417
- let { promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers();
418
- let { promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers();
419
- let decoderIsFlushed = false;
420
- let ended = false;
421
- let terminated = false;
422
-
423
- // This stores errors that are "out of band" in the sense that they didn't occur in the normal flow of this
424
- // method but instead in a different context. This error should not go unnoticed and must be bubbled up to
425
- // the consumer.
426
- let outOfBandError = null as Error | null;
427
-
428
- // The following is the "pump" process that keeps pumping packets into the decoder
429
- (async () => {
430
- const decoderError = new Error();
431
- const decoder = await this._createDecoder((sample) => {
432
- onQueueDequeue();
433
- if (sample.timestamp >= endTimestamp) {
434
- ended = true;
435
- }
436
-
437
- if (ended) {
438
- sample.close();
439
- return;
440
- }
441
-
442
- if (lastSample) {
443
- if (sample.timestamp > startTimestamp) {
444
- // We don't know ahead of time what the first first is. This is because the first first is the
445
- // last first whose timestamp is less than or equal to the start timestamp. Therefore we need to
446
- // wait for the first first after the start timestamp, and then we'll know that the previous
447
- // first was the first first.
448
- sampleQueue.push(lastSample);
449
- firstSampleQueued = true;
450
- } else {
451
- lastSample.close();
452
- }
453
- }
454
-
455
- if (sample.timestamp >= startTimestamp) {
456
- sampleQueue.push(sample);
457
- firstSampleQueued = true;
458
- }
459
-
460
- lastSample = firstSampleQueued ? null : sample;
461
-
462
- if (sampleQueue.length > 0) {
463
- onQueueNotEmpty();
464
- ({ promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers());
465
- }
466
- }, (error) => {
467
- if (!outOfBandError) {
468
- error.stack = decoderError.stack; // Provide a more useful stack trace
469
- outOfBandError = error;
470
- onQueueNotEmpty();
471
- }
472
- });
473
-
474
- const packetSink = this._createPacketSink();
475
- const keyPacket = await packetSink.getKeyPacket(startTimestamp, { verifyKeyPackets: true })
476
- ?? await packetSink.getFirstPacket();
477
- if (!keyPacket) {
478
- return;
479
- }
480
-
481
- let currentPacket: EncodedPacket | null = keyPacket;
482
-
483
- let endPacket: EncodedPacket | undefined = undefined;
484
- if (endTimestamp < Infinity) {
485
- // When an end timestamp is set, we cannot simply use that for the packet iterator due to out-of-order
486
- // frames (B-frames). Instead, we'll need to keep decoding packets until we get a frame that exceeds
487
- // this end time. However, we can still put a bound on it: Since key frames are by definition never
488
- // out of order, we can stop at the first key frame after the end timestamp.
489
- const packet = await packetSink.getPacket(endTimestamp);
490
- const keyPacket = !packet
491
- ? null
492
- : packet.type === 'key' && packet.timestamp === endTimestamp
493
- ? packet
494
- : await packetSink.getNextKeyPacket(packet, { verifyKeyPackets: true });
495
-
496
- if (keyPacket) {
497
- endPacket = keyPacket;
498
- }
499
- }
500
-
501
- const packets = packetSink.packets(keyPacket, endPacket);
502
- await packets.next(); // Skip the start packet as we already have it
503
-
504
- while (currentPacket && !ended && !this._track.input._disposed) {
505
- const maxQueueSize = computeMaxQueueSize(sampleQueue.length);
506
- if (sampleQueue.length + decoder.getDecodeQueueSize() > maxQueueSize) {
507
- ({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers());
508
- await queueDequeue;
509
- continue;
510
- }
511
-
512
- decoder.decode(currentPacket);
513
-
514
- const packetResult = await packets.next();
515
- if (packetResult.done) {
516
- break;
517
- }
518
-
519
- currentPacket = packetResult.value;
520
- }
521
-
522
- await packets.return();
523
-
524
- if (!terminated && !this._track.input._disposed) {
525
- await decoder.flush();
526
- }
527
- decoder.close();
528
-
529
- if (!firstSampleQueued && lastSample) {
530
- sampleQueue.push(lastSample);
531
- }
532
-
533
- decoderIsFlushed = true;
534
- onQueueNotEmpty(); // To unstuck the generator
535
- })().catch((error: Error) => {
536
- if (!outOfBandError) {
537
- outOfBandError = error;
538
- onQueueNotEmpty();
539
- }
540
- });
541
-
542
- const track = this._track;
543
- const closeSamples = () => {
544
- lastSample?.close();
545
- for (const sample of sampleQueue) {
546
- sample.close();
547
- }
548
- };
549
-
550
- return {
551
- async next() {
552
- while (true) {
553
- if (track.input._disposed) {
554
- closeSamples();
555
- throw new InputDisposedError();
556
- } else if (terminated) {
557
- return { value: undefined, done: true };
558
- } else if (outOfBandError) {
559
- closeSamples();
560
- throw outOfBandError;
561
- } else if (sampleQueue.length > 0) {
562
- const value = sampleQueue.shift()!;
563
- onQueueDequeue();
564
- return { value, done: false };
565
- } else if (!decoderIsFlushed) {
566
- await queueNotEmpty;
567
- } else {
568
- return { value: undefined, done: true };
569
- }
570
- }
571
- },
572
- async return() {
573
- terminated = true;
574
- ended = true;
575
- onQueueDequeue();
576
- onQueueNotEmpty();
577
- closeSamples();
578
-
579
- return { value: undefined, done: true };
580
- },
581
- async throw(error) {
582
- throw error;
583
- },
584
- [Symbol.asyncIterator]() {
585
- return this;
586
- },
587
- };
588
- }
589
-
590
- /** @internal */
591
- protected mediaSamplesAtTimestamps(
592
- timestamps: AnyIterable<number>,
593
- ): AsyncGenerator<MediaSample | null, void, unknown> {
594
- validateAnyIterable(timestamps);
595
- const timestampIterator = toAsyncIterator(timestamps);
596
- const timestampsOfInterest: number[] = [];
597
-
598
- const sampleQueue: (MediaSample | null)[] = [];
599
- let { promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers();
600
- let { promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers();
601
- let decoderIsFlushed = false;
602
- let terminated = false;
603
-
604
- // This stores errors that are "out of band" in the sense that they didn't occur in the normal flow of this
605
- // method but instead in a different context. This error should not go unnoticed and must be bubbled up to
606
- // the consumer.
607
- let outOfBandError = null as Error | null;
608
-
609
- const pushToQueue = (sample: MediaSample | null) => {
610
- sampleQueue.push(sample);
611
- onQueueNotEmpty();
612
- ({ promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers());
613
- };
614
-
615
- // The following is the "pump" process that keeps pumping packets into the decoder
616
- (async () => {
617
- const decoderError = new Error();
618
- const decoder = await this._createDecoder((sample) => {
619
- onQueueDequeue();
620
-
621
- if (terminated) {
622
- sample.close();
623
- return;
624
- }
625
-
626
- let sampleUses = 0;
627
- while (
628
- timestampsOfInterest.length > 0
629
- && sample.timestamp - timestampsOfInterest[0]! > -1e-10 // Give it a little epsilon
630
- ) {
631
- sampleUses++;
632
- timestampsOfInterest.shift();
633
- }
634
-
635
- if (sampleUses > 0) {
636
- for (let i = 0; i < sampleUses; i++) {
637
- // Clone the sample if we need to emit it multiple times
638
- pushToQueue((i < sampleUses - 1 ? sample.clone() : sample) as MediaSample);
639
- }
640
- } else {
641
- sample.close();
642
- }
643
- }, (error) => {
644
- if (!outOfBandError) {
645
- error.stack = decoderError.stack; // Provide a more useful stack trace
646
- outOfBandError = error;
647
- onQueueNotEmpty();
648
- }
649
- });
650
-
651
- const packetSink = this._createPacketSink();
652
- let lastPacket: EncodedPacket | null = null;
653
- let lastKeyPacket: EncodedPacket | null = null;
654
-
655
- // The end sequence number (inclusive) in the next batch of packets that will be decoded. The batch starts
656
- // at the last key frame and goes until this sequence number.
657
- let maxSequenceNumber = -1;
658
-
659
- const decodePackets = async () => {
660
- assert(lastKeyPacket);
661
-
662
- // Start at the current key packet
663
- let currentPacket = lastKeyPacket;
664
- decoder.decode(currentPacket);
665
-
666
- while (currentPacket.sequenceNumber < maxSequenceNumber) {
667
- const maxQueueSize = computeMaxQueueSize(sampleQueue.length);
668
- while (sampleQueue.length + decoder.getDecodeQueueSize() > maxQueueSize && !terminated) {
669
- ({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers());
670
- await queueDequeue;
671
- }
672
-
673
- if (terminated) {
674
- break;
675
- }
676
-
677
- const nextPacket = await packetSink.getNextPacket(currentPacket);
678
- assert(nextPacket);
679
-
680
- decoder.decode(nextPacket);
681
- currentPacket = nextPacket;
682
- }
683
-
684
- maxSequenceNumber = -1;
685
- };
686
-
687
- const flushDecoder = async () => {
688
- await decoder.flush();
689
-
690
- // We don't expect this list to have any elements in it anymore, but in case it does, let's emit
691
- // nulls for every remaining element, then clear it.
692
- for (let i = 0; i < timestampsOfInterest.length; i++) {
693
- pushToQueue(null);
694
- }
695
- timestampsOfInterest.length = 0;
696
- };
697
-
698
- for await (const timestamp of timestampIterator) {
699
- validateTimestamp(timestamp);
700
-
701
- if (terminated || this._track.input._disposed) {
702
- break;
703
- }
704
-
705
- const targetPacket = await packetSink.getPacket(timestamp);
706
- const keyPacket = targetPacket && await packetSink.getKeyPacket(timestamp, { verifyKeyPackets: true });
707
-
708
- if (!keyPacket) {
709
- if (maxSequenceNumber !== -1) {
710
- await decodePackets();
711
- await flushDecoder();
712
- }
713
-
714
- pushToQueue(null);
715
- lastPacket = null;
716
- continue;
717
- }
718
-
719
- // Check if the key packet has changed or if we're going back in time
720
- if (
721
- lastPacket
722
- && (
723
- keyPacket.sequenceNumber !== lastKeyPacket!.sequenceNumber
724
- || targetPacket.timestamp < lastPacket.timestamp
725
- )
726
- ) {
727
- await decodePackets();
728
- await flushDecoder(); // Always flush here, improves decoder compatibility
729
- }
730
-
731
- timestampsOfInterest.push(targetPacket.timestamp);
732
- maxSequenceNumber = Math.max(targetPacket.sequenceNumber, maxSequenceNumber);
733
-
734
- lastPacket = targetPacket;
735
- lastKeyPacket = keyPacket;
736
- }
737
-
738
- if (!terminated && !this._track.input._disposed) {
739
- if (maxSequenceNumber !== -1) {
740
- // We still need to decode packets
741
- await decodePackets();
742
- }
743
-
744
- await flushDecoder();
745
- }
746
- decoder.close();
747
-
748
- decoderIsFlushed = true;
749
- onQueueNotEmpty(); // To unstuck the generator
750
- })().catch((error: Error) => {
751
- if (!outOfBandError) {
752
- outOfBandError = error;
753
- onQueueNotEmpty();
754
- }
755
- });
756
-
757
- const track = this._track;
758
- const closeSamples = () => {
759
- for (const sample of sampleQueue) {
760
- sample?.close();
761
- }
762
- };
763
-
764
- return {
765
- async next() {
766
- while (true) {
767
- if (track.input._disposed) {
768
- closeSamples();
769
- throw new InputDisposedError();
770
- } else if (terminated) {
771
- return { value: undefined, done: true };
772
- } else if (outOfBandError) {
773
- closeSamples();
774
- throw outOfBandError;
775
- } else if (sampleQueue.length > 0) {
776
- const value = sampleQueue.shift();
777
- assert(value !== undefined);
778
- onQueueDequeue();
779
- return { value, done: false };
780
- } else if (!decoderIsFlushed) {
781
- await queueNotEmpty;
782
- } else {
783
- return { value: undefined, done: true };
784
- }
785
- }
786
- },
787
- async return() {
788
- terminated = true;
789
- onQueueDequeue();
790
- onQueueNotEmpty();
791
- closeSamples();
792
-
793
- return { value: undefined, done: true };
794
- },
795
- async throw(error) {
796
- throw error;
797
- },
798
- [Symbol.asyncIterator]() {
799
- return this;
800
- },
801
- };
802
- }
803
- }
804
-
805
- const computeMaxQueueSize = (decodedSampleQueueSize: number) => {
806
- // If we have decoded samples lying around, limit the total queue size to a small value (decoded samples can use up
807
- // a lot of memory). If not, we're fine with a much bigger queue of encoded packets waiting to be decoded. In fact,
808
- // some decoders only start flushing out decoded chunks when the packet queue is large enough.
809
- return decodedSampleQueueSize === 0 ? 40 : 8;
810
- };
811
-
812
- class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
813
- decoder: VideoDecoder | null = null;
814
-
815
- customDecoder: CustomVideoDecoder | null = null;
816
- customDecoderCallSerializer = new CallSerializer();
817
- customDecoderQueueSize = 0;
818
-
819
- inputTimestamps: number[] = []; // Timestamps input into the decoder, sorted.
820
- sampleQueue: VideoSample[] = []; // Safari-specific thing, check usage.
821
- currentPacketIndex = 0;
822
- raslSkipped = false; // For HEVC stuff
823
-
824
- // Alpha stuff
825
- alphaDecoder: VideoDecoder | null = null;
826
- alphaHadKeyframe = false;
827
- colorQueue: VideoFrame[] = [];
828
- alphaQueue: (VideoFrame | null)[] = [];
829
- merger: ColorAlphaMerger | null = null;
830
- mergerCreationFailed = false;
831
- decodedAlphaChunkCount = 0;
832
- alphaDecoderQueueSize = 0;
833
- /** Each value is the number of decoded alpha chunks at which a null alpha frame should be added. */
834
- nullAlphaFrameQueue: number[] = [];
835
- currentAlphaPacketIndex = 0;
836
- alphaRaslSkipped = false; // For HEVC stuff
837
-
838
- constructor(
839
- onSample: (sample: VideoSample) => unknown,
840
- onError: (error: Error) => unknown,
841
- public codec: VideoCodec,
842
- public decoderConfig: VideoDecoderConfig,
843
- public rotation: Rotation,
844
- public timeResolution: number,
845
- ) {
846
- super(onSample, onError);
847
-
848
- const MatchingCustomDecoder = customVideoDecoders.find(x => x.supports(codec, decoderConfig));
849
- if (MatchingCustomDecoder) {
850
- // @ts-expect-error "Can't create instance of abstract class 🤓"
851
- this.customDecoder = new MatchingCustomDecoder() as CustomVideoDecoder;
852
- // @ts-expect-error It's technically readonly
853
- this.customDecoder.codec = codec;
854
- // @ts-expect-error It's technically readonly
855
- this.customDecoder.config = decoderConfig;
856
- // @ts-expect-error It's technically readonly
857
- this.customDecoder.onSample = (sample) => {
858
- if (!(sample instanceof VideoSample)) {
859
- throw new TypeError('The argument passed to onSample must be a VideoSample.');
860
- }
861
-
862
- this.finalizeAndEmitSample(sample);
863
- };
864
-
865
- void this.customDecoderCallSerializer.call(() => this.customDecoder!.init());
866
- } else {
867
- const colorHandler = (frame: VideoFrame) => {
868
- if (this.alphaQueue.length > 0) {
869
- // Even when no alpha data is present (most of the time), there will be nulls in this queue
870
- const alphaFrame = this.alphaQueue.shift();
871
- assert(alphaFrame !== undefined);
872
-
873
- this.mergeAlpha(frame, alphaFrame);
874
- } else {
875
- this.colorQueue.push(frame);
876
- }
877
- };
878
-
879
- if (codec === 'avc' && this.decoderConfig.description && isChromium()) {
880
- // Chromium has/had a bug with playing interlaced AVC (https://issues.chromium.org/issues/456919096)
881
- // which can be worked around by requesting that software decoding be used. So, here we peek into the
882
- // AVC description, if present, and switch to software decoding if we find interlaced content.
883
- const record = deserializeAvcDecoderConfigurationRecord(toUint8Array(this.decoderConfig.description));
884
- if (record && record.sequenceParameterSets.length > 0) {
885
- const sps = parseAvcSps(record.sequenceParameterSets[0]!);
886
- if (sps && sps.frameMbsOnlyFlag === 0) {
887
- this.decoderConfig = {
888
- ...this.decoderConfig,
889
- hardwareAcceleration: 'prefer-software',
890
- };
891
- }
892
- }
893
- }
894
-
895
- this.decoder = new VideoDecoder({
896
- output: (frame) => {
897
- try {
898
- colorHandler(frame);
899
- } catch (error) {
900
- this.onError(error as Error);
901
- }
902
- },
903
- error: onError,
904
- });
905
- this.decoder.configure(this.decoderConfig);
906
- }
907
- }
908
-
909
- getDecodeQueueSize() {
910
- if (this.customDecoder) {
911
- return this.customDecoderQueueSize;
912
- } else {
913
- assert(this.decoder);
914
-
915
- return Math.max(
916
- this.decoder.decodeQueueSize,
917
- this.alphaDecoder?.decodeQueueSize ?? 0,
918
- );
919
- }
920
- }
921
-
922
- decode(packet: EncodedPacket) {
923
- if (this.codec === 'hevc' && this.currentPacketIndex > 0 && !this.raslSkipped) {
924
- if (this.hasHevcRaslPicture(packet.data)) {
925
- return; // Drop
926
- }
927
-
928
- this.raslSkipped = true;
929
- }
930
-
931
- this.currentPacketIndex++;
932
-
933
- if (this.customDecoder) {
934
- this.customDecoderQueueSize++;
935
- void this.customDecoderCallSerializer
936
- .call(() => this.customDecoder!.decode(packet))
937
- .then(() => this.customDecoderQueueSize--);
938
- } else {
939
- assert(this.decoder);
940
-
941
- if (!isWebKit()) {
942
- insertSorted(this.inputTimestamps, packet.timestamp, x => x);
943
- }
944
-
945
- this.decoder.decode(packet.toEncodedVideoChunk());
946
- this.decodeAlphaData(packet);
947
- }
948
- }
949
-
950
- decodeAlphaData(packet: EncodedPacket) {
951
- if (!packet.sideData.alpha || this.mergerCreationFailed) {
952
- // No alpha side data in the packet, most common case
953
- this.pushNullAlphaFrame();
954
- return;
955
- }
956
-
957
- if (!this.merger) {
958
- try {
959
- this.merger = new ColorAlphaMerger();
960
- } catch (error) {
961
- console.error('Due to an error, only color data will be decoded.', error);
962
-
963
- this.mergerCreationFailed = true;
964
- this.decodeAlphaData(packet); // Go again
965
-
966
- return;
967
- }
968
- }
969
-
970
- // Check if we need to set up the alpha decoder
971
- if (!this.alphaDecoder) {
972
- const alphaHandler = (frame: VideoFrame) => {
973
- this.alphaDecoderQueueSize--;
974
-
975
- if (this.colorQueue.length > 0) {
976
- const colorFrame = this.colorQueue.shift();
977
- assert(colorFrame !== undefined);
978
-
979
- this.mergeAlpha(colorFrame, frame);
980
- } else {
981
- this.alphaQueue.push(frame);
982
- }
983
-
984
- // Check if any null frames have been queued for this point
985
- this.decodedAlphaChunkCount++;
986
- while (
987
- this.nullAlphaFrameQueue.length > 0
988
- && this.nullAlphaFrameQueue[0] === this.decodedAlphaChunkCount
989
- ) {
990
- this.nullAlphaFrameQueue.shift();
991
-
992
- if (this.colorQueue.length > 0) {
993
- const colorFrame = this.colorQueue.shift();
994
- assert(colorFrame !== undefined);
995
-
996
- this.mergeAlpha(colorFrame, null);
997
- } else {
998
- this.alphaQueue.push(null);
999
- }
1000
- }
1001
- };
1002
-
1003
- this.alphaDecoder = new VideoDecoder({
1004
- output: (frame) => {
1005
- try {
1006
- alphaHandler(frame);
1007
- } catch (error) {
1008
- this.onError(error as Error);
1009
- }
1010
- },
1011
- error: this.onError,
1012
- });
1013
- this.alphaDecoder.configure(this.decoderConfig);
1014
- }
1015
-
1016
- const type = determineVideoPacketType(this.codec, this.decoderConfig, packet.sideData.alpha);
1017
-
1018
- // Alpha packets might follow a different key frame rhythm than the main packets. Therefore, before we start
1019
- // decoding, we must first find a packet that's actually a key frame. Until then, we treat the image as opaque.
1020
- if (!this.alphaHadKeyframe) {
1021
- this.alphaHadKeyframe = type === 'key';
1022
- }
1023
-
1024
- if (this.alphaHadKeyframe) {
1025
- // Same RASL skipping logic as for color, unlikely to be hit (since who uses HEVC with separate alpha??) but
1026
- // here for symmetry.
1027
- if (this.codec === 'hevc' && this.currentAlphaPacketIndex > 0 && !this.alphaRaslSkipped) {
1028
- if (this.hasHevcRaslPicture(packet.sideData.alpha)) {
1029
- this.pushNullAlphaFrame();
1030
- return;
1031
- }
1032
-
1033
- this.alphaRaslSkipped = true;
1034
- }
1035
-
1036
- this.currentAlphaPacketIndex++;
1037
- this.alphaDecoder.decode(packet.alphaToEncodedVideoChunk(type ?? packet.type));
1038
- this.alphaDecoderQueueSize++;
1039
- } else {
1040
- this.pushNullAlphaFrame();
1041
- }
1042
- }
1043
-
1044
- pushNullAlphaFrame() {
1045
- if (this.alphaDecoderQueueSize === 0) {
1046
- // Easy
1047
- this.alphaQueue.push(null);
1048
- } else {
1049
- // There are still alpha chunks being decoded, so pushing `null` immediately would result in out-of-order
1050
- // data and be incorrect. Instead, we need to enqueue a "null frame" for when the current decoder workload
1051
- // has finished.
1052
- this.nullAlphaFrameQueue.push(this.decodedAlphaChunkCount + this.alphaDecoderQueueSize);
1053
- }
1054
- }
1055
-
1056
- /**
1057
- * If we're using HEVC, we need to make sure to skip any RASL slices that follow a non-IDR key frame such as
1058
- * CRA_NUT. This is because RASL slices cannot be decoded without data before the CRA_NUT. Browsers behave
1059
- * differently here: Chromium drops the packets, Safari throws a decoder error. Either way, it's not good
1060
- * and causes bugs upstream. So, let's take the dropping into our own hands.
1061
- */
1062
- hasHevcRaslPicture(packetData: Uint8Array) {
1063
- const nalUnits = extractHevcNalUnits(packetData, this.decoderConfig);
1064
- return nalUnits.some((x) => {
1065
- const type = extractNalUnitTypeForHevc(x);
1066
- return type === HevcNalUnitType.RASL_N || type === HevcNalUnitType.RASL_R;
1067
- });
1068
- }
1069
-
1070
- /** Handler for the WebCodecs VideoDecoder for ironing out browser differences. */
1071
- sampleHandler(sample: VideoSample) {
1072
- if (isWebKit()) {
1073
- // For correct B-frame handling, we don't just hand over the frames directly but instead add them to
1074
- // a queue, because we want to ensure frames are emitted in presentation order. We flush the queue
1075
- // each time we receive a frame with a timestamp larger than the highest we've seen so far, as we
1076
- // can sure that is not a B-frame. Typically, WebCodecs automatically guarantees that frames are
1077
- // emitted in presentation order, but Safari doesn't always follow this rule.
1078
- if (this.sampleQueue.length > 0 && (sample.timestamp >= last(this.sampleQueue)!.timestamp)) {
1079
- for (const sample of this.sampleQueue) {
1080
- this.finalizeAndEmitSample(sample);
1081
- }
1082
-
1083
- this.sampleQueue.length = 0;
1084
- }
1085
-
1086
- insertSorted(this.sampleQueue, sample, x => x.timestamp);
1087
- } else {
1088
- // Assign it the next earliest timestamp from the input. We do this because browsers, by spec, are
1089
- // required to emit decoded frames in presentation order *while* retaining the timestamp of their
1090
- // originating EncodedVideoChunk. For files with B-frames but no out-of-order timestamps (like a
1091
- // missing ctts box, for example), this causes a mismatch. We therefore fix the timestamps and
1092
- // ensure they are sorted by doing this.
1093
- const timestamp = this.inputTimestamps.shift();
1094
-
1095
- // There's no way we'd have more decoded frames than encoded packets we passed in. Actually, the
1096
- // correspondence should be 1:1.
1097
- assert(timestamp !== undefined);
1098
-
1099
- sample.setTimestamp(timestamp);
1100
- this.finalizeAndEmitSample(sample);
1101
- }
1102
- }
1103
-
1104
- finalizeAndEmitSample(sample: VideoSample) {
1105
- // Round the timestamps to the time resolution
1106
- sample.setTimestamp(Math.round(sample.timestamp * this.timeResolution) / this.timeResolution);
1107
- sample.setDuration(Math.round(sample.duration * this.timeResolution) / this.timeResolution);
1108
- sample.setRotation(this.rotation);
1109
-
1110
- this.onSample(sample);
1111
- }
1112
-
1113
- mergeAlpha(color: VideoFrame, alpha: VideoFrame | null) {
1114
- if (!alpha) {
1115
- // Nothing needs to be merged
1116
- const finalSample = new VideoSample(color);
1117
- this.sampleHandler(finalSample);
1118
-
1119
- return;
1120
- }
1121
-
1122
- assert(this.merger);
1123
-
1124
- this.merger.update(color, alpha);
1125
- color.close();
1126
- alpha.close();
1127
-
1128
- const finalFrame = new VideoFrame(this.merger.canvas, {
1129
- timestamp: color.timestamp,
1130
- duration: color.duration ?? undefined,
1131
- });
1132
-
1133
- const finalSample = new VideoSample(finalFrame);
1134
- this.sampleHandler(finalSample);
1135
- }
1136
-
1137
- async flush() {
1138
- if (this.customDecoder) {
1139
- await this.customDecoderCallSerializer.call(() => this.customDecoder!.flush());
1140
- } else {
1141
- assert(this.decoder);
1142
- await Promise.all([
1143
- this.decoder.flush(),
1144
- this.alphaDecoder?.flush(),
1145
- ]);
1146
-
1147
- this.colorQueue.forEach(x => x.close());
1148
- this.colorQueue.length = 0;
1149
- this.alphaQueue.forEach(x => x?.close());
1150
- this.alphaQueue.length = 0;
1151
-
1152
- this.alphaHadKeyframe = false;
1153
- this.decodedAlphaChunkCount = 0;
1154
- this.alphaDecoderQueueSize = 0;
1155
- this.nullAlphaFrameQueue.length = 0;
1156
- this.currentAlphaPacketIndex = 0;
1157
- this.alphaRaslSkipped = false;
1158
- }
1159
-
1160
- if (isWebKit()) {
1161
- for (const sample of this.sampleQueue) {
1162
- this.finalizeAndEmitSample(sample);
1163
- }
1164
-
1165
- this.sampleQueue.length = 0;
1166
- }
1167
-
1168
- this.currentPacketIndex = 0;
1169
- this.raslSkipped = false;
1170
- }
1171
-
1172
- close() {
1173
- if (this.customDecoder) {
1174
- void this.customDecoderCallSerializer.call(() => this.customDecoder!.close());
1175
- } else {
1176
- assert(this.decoder);
1177
- this.decoder.close();
1178
- this.alphaDecoder?.close();
1179
-
1180
- this.colorQueue.forEach(x => x.close());
1181
- this.colorQueue.length = 0;
1182
- this.alphaQueue.forEach(x => x?.close());
1183
- this.alphaQueue.length = 0;
1184
-
1185
- this.merger?.close();
1186
- }
1187
-
1188
- for (const sample of this.sampleQueue) {
1189
- sample.close();
1190
- }
1191
- this.sampleQueue.length = 0;
1192
- }
1193
- }
1194
-
1195
- /** Utility class that merges together color and alpha information using simple WebGL 2 shaders. */
1196
- class ColorAlphaMerger {
1197
- canvas: OffscreenCanvas | HTMLCanvasElement;
1198
- private gl: WebGL2RenderingContext;
1199
- private program: WebGLProgram;
1200
- private vao: WebGLVertexArrayObject;
1201
- private colorTexture: WebGLTexture;
1202
- private alphaTexture: WebGLTexture;
1203
-
1204
- constructor() {
1205
- // Canvas will be resized later
1206
- if (typeof OffscreenCanvas !== 'undefined') {
1207
- // Prefer OffscreenCanvas for Worker environments
1208
- this.canvas = new OffscreenCanvas(300, 150);
1209
- } else {
1210
- this.canvas = document.createElement('canvas');
1211
- }
1212
-
1213
- const gl = this.canvas.getContext('webgl2', {
1214
- premultipliedAlpha: false,
1215
- }) as unknown as WebGL2RenderingContext | null; // Casting because of some TypeScript weirdness
1216
- if (!gl) {
1217
- throw new Error('Couldn\'t acquire WebGL 2 context.');
1218
- }
1219
-
1220
- this.gl = gl;
1221
- this.program = this.createProgram();
1222
- this.vao = this.createVAO();
1223
- this.colorTexture = this.createTexture();
1224
- this.alphaTexture = this.createTexture();
1225
-
1226
- this.gl.useProgram(this.program);
1227
- this.gl.uniform1i(this.gl.getUniformLocation(this.program, 'u_colorTexture'), 0);
1228
- this.gl.uniform1i(this.gl.getUniformLocation(this.program, 'u_alphaTexture'), 1);
1229
- }
1230
-
1231
- private createProgram(): WebGLProgram {
1232
- const vertexShader = this.createShader(this.gl.VERTEX_SHADER, `#version 300 es
1233
- in vec2 a_position;
1234
- in vec2 a_texCoord;
1235
- out vec2 v_texCoord;
1236
-
1237
- void main() {
1238
- gl_Position = vec4(a_position, 0.0, 1.0);
1239
- v_texCoord = a_texCoord;
1240
- }
1241
- `);
1242
-
1243
- const fragmentShader = this.createShader(this.gl.FRAGMENT_SHADER, `#version 300 es
1244
- precision highp float;
1245
-
1246
- uniform sampler2D u_colorTexture;
1247
- uniform sampler2D u_alphaTexture;
1248
- in vec2 v_texCoord;
1249
- out vec4 fragColor;
1250
-
1251
- void main() {
1252
- vec3 color = texture(u_colorTexture, v_texCoord).rgb;
1253
- float alpha = texture(u_alphaTexture, v_texCoord).r;
1254
- fragColor = vec4(color, alpha);
1255
- }
1256
- `);
1257
-
1258
- const program = this.gl.createProgram();
1259
- this.gl.attachShader(program, vertexShader);
1260
- this.gl.attachShader(program, fragmentShader);
1261
- this.gl.linkProgram(program);
1262
-
1263
- return program;
1264
- }
1265
-
1266
- private createShader(type: number, source: string): WebGLShader {
1267
- const shader = this.gl.createShader(type)!;
1268
- this.gl.shaderSource(shader, source);
1269
- this.gl.compileShader(shader);
1270
- return shader;
1271
- }
1272
-
1273
- private createVAO(): WebGLVertexArrayObject {
1274
- const vao = this.gl.createVertexArray();
1275
- this.gl.bindVertexArray(vao);
1276
-
1277
- const vertices = new Float32Array([
1278
- -1, -1, 0, 1,
1279
- 1, -1, 1, 1,
1280
- -1, 1, 0, 0,
1281
- 1, 1, 1, 0,
1282
- ]);
1283
-
1284
- const buffer = this.gl.createBuffer();
1285
- this.gl.bindBuffer(this.gl.ARRAY_BUFFER, buffer);
1286
- this.gl.bufferData(this.gl.ARRAY_BUFFER, vertices, this.gl.STATIC_DRAW);
1287
-
1288
- const positionLocation = this.gl.getAttribLocation(this.program, 'a_position');
1289
- const texCoordLocation = this.gl.getAttribLocation(this.program, 'a_texCoord');
1290
-
1291
- this.gl.enableVertexAttribArray(positionLocation);
1292
- this.gl.vertexAttribPointer(positionLocation, 2, this.gl.FLOAT, false, 16, 0);
1293
-
1294
- this.gl.enableVertexAttribArray(texCoordLocation);
1295
- this.gl.vertexAttribPointer(texCoordLocation, 2, this.gl.FLOAT, false, 16, 8);
1296
-
1297
- return vao;
1298
- }
1299
-
1300
- private createTexture(): WebGLTexture {
1301
- const texture = this.gl.createTexture();
1302
-
1303
- this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
1304
- this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
1305
- this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
1306
- this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR);
1307
- this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.LINEAR);
1308
-
1309
- return texture;
1310
- }
1311
-
1312
- update(color: VideoFrame, alpha: VideoFrame): void {
1313
- if (color.displayWidth !== this.canvas.width || color.displayHeight !== this.canvas.height) {
1314
- this.canvas.width = color.displayWidth;
1315
- this.canvas.height = color.displayHeight;
1316
- }
1317
-
1318
- this.gl.activeTexture(this.gl.TEXTURE0);
1319
- this.gl.bindTexture(this.gl.TEXTURE_2D, this.colorTexture);
1320
- this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, color);
1321
-
1322
- this.gl.activeTexture(this.gl.TEXTURE1);
1323
- this.gl.bindTexture(this.gl.TEXTURE_2D, this.alphaTexture);
1324
- this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, alpha);
1325
-
1326
- this.gl.viewport(0, 0, this.canvas.width, this.canvas.height);
1327
- this.gl.clear(this.gl.COLOR_BUFFER_BIT);
1328
-
1329
- this.gl.bindVertexArray(this.vao);
1330
- this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4);
1331
- }
1332
-
1333
- close() {
1334
- this.gl.getExtension('WEBGL_lose_context')?.loseContext();
1335
- this.gl = null as unknown as WebGL2RenderingContext;
1336
- }
1337
- }
1338
-
1339
- /**
1340
- * A sink that retrieves decoded video samples (video frames) from a video track.
1341
- * @group Media sinks
1342
- * @public
1343
- */
1344
- export class VideoSampleSink extends BaseMediaSampleSink<VideoSample> {
1345
- /** @internal */
1346
- _track: InputVideoTrack;
1347
-
1348
- /** Creates a new {@link VideoSampleSink} for the given {@link InputVideoTrack}. */
1349
- constructor(videoTrack: InputVideoTrack) {
1350
- if (!(videoTrack instanceof InputVideoTrack)) {
1351
- throw new TypeError('videoTrack must be an InputVideoTrack.');
1352
- }
1353
-
1354
- super();
1355
-
1356
- this._track = videoTrack;
1357
- }
1358
-
1359
- /** @internal */
1360
- async _createDecoder(
1361
- onSample: (sample: VideoSample) => unknown,
1362
- onError: (error: Error) => unknown,
1363
- ) {
1364
- if (!(await this._track.canDecode())) {
1365
- throw new Error(
1366
- 'This video track cannot be decoded by this browser. Make sure to check decodability before using'
1367
- + ' a track.',
1368
- );
1369
- }
1370
-
1371
- const codec = this._track.codec;
1372
- const rotation = this._track.rotation;
1373
- const decoderConfig = await this._track.getDecoderConfig();
1374
- const timeResolution = this._track.timeResolution;
1375
- assert(codec && decoderConfig);
1376
-
1377
- return new VideoDecoderWrapper(onSample, onError, codec, decoderConfig, rotation, timeResolution);
1378
- }
1379
-
1380
- /** @internal */
1381
- _createPacketSink() {
1382
- return new EncodedPacketSink(this._track);
1383
- }
1384
-
1385
- /**
1386
- * Retrieves the video sample (frame) corresponding to the given timestamp, in seconds. More specifically, returns
1387
- * the last video sample (in presentation order) with a start timestamp less than or equal to the given timestamp.
1388
- * Returns null if the timestamp is before the track's first timestamp.
1389
- *
1390
- * @param timestamp - The timestamp used for retrieval, in seconds.
1391
- */
1392
- async getSample(timestamp: number) {
1393
- validateTimestamp(timestamp);
1394
-
1395
- for await (const sample of this.mediaSamplesAtTimestamps([timestamp])) {
1396
- return sample;
1397
- }
1398
- throw new Error('Internal error: Iterator returned nothing.');
1399
- }
1400
-
1401
- /**
1402
- * Creates an async iterator that yields the video samples (frames) of this track in presentation order. This method
1403
- * will intelligently pre-decode a few frames ahead to enable fast iteration.
1404
- *
1405
- * @param startTimestamp - The timestamp in seconds at which to start yielding samples (inclusive).
1406
- * @param endTimestamp - The timestamp in seconds at which to stop yielding samples (exclusive).
1407
- */
1408
- samples(startTimestamp = 0, endTimestamp = Infinity) {
1409
- return this.mediaSamplesInRange(startTimestamp, endTimestamp);
1410
- }
1411
-
1412
- /**
1413
- * Creates an async iterator that yields a video sample (frame) for each timestamp in the argument. This method
1414
- * uses an optimized decoding pipeline if these timestamps are monotonically sorted, decoding each packet at most
1415
- * once, and is therefore more efficient than manually getting the sample for every timestamp. The iterator may
1416
- * yield null if no frame is available for a given timestamp.
1417
- *
1418
- * @param timestamps - An iterable or async iterable of timestamps in seconds.
1419
- */
1420
- samplesAtTimestamps(timestamps: AnyIterable<number>) {
1421
- return this.mediaSamplesAtTimestamps(timestamps);
1422
- }
1423
- }
1424
-
1425
- /**
1426
- * A canvas with additional timing information (timestamp & duration).
1427
- * @group Media sinks
1428
- * @public
1429
- */
1430
- export type WrappedCanvas = {
1431
- /** A canvas element or offscreen canvas. */
1432
- canvas: HTMLCanvasElement | OffscreenCanvas;
1433
- /** The timestamp of the corresponding video sample, in seconds. */
1434
- timestamp: number;
1435
- /** The duration of the corresponding video sample, in seconds. */
1436
- duration: number;
1437
- };
1438
-
1439
- /**
1440
- * Options for constructing a CanvasSink.
1441
- * @group Media sinks
1442
- * @public
1443
- */
1444
- export type CanvasSinkOptions = {
1445
- /**
1446
- * Whether the output canvases should have transparency instead of a black background. Defaults to `false`. Set
1447
- * this to `true` when using this sink to read transparent videos.
1448
- */
1449
- alpha?: boolean;
1450
- /**
1451
- * The width of the output canvas in pixels, defaulting to the display width of the video track. If height is not
1452
- * set, it will be deduced automatically based on aspect ratio.
1453
- */
1454
- width?: number;
1455
- /**
1456
- * The height of the output canvas in pixels, defaulting to the display height of the video track. If width is not
1457
- * set, it will be deduced automatically based on aspect ratio.
1458
- */
1459
- height?: number;
1460
- /**
1461
- * The fitting algorithm in case both width and height are set.
1462
- *
1463
- * - `'fill'` will stretch the image to fill the entire box, potentially altering aspect ratio.
1464
- * - `'contain'` will contain the entire image within the box while preserving aspect ratio. This may lead to
1465
- * letterboxing.
1466
- * - `'cover'` will scale the image until the entire box is filled, while preserving aspect ratio.
1467
- */
1468
- fit?: 'fill' | 'contain' | 'cover';
1469
- /**
1470
- * The clockwise rotation by which to rotate the raw video frame. Defaults to the rotation set in the file metadata.
1471
- * Rotation is applied before resizing.
1472
- */
1473
- rotation?: Rotation;
1474
- /**
1475
- * Specifies the rectangular region of the input video to crop to. The crop region will automatically be clamped to
1476
- * the dimensions of the input video track. Cropping is performed after rotation but before resizing.
1477
- */
1478
- crop?: CropRectangle;
1479
- /**
1480
- * When set, specifies the number of canvases in the pool. These canvases will be reused in a ring buffer /
1481
- * round-robin type fashion. This keeps the amount of allocated VRAM constant and relieves the browser from
1482
- * constantly allocating/deallocating canvases. A pool size of 0 or `undefined` disables the pool and means a new
1483
- * canvas is created each time.
1484
- */
1485
- poolSize?: number;
1486
- };
1487
-
1488
- /**
1489
- * A sink that renders video samples (frames) of the given video track to canvases. This is often more useful than
1490
- * directly retrieving frames, as it comes with common preprocessing steps such as resizing or applying rotation
1491
- * metadata.
1492
- *
1493
- * This sink will yield `HTMLCanvasElement`s when in a DOM context, and `OffscreenCanvas`es otherwise.
1494
- *
1495
- * @group Media sinks
1496
- * @public
1497
- */
1498
- export class CanvasSink {
1499
- /** @internal */
1500
- _videoTrack: InputVideoTrack;
1501
- /** @internal */
1502
- _alpha: boolean;
1503
- /** @internal */
1504
- _width: number;
1505
- /** @internal */
1506
- _height: number;
1507
- /** @internal */
1508
- _fit: 'fill' | 'contain' | 'cover';
1509
- /** @internal */
1510
- _rotation: Rotation;
1511
- /** @internal */
1512
- _crop?: { left: number; top: number; width: number; height: number };
1513
- /** @internal */
1514
- _videoSampleSink: VideoSampleSink;
1515
- /** @internal */
1516
- _canvasPool: (HTMLCanvasElement | OffscreenCanvas | null)[];
1517
- /** @internal */
1518
- _nextCanvasIndex = 0;
1519
-
1520
- /** Creates a new {@link CanvasSink} for the given {@link InputVideoTrack}. */
1521
- constructor(videoTrack: InputVideoTrack, options: CanvasSinkOptions = {}) {
1522
- if (!(videoTrack instanceof InputVideoTrack)) {
1523
- throw new TypeError('videoTrack must be an InputVideoTrack.');
1524
- }
1525
- if (options && typeof options !== 'object') {
1526
- throw new TypeError('options must be an object.');
1527
- }
1528
- if (options.alpha !== undefined && typeof options.alpha !== 'boolean') {
1529
- throw new TypeError('options.alpha, when provided, must be a boolean.');
1530
- }
1531
- if (options.width !== undefined && (!Number.isInteger(options.width) || options.width <= 0)) {
1532
- throw new TypeError('options.width, when defined, must be a positive integer.');
1533
- }
1534
- if (options.height !== undefined && (!Number.isInteger(options.height) || options.height <= 0)) {
1535
- throw new TypeError('options.height, when defined, must be a positive integer.');
1536
- }
1537
- if (options.fit !== undefined && !['fill', 'contain', 'cover'].includes(options.fit)) {
1538
- throw new TypeError('options.fit, when provided, must be one of "fill", "contain", or "cover".');
1539
- }
1540
- if (
1541
- options.width !== undefined
1542
- && options.height !== undefined
1543
- && options.fit === undefined
1544
- ) {
1545
- throw new TypeError(
1546
- 'When both options.width and options.height are provided, options.fit must also be provided.',
1547
- );
1548
- }
1549
- if (options.rotation !== undefined && ![0, 90, 180, 270].includes(options.rotation)) {
1550
- throw new TypeError('options.rotation, when provided, must be 0, 90, 180 or 270.');
1551
- }
1552
- if (options.crop !== undefined) {
1553
- validateCropRectangle(options.crop, 'options.');
1554
- }
1555
- if (
1556
- options.poolSize !== undefined
1557
- && (typeof options.poolSize !== 'number' || !Number.isInteger(options.poolSize) || options.poolSize < 0)
1558
- ) {
1559
- throw new TypeError('poolSize must be a non-negative integer.');
1560
- }
1561
-
1562
- const rotation = options.rotation ?? videoTrack.rotation;
1563
-
1564
- const [rotatedWidth, rotatedHeight] = rotation % 180 === 0
1565
- ? [videoTrack.codedWidth, videoTrack.codedHeight]
1566
- : [videoTrack.codedHeight, videoTrack.codedWidth];
1567
-
1568
- const crop = options.crop;
1569
- if (crop) {
1570
- clampCropRectangle(crop, rotatedWidth, rotatedHeight);
1571
- }
1572
-
1573
- let [width, height] = crop
1574
- ? [crop.width, crop.height]
1575
- : [rotatedWidth, rotatedHeight];
1576
- const originalAspectRatio = width / height;
1577
-
1578
- // If width and height aren't defined together, deduce the missing value using the aspect ratio
1579
- if (options.width !== undefined && options.height === undefined) {
1580
- width = options.width;
1581
- height = Math.round(width / originalAspectRatio);
1582
- } else if (options.width === undefined && options.height !== undefined) {
1583
- height = options.height;
1584
- width = Math.round(height * originalAspectRatio);
1585
- } else if (options.width !== undefined && options.height !== undefined) {
1586
- width = options.width;
1587
- height = options.height;
1588
- }
1589
-
1590
- this._videoTrack = videoTrack;
1591
- this._alpha = options.alpha ?? false;
1592
- this._width = width;
1593
- this._height = height;
1594
- this._rotation = rotation;
1595
- this._crop = crop;
1596
- this._fit = options.fit ?? 'fill';
1597
- this._videoSampleSink = new VideoSampleSink(videoTrack);
1598
- this._canvasPool = Array.from({ length: options.poolSize ?? 0 }, () => null);
1599
- }
1600
-
1601
- /** @internal */
1602
- _videoSampleToWrappedCanvas(sample: VideoSample): WrappedCanvas {
1603
- let canvas = this._canvasPool[this._nextCanvasIndex];
1604
- let canvasIsNew = false;
1605
-
1606
- if (!canvas) {
1607
- if (typeof document !== 'undefined') {
1608
- // Prefer an HTMLCanvasElement
1609
- canvas = document.createElement('canvas');
1610
- canvas.width = this._width;
1611
- canvas.height = this._height;
1612
- } else {
1613
- canvas = new OffscreenCanvas(this._width, this._height);
1614
- }
1615
-
1616
- if (this._canvasPool.length > 0) {
1617
- this._canvasPool[this._nextCanvasIndex] = canvas;
1618
- }
1619
-
1620
- canvasIsNew = true;
1621
- }
1622
-
1623
- if (this._canvasPool.length > 0) {
1624
- this._nextCanvasIndex = (this._nextCanvasIndex + 1) % this._canvasPool.length;
1625
- }
1626
-
1627
- const context = canvas.getContext('2d', {
1628
- alpha: this._alpha || isFirefox(), // Firefox has VideoFrame glitches with opaque canvases
1629
- }) as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
1630
- assert(context);
1631
-
1632
- context.resetTransform();
1633
-
1634
- if (!canvasIsNew) {
1635
- if (!this._alpha && isFirefox()) {
1636
- context.fillStyle = 'black';
1637
- context.fillRect(0, 0, this._width, this._height);
1638
- } else {
1639
- context.clearRect(0, 0, this._width, this._height);
1640
- }
1641
- }
1642
-
1643
- sample.drawWithFit(context, {
1644
- fit: this._fit,
1645
- rotation: this._rotation,
1646
- crop: this._crop,
1647
- });
1648
-
1649
- const result = {
1650
- canvas,
1651
- timestamp: sample.timestamp,
1652
- duration: sample.duration,
1653
- };
1654
-
1655
- sample.close();
1656
- return result;
1657
- }
1658
-
1659
- /**
1660
- * Retrieves a canvas with the video frame corresponding to the given timestamp, in seconds. More specifically,
1661
- * returns the last video frame (in presentation order) with a start timestamp less than or equal to the given
1662
- * timestamp. Returns null if the timestamp is before the track's first timestamp.
1663
- *
1664
- * @param timestamp - The timestamp used for retrieval, in seconds.
1665
- */
1666
- async getCanvas(timestamp: number) {
1667
- validateTimestamp(timestamp);
1668
-
1669
- const sample = await this._videoSampleSink.getSample(timestamp);
1670
- return sample && this._videoSampleToWrappedCanvas(sample);
1671
- }
1672
-
1673
- /**
1674
- * Creates an async iterator that yields canvases with the video frames of this track in presentation order. This
1675
- * method will intelligently pre-decode a few frames ahead to enable fast iteration.
1676
- *
1677
- * @param startTimestamp - The timestamp in seconds at which to start yielding canvases (inclusive).
1678
- * @param endTimestamp - The timestamp in seconds at which to stop yielding canvases (exclusive).
1679
- */
1680
- canvases(startTimestamp = 0, endTimestamp = Infinity) {
1681
- return mapAsyncGenerator(
1682
- this._videoSampleSink.samples(startTimestamp, endTimestamp),
1683
- sample => this._videoSampleToWrappedCanvas(sample),
1684
- );
1685
- }
1686
-
1687
- /**
1688
- * Creates an async iterator that yields a canvas for each timestamp in the argument. This method uses an optimized
1689
- * decoding pipeline if these timestamps are monotonically sorted, decoding each packet at most once, and is
1690
- * therefore more efficient than manually getting the canvas for every timestamp. The iterator may yield null if
1691
- * no frame is available for a given timestamp.
1692
- *
1693
- * @param timestamps - An iterable or async iterable of timestamps in seconds.
1694
- */
1695
- canvasesAtTimestamps(timestamps: AnyIterable<number>) {
1696
- return mapAsyncGenerator(
1697
- this._videoSampleSink.samplesAtTimestamps(timestamps),
1698
- sample => sample && this._videoSampleToWrappedCanvas(sample),
1699
- );
1700
- }
1701
- }
1702
-
1703
- class AudioDecoderWrapper extends DecoderWrapper<AudioSample> {
1704
- decoder: AudioDecoder | null = null;
1705
-
1706
- customDecoder: CustomAudioDecoder | null = null;
1707
- customDecoderCallSerializer = new CallSerializer();
1708
- customDecoderQueueSize = 0;
1709
-
1710
- // Internal state to accumulate a precise current timestamp based on audio durations, not the (potentially
1711
- // inaccurate) packet timestamps.
1712
- currentTimestamp: number | null = null;
1713
-
1714
- constructor(
1715
- onSample: (sample: AudioSample) => unknown,
1716
- onError: (error: Error) => unknown,
1717
- codec: AudioCodec,
1718
- decoderConfig: AudioDecoderConfig,
1719
- ) {
1720
- super(onSample, onError);
1721
-
1722
- const sampleHandler = (sample: AudioSample) => {
1723
- if (
1724
- this.currentTimestamp === null
1725
- || Math.abs(sample.timestamp - this.currentTimestamp) >= sample.duration
1726
- ) {
1727
- // We need to sync with the sample timestamp again
1728
- this.currentTimestamp = sample.timestamp;
1729
- }
1730
-
1731
- const preciseTimestamp = this.currentTimestamp;
1732
- this.currentTimestamp += sample.duration;
1733
-
1734
- if (sample.numberOfFrames === 0) {
1735
- // We skip zero-data (empty) AudioSamples. These are sometimes emitted, for example, by Firefox when it
1736
- // decodes Vorbis (at the start).
1737
- sample.close();
1738
- return;
1739
- }
1740
-
1741
- // Round the timestamp to the sample rate
1742
- const sampleRate = decoderConfig.sampleRate;
1743
- sample.setTimestamp(Math.round(preciseTimestamp * sampleRate) / sampleRate);
1744
-
1745
- onSample(sample);
1746
- };
1747
-
1748
- const MatchingCustomDecoder = customAudioDecoders.find(x => x.supports(codec, decoderConfig));
1749
- if (MatchingCustomDecoder) {
1750
- // @ts-expect-error "Can't create instance of abstract class 🤓"
1751
- this.customDecoder = new MatchingCustomDecoder() as CustomAudioDecoder;
1752
- // @ts-expect-error It's technically readonly
1753
- this.customDecoder.codec = codec;
1754
- // @ts-expect-error It's technically readonly
1755
- this.customDecoder.config = decoderConfig;
1756
- // @ts-expect-error It's technically readonly
1757
- this.customDecoder.onSample = (sample) => {
1758
- if (!(sample instanceof AudioSample)) {
1759
- throw new TypeError('The argument passed to onSample must be an AudioSample.');
1760
- }
1761
-
1762
- sampleHandler(sample);
1763
- };
1764
-
1765
- void this.customDecoderCallSerializer.call(() => this.customDecoder!.init());
1766
- } else {
1767
- this.decoder = new AudioDecoder({
1768
- output: (data) => {
1769
- try {
1770
- sampleHandler(new AudioSample(data));
1771
- } catch (error) {
1772
- this.onError(error as Error);
1773
- }
1774
- },
1775
- error: onError,
1776
- });
1777
- this.decoder.configure(decoderConfig);
1778
- }
1779
- }
1780
-
1781
- getDecodeQueueSize() {
1782
- if (this.customDecoder) {
1783
- return this.customDecoderQueueSize;
1784
- } else {
1785
- assert(this.decoder);
1786
- return this.decoder.decodeQueueSize;
1787
- }
1788
- }
1789
-
1790
- decode(packet: EncodedPacket) {
1791
- if (this.customDecoder) {
1792
- this.customDecoderQueueSize++;
1793
- void this.customDecoderCallSerializer
1794
- .call(() => this.customDecoder!.decode(packet))
1795
- .then(() => this.customDecoderQueueSize--);
1796
- } else {
1797
- assert(this.decoder);
1798
- this.decoder.decode(packet.toEncodedAudioChunk());
1799
- }
1800
- }
1801
-
1802
- flush() {
1803
- if (this.customDecoder) {
1804
- return this.customDecoderCallSerializer.call(() => this.customDecoder!.flush());
1805
- } else {
1806
- assert(this.decoder);
1807
- return this.decoder.flush();
1808
- }
1809
- }
1810
-
1811
- close() {
1812
- if (this.customDecoder) {
1813
- void this.customDecoderCallSerializer.call(() => this.customDecoder!.close());
1814
- } else {
1815
- assert(this.decoder);
1816
- this.decoder.close();
1817
- }
1818
- }
1819
- }
1820
-
1821
- // There are a lot of PCM variants not natively supported by the browser and by AudioData. Therefore we need a simple
1822
- // decoder that maps any input PCM format into a PCM format supported by the browser.
1823
- class PcmAudioDecoderWrapper extends DecoderWrapper<AudioSample> {
1824
- codec: PcmAudioCodec;
1825
-
1826
- inputSampleSize: 1 | 2 | 3 | 4 | 8;
1827
- readInputValue: (view: DataView, byteOffset: number) => number;
1828
-
1829
- outputSampleSize: 1 | 2 | 4;
1830
- outputFormat: 'u8' | 's16' | 's32' | 'f32';
1831
- writeOutputValue: (view: DataView, byteOffset: number, value: number) => void;
1832
-
1833
- // Internal state to accumulate a precise current timestamp based on audio durations, not the (potentially
1834
- // inaccurate) packet timestamps.
1835
- currentTimestamp: number | null = null;
1836
-
1837
- constructor(
1838
- onSample: (sample: AudioSample) => unknown,
1839
- onError: (error: Error) => unknown,
1840
- public decoderConfig: AudioDecoderConfig,
1841
- ) {
1842
- super(onSample, onError);
1843
-
1844
- assert((PCM_AUDIO_CODECS as readonly string[]).includes(decoderConfig.codec));
1845
- this.codec = decoderConfig.codec as PcmAudioCodec;
1846
-
1847
- const { dataType, sampleSize, littleEndian } = parsePcmCodec(this.codec);
1848
- this.inputSampleSize = sampleSize;
1849
-
1850
- switch (sampleSize) {
1851
- case 1: {
1852
- if (dataType === 'unsigned') {
1853
- this.readInputValue = (view, byteOffset) => view.getUint8(byteOffset) - 2 ** 7;
1854
- } else if (dataType === 'signed') {
1855
- this.readInputValue = (view, byteOffset) => view.getInt8(byteOffset);
1856
- } else if (dataType === 'ulaw') {
1857
- this.readInputValue = (view, byteOffset) => fromUlaw(view.getUint8(byteOffset));
1858
- } else if (dataType === 'alaw') {
1859
- this.readInputValue = (view, byteOffset) => fromAlaw(view.getUint8(byteOffset));
1860
- } else {
1861
- assert(false);
1862
- }
1863
- }; break;
1864
- case 2: {
1865
- if (dataType === 'unsigned') {
1866
- this.readInputValue = (view, byteOffset) => view.getUint16(byteOffset, littleEndian) - 2 ** 15;
1867
- } else if (dataType === 'signed') {
1868
- this.readInputValue = (view, byteOffset) => view.getInt16(byteOffset, littleEndian);
1869
- } else {
1870
- assert(false);
1871
- }
1872
- }; break;
1873
- case 3: {
1874
- if (dataType === 'unsigned') {
1875
- this.readInputValue = (view, byteOffset) => getUint24(view, byteOffset, littleEndian) - 2 ** 23;
1876
- } else if (dataType === 'signed') {
1877
- this.readInputValue = (view, byteOffset) => getInt24(view, byteOffset, littleEndian);
1878
- } else {
1879
- assert(false);
1880
- }
1881
- }; break;
1882
- case 4: {
1883
- if (dataType === 'unsigned') {
1884
- this.readInputValue = (view, byteOffset) => view.getUint32(byteOffset, littleEndian) - 2 ** 31;
1885
- } else if (dataType === 'signed') {
1886
- this.readInputValue = (view, byteOffset) => view.getInt32(byteOffset, littleEndian);
1887
- } else if (dataType === 'float') {
1888
- this.readInputValue = (view, byteOffset) => view.getFloat32(byteOffset, littleEndian);
1889
- } else {
1890
- assert(false);
1891
- }
1892
- }; break;
1893
- case 8: {
1894
- if (dataType === 'float') {
1895
- this.readInputValue = (view, byteOffset) => view.getFloat64(byteOffset, littleEndian);
1896
- } else {
1897
- assert(false);
1898
- }
1899
- }; break;
1900
- default: {
1901
- assertNever(sampleSize);
1902
- assert(false);
1903
- };
1904
- }
1905
-
1906
- switch (sampleSize) {
1907
- case 1: {
1908
- if (dataType === 'ulaw' || dataType === 'alaw') {
1909
- this.outputSampleSize = 2;
1910
- this.outputFormat = 's16';
1911
- this.writeOutputValue = (view, byteOffset, value) => view.setInt16(byteOffset, value, true);
1912
- } else {
1913
- this.outputSampleSize = 1;
1914
- this.outputFormat = 'u8';
1915
- this.writeOutputValue = (view, byteOffset, value) => view.setUint8(byteOffset, value + 2 ** 7);
1916
- }
1917
- }; break;
1918
- case 2: {
1919
- this.outputSampleSize = 2;
1920
- this.outputFormat = 's16';
1921
- this.writeOutputValue = (view, byteOffset, value) => view.setInt16(byteOffset, value, true);
1922
- }; break;
1923
- case 3: {
1924
- this.outputSampleSize = 4;
1925
- this.outputFormat = 's32';
1926
- // From https://www.w3.org/TR/webcodecs:
1927
- // AudioData containing 24-bit samples SHOULD store those samples in s32 or f32. When samples are
1928
- // stored in s32, each sample MUST be left-shifted by 8 bits.
1929
- this.writeOutputValue = (view, byteOffset, value) => view.setInt32(byteOffset, value << 8, true);
1930
- }; break;
1931
- case 4: {
1932
- this.outputSampleSize = 4;
1933
-
1934
- if (dataType === 'float') {
1935
- this.outputFormat = 'f32';
1936
- this.writeOutputValue = (view, byteOffset, value) => view.setFloat32(byteOffset, value, true);
1937
- } else {
1938
- this.outputFormat = 's32';
1939
- this.writeOutputValue = (view, byteOffset, value) => view.setInt32(byteOffset, value, true);
1940
- }
1941
- }; break;
1942
- case 8: {
1943
- this.outputSampleSize = 4;
1944
-
1945
- this.outputFormat = 'f32';
1946
- this.writeOutputValue = (view, byteOffset, value) => view.setFloat32(byteOffset, value, true);
1947
- }; break;
1948
- default: {
1949
- assertNever(sampleSize);
1950
- assert(false);
1951
- };
1952
- };
1953
- }
1954
-
1955
- getDecodeQueueSize() {
1956
- return 0;
1957
- }
1958
-
1959
- decode(packet: EncodedPacket) {
1960
- const inputView = toDataView(packet.data);
1961
-
1962
- const numberOfFrames = packet.byteLength / this.decoderConfig.numberOfChannels / this.inputSampleSize;
1963
-
1964
- const outputBufferSize = numberOfFrames * this.decoderConfig.numberOfChannels * this.outputSampleSize;
1965
- const outputBuffer = new ArrayBuffer(outputBufferSize);
1966
- const outputView = new DataView(outputBuffer);
1967
-
1968
- for (let i = 0; i < numberOfFrames * this.decoderConfig.numberOfChannels; i++) {
1969
- const inputIndex = i * this.inputSampleSize;
1970
- const outputIndex = i * this.outputSampleSize;
1971
-
1972
- const value = this.readInputValue(inputView, inputIndex);
1973
- this.writeOutputValue(outputView, outputIndex, value);
1974
- }
1975
-
1976
- const preciseDuration = numberOfFrames / this.decoderConfig.sampleRate;
1977
- if (this.currentTimestamp === null || Math.abs(packet.timestamp - this.currentTimestamp) >= preciseDuration) {
1978
- // We need to sync with the packet timestamp again
1979
- this.currentTimestamp = packet.timestamp;
1980
- }
1981
-
1982
- const preciseTimestamp = this.currentTimestamp;
1983
- this.currentTimestamp += preciseDuration;
1984
-
1985
- const audioSample = new AudioSample({
1986
- format: this.outputFormat,
1987
- data: outputBuffer,
1988
- numberOfChannels: this.decoderConfig.numberOfChannels,
1989
- sampleRate: this.decoderConfig.sampleRate,
1990
- numberOfFrames,
1991
- timestamp: preciseTimestamp,
1992
- });
1993
-
1994
- this.onSample(audioSample);
1995
- }
1996
-
1997
- async flush() {
1998
- // Do nothing
1999
- }
2000
-
2001
- close() {
2002
- // Do nothing
2003
- }
2004
- }
2005
-
2006
- /**
2007
- * Sink for retrieving decoded audio samples from an audio track.
2008
- * @group Media sinks
2009
- * @public
2010
- */
2011
- export class AudioSampleSink extends BaseMediaSampleSink<AudioSample> {
2012
- /** @internal */
2013
- _track: InputAudioTrack;
2014
-
2015
- /** Creates a new {@link AudioSampleSink} for the given {@link InputAudioTrack}. */
2016
- constructor(audioTrack: InputAudioTrack) {
2017
- if (!(audioTrack instanceof InputAudioTrack)) {
2018
- throw new TypeError('audioTrack must be an InputAudioTrack.');
2019
- }
2020
-
2021
- super();
2022
-
2023
- this._track = audioTrack;
2024
- }
2025
-
2026
- /** @internal */
2027
- async _createDecoder(
2028
- onSample: (sample: AudioSample) => unknown,
2029
- onError: (error: Error) => unknown,
2030
- ) {
2031
- if (!(await this._track.canDecode())) {
2032
- throw new Error(
2033
- 'This audio track cannot be decoded by this browser. Make sure to check decodability before using'
2034
- + ' a track.',
2035
- );
2036
- }
2037
-
2038
- const codec = this._track.codec;
2039
- const decoderConfig = await this._track.getDecoderConfig();
2040
- assert(codec && decoderConfig);
2041
-
2042
- if ((PCM_AUDIO_CODECS as readonly string[]).includes(decoderConfig.codec)) {
2043
- return new PcmAudioDecoderWrapper(onSample, onError, decoderConfig);
2044
- } else {
2045
- return new AudioDecoderWrapper(onSample, onError, codec, decoderConfig);
2046
- }
2047
- }
2048
-
2049
- /** @internal */
2050
- _createPacketSink() {
2051
- return new EncodedPacketSink(this._track);
2052
- }
2053
-
2054
- /**
2055
- * Retrieves the audio sample corresponding to the given timestamp, in seconds. More specifically, returns
2056
- * the last audio sample (in presentation order) with a start timestamp less than or equal to the given timestamp.
2057
- * Returns null if the timestamp is before the track's first timestamp.
2058
- *
2059
- * @param timestamp - The timestamp used for retrieval, in seconds.
2060
- */
2061
- async getSample(timestamp: number) {
2062
- validateTimestamp(timestamp);
2063
-
2064
- for await (const sample of this.mediaSamplesAtTimestamps([timestamp])) {
2065
- return sample;
2066
- }
2067
- throw new Error('Internal error: Iterator returned nothing.');
2068
- }
2069
-
2070
- /**
2071
- * Creates an async iterator that yields the audio samples of this track in presentation order. This method
2072
- * will intelligently pre-decode a few samples ahead to enable fast iteration.
2073
- *
2074
- * @param startTimestamp - The timestamp in seconds at which to start yielding samples (inclusive).
2075
- * @param endTimestamp - The timestamp in seconds at which to stop yielding samples (exclusive).
2076
- */
2077
- samples(startTimestamp = 0, endTimestamp = Infinity) {
2078
- return this.mediaSamplesInRange(startTimestamp, endTimestamp);
2079
- }
2080
-
2081
- /**
2082
- * Creates an async iterator that yields an audio sample for each timestamp in the argument. This method
2083
- * uses an optimized decoding pipeline if these timestamps are monotonically sorted, decoding each packet at most
2084
- * once, and is therefore more efficient than manually getting the sample for every timestamp. The iterator may
2085
- * yield null if no sample is available for a given timestamp.
2086
- *
2087
- * @param timestamps - An iterable or async iterable of timestamps in seconds.
2088
- */
2089
- samplesAtTimestamps(timestamps: AnyIterable<number>) {
2090
- return this.mediaSamplesAtTimestamps(timestamps);
2091
- }
2092
- }
2093
-
2094
- /**
2095
- * An AudioBuffer with additional timing information (timestamp & duration).
2096
- * @group Media sinks
2097
- * @public
2098
- */
2099
- export type WrappedAudioBuffer = {
2100
- /** An AudioBuffer. */
2101
- buffer: AudioBuffer;
2102
- /** The timestamp of the corresponding audio sample, in seconds. */
2103
- timestamp: number;
2104
- /** The duration of the corresponding audio sample, in seconds. */
2105
- duration: number;
2106
- };
2107
-
2108
- /**
2109
- * A sink that retrieves decoded audio samples from an audio track and converts them to `AudioBuffer` instances. This is
2110
- * often more useful than directly retrieving audio samples, as audio buffers can be directly used with the
2111
- * Web Audio API.
2112
- * @group Media sinks
2113
- * @public
2114
- */
2115
- export class AudioBufferSink {
2116
- /** @internal */
2117
- _audioSampleSink: AudioSampleSink;
2118
-
2119
- /** Creates a new {@link AudioBufferSink} for the given {@link InputAudioTrack}. */
2120
- constructor(audioTrack: InputAudioTrack) {
2121
- if (!(audioTrack instanceof InputAudioTrack)) {
2122
- throw new TypeError('audioTrack must be an InputAudioTrack.');
2123
- }
2124
-
2125
- this._audioSampleSink = new AudioSampleSink(audioTrack);
2126
- }
2127
-
2128
- /** @internal */
2129
- _audioSampleToWrappedArrayBuffer(sample: AudioSample): WrappedAudioBuffer {
2130
- return {
2131
- buffer: sample.toAudioBuffer(),
2132
- timestamp: sample.timestamp,
2133
- duration: sample.duration,
2134
- };
2135
- }
2136
-
2137
- /**
2138
- * Retrieves the audio buffer corresponding to the given timestamp, in seconds. More specifically, returns
2139
- * the last audio buffer (in presentation order) with a start timestamp less than or equal to the given timestamp.
2140
- * Returns null if the timestamp is before the track's first timestamp.
2141
- *
2142
- * @param timestamp - The timestamp used for retrieval, in seconds.
2143
- */
2144
- async getBuffer(timestamp: number) {
2145
- validateTimestamp(timestamp);
2146
-
2147
- const data = await this._audioSampleSink.getSample(timestamp);
2148
- return data && this._audioSampleToWrappedArrayBuffer(data);
2149
- }
2150
-
2151
- /**
2152
- * Creates an async iterator that yields audio buffers of this track in presentation order. This method
2153
- * will intelligently pre-decode a few buffers ahead to enable fast iteration.
2154
- *
2155
- * @param startTimestamp - The timestamp in seconds at which to start yielding buffers (inclusive).
2156
- * @param endTimestamp - The timestamp in seconds at which to stop yielding buffers (exclusive).
2157
- */
2158
- buffers(startTimestamp = 0, endTimestamp = Infinity) {
2159
- return mapAsyncGenerator(
2160
- this._audioSampleSink.samples(startTimestamp, endTimestamp),
2161
- data => this._audioSampleToWrappedArrayBuffer(data),
2162
- );
2163
- }
2164
-
2165
- /**
2166
- * Creates an async iterator that yields an audio buffer for each timestamp in the argument. This method
2167
- * uses an optimized decoding pipeline if these timestamps are monotonically sorted, decoding each packet at most
2168
- * once, and is therefore more efficient than manually getting the buffer for every timestamp. The iterator may
2169
- * yield null if no buffer is available for a given timestamp.
2170
- *
2171
- * @param timestamps - An iterable or async iterable of timestamps in seconds.
2172
- */
2173
- buffersAtTimestamps(timestamps: AnyIterable<number>) {
2174
- return mapAsyncGenerator(
2175
- this._audioSampleSink.samplesAtTimestamps(timestamps),
2176
- data => data && this._audioSampleToWrappedArrayBuffer(data),
2177
- );
2178
- }
2179
- }