@kenzuya/mediabunny 1.26.0 → 1.28.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. package/README.md +1 -1
  2. package/dist/bundles/{mediabunny.mjs → mediabunny.js} +21963 -21388
  3. package/dist/bundles/mediabunny.min.js +490 -0
  4. package/dist/modules/shared/mp3-misc.d.ts.map +1 -1
  5. package/dist/modules/src/adts/adts-demuxer.d.ts +6 -6
  6. package/dist/modules/src/adts/adts-demuxer.d.ts.map +1 -1
  7. package/dist/modules/src/adts/adts-muxer.d.ts +4 -4
  8. package/dist/modules/src/adts/adts-muxer.d.ts.map +1 -1
  9. package/dist/modules/src/adts/adts-reader.d.ts +1 -1
  10. package/dist/modules/src/adts/adts-reader.d.ts.map +1 -1
  11. package/dist/modules/src/avi/avi-demuxer.d.ts +44 -0
  12. package/dist/modules/src/avi/avi-demuxer.d.ts.map +1 -0
  13. package/dist/modules/src/avi/avi-misc.d.ts +88 -0
  14. package/dist/modules/src/avi/avi-misc.d.ts.map +1 -0
  15. package/dist/modules/src/avi/avi-muxer.d.ts +45 -0
  16. package/dist/modules/src/avi/avi-muxer.d.ts.map +1 -0
  17. package/dist/modules/src/avi/riff-writer.d.ts +26 -0
  18. package/dist/modules/src/avi/riff-writer.d.ts.map +1 -0
  19. package/dist/modules/src/codec-data.d.ts +8 -3
  20. package/dist/modules/src/codec-data.d.ts.map +1 -1
  21. package/dist/modules/src/codec.d.ts +10 -10
  22. package/dist/modules/src/codec.d.ts.map +1 -1
  23. package/dist/modules/src/conversion.d.ts +33 -16
  24. package/dist/modules/src/conversion.d.ts.map +1 -1
  25. package/dist/modules/src/custom-coder.d.ts +8 -8
  26. package/dist/modules/src/custom-coder.d.ts.map +1 -1
  27. package/dist/modules/src/demuxer.d.ts +3 -3
  28. package/dist/modules/src/demuxer.d.ts.map +1 -1
  29. package/dist/modules/src/encode.d.ts +8 -8
  30. package/dist/modules/src/encode.d.ts.map +1 -1
  31. package/dist/modules/src/flac/flac-demuxer.d.ts +7 -7
  32. package/dist/modules/src/flac/flac-demuxer.d.ts.map +1 -1
  33. package/dist/modules/src/flac/flac-misc.d.ts +3 -3
  34. package/dist/modules/src/flac/flac-misc.d.ts.map +1 -1
  35. package/dist/modules/src/flac/flac-muxer.d.ts +5 -5
  36. package/dist/modules/src/flac/flac-muxer.d.ts.map +1 -1
  37. package/dist/modules/src/id3.d.ts +3 -3
  38. package/dist/modules/src/id3.d.ts.map +1 -1
  39. package/dist/modules/src/index.d.ts +20 -20
  40. package/dist/modules/src/index.d.ts.map +1 -1
  41. package/dist/modules/src/input-format.d.ts +22 -0
  42. package/dist/modules/src/input-format.d.ts.map +1 -1
  43. package/dist/modules/src/input-track.d.ts +8 -8
  44. package/dist/modules/src/input-track.d.ts.map +1 -1
  45. package/dist/modules/src/input.d.ts +12 -12
  46. package/dist/modules/src/isobmff/isobmff-boxes.d.ts +2 -2
  47. package/dist/modules/src/isobmff/isobmff-boxes.d.ts.map +1 -1
  48. package/dist/modules/src/isobmff/isobmff-demuxer.d.ts +12 -12
  49. package/dist/modules/src/isobmff/isobmff-demuxer.d.ts.map +1 -1
  50. package/dist/modules/src/isobmff/isobmff-misc.d.ts.map +1 -1
  51. package/dist/modules/src/isobmff/isobmff-muxer.d.ts +11 -11
  52. package/dist/modules/src/isobmff/isobmff-muxer.d.ts.map +1 -1
  53. package/dist/modules/src/isobmff/isobmff-reader.d.ts +2 -2
  54. package/dist/modules/src/isobmff/isobmff-reader.d.ts.map +1 -1
  55. package/dist/modules/src/matroska/ebml.d.ts +3 -3
  56. package/dist/modules/src/matroska/ebml.d.ts.map +1 -1
  57. package/dist/modules/src/matroska/matroska-demuxer.d.ts +13 -13
  58. package/dist/modules/src/matroska/matroska-demuxer.d.ts.map +1 -1
  59. package/dist/modules/src/matroska/matroska-input.d.ts +33 -0
  60. package/dist/modules/src/matroska/matroska-input.d.ts.map +1 -0
  61. package/dist/modules/src/matroska/matroska-misc.d.ts.map +1 -1
  62. package/dist/modules/src/matroska/matroska-muxer.d.ts +5 -5
  63. package/dist/modules/src/matroska/matroska-muxer.d.ts.map +1 -1
  64. package/dist/modules/src/media-sink.d.ts +5 -5
  65. package/dist/modules/src/media-sink.d.ts.map +1 -1
  66. package/dist/modules/src/media-source.d.ts +22 -4
  67. package/dist/modules/src/media-source.d.ts.map +1 -1
  68. package/dist/modules/src/metadata.d.ts +2 -2
  69. package/dist/modules/src/metadata.d.ts.map +1 -1
  70. package/dist/modules/src/misc.d.ts +5 -4
  71. package/dist/modules/src/misc.d.ts.map +1 -1
  72. package/dist/modules/src/mp3/mp3-demuxer.d.ts +7 -7
  73. package/dist/modules/src/mp3/mp3-demuxer.d.ts.map +1 -1
  74. package/dist/modules/src/mp3/mp3-muxer.d.ts +4 -4
  75. package/dist/modules/src/mp3/mp3-muxer.d.ts.map +1 -1
  76. package/dist/modules/src/mp3/mp3-reader.d.ts +2 -2
  77. package/dist/modules/src/mp3/mp3-reader.d.ts.map +1 -1
  78. package/dist/modules/src/mp3/mp3-writer.d.ts +1 -1
  79. package/dist/modules/src/mp3/mp3-writer.d.ts.map +1 -1
  80. package/dist/modules/src/muxer.d.ts +4 -4
  81. package/dist/modules/src/muxer.d.ts.map +1 -1
  82. package/dist/modules/src/node.d.ts +1 -1
  83. package/dist/modules/src/ogg/ogg-demuxer.d.ts +7 -7
  84. package/dist/modules/src/ogg/ogg-demuxer.d.ts.map +1 -1
  85. package/dist/modules/src/ogg/ogg-misc.d.ts +1 -1
  86. package/dist/modules/src/ogg/ogg-misc.d.ts.map +1 -1
  87. package/dist/modules/src/ogg/ogg-muxer.d.ts +5 -5
  88. package/dist/modules/src/ogg/ogg-muxer.d.ts.map +1 -1
  89. package/dist/modules/src/ogg/ogg-reader.d.ts +1 -1
  90. package/dist/modules/src/ogg/ogg-reader.d.ts.map +1 -1
  91. package/dist/modules/src/output-format.d.ts +51 -6
  92. package/dist/modules/src/output-format.d.ts.map +1 -1
  93. package/dist/modules/src/output.d.ts +13 -13
  94. package/dist/modules/src/output.d.ts.map +1 -1
  95. package/dist/modules/src/packet.d.ts +1 -1
  96. package/dist/modules/src/packet.d.ts.map +1 -1
  97. package/dist/modules/src/pcm.d.ts.map +1 -1
  98. package/dist/modules/src/reader.d.ts +2 -2
  99. package/dist/modules/src/reader.d.ts.map +1 -1
  100. package/dist/modules/src/sample.d.ts +57 -15
  101. package/dist/modules/src/sample.d.ts.map +1 -1
  102. package/dist/modules/src/source.d.ts +3 -3
  103. package/dist/modules/src/source.d.ts.map +1 -1
  104. package/dist/modules/src/subtitles.d.ts +1 -1
  105. package/dist/modules/src/subtitles.d.ts.map +1 -1
  106. package/dist/modules/src/target.d.ts +2 -2
  107. package/dist/modules/src/target.d.ts.map +1 -1
  108. package/dist/modules/src/tsconfig.tsbuildinfo +1 -1
  109. package/dist/modules/src/wave/riff-writer.d.ts +1 -1
  110. package/dist/modules/src/wave/riff-writer.d.ts.map +1 -1
  111. package/dist/modules/src/wave/wave-demuxer.d.ts +6 -6
  112. package/dist/modules/src/wave/wave-demuxer.d.ts.map +1 -1
  113. package/dist/modules/src/wave/wave-muxer.d.ts +4 -4
  114. package/dist/modules/src/wave/wave-muxer.d.ts.map +1 -1
  115. package/dist/modules/src/writer.d.ts +1 -1
  116. package/dist/modules/src/writer.d.ts.map +1 -1
  117. package/dist/packages/eac3/eac3.wasm +0 -0
  118. package/dist/packages/eac3/mediabunny-eac3.js +1058 -0
  119. package/dist/packages/eac3/mediabunny-eac3.min.js +44 -0
  120. package/dist/packages/mp3-encoder/mediabunny-mp3-encoder.js +694 -0
  121. package/dist/packages/mp3-encoder/mediabunny-mp3-encoder.min.js +58 -0
  122. package/dist/packages/mpeg4/mediabunny-mpeg4.js +1198 -0
  123. package/dist/packages/mpeg4/mediabunny-mpeg4.min.js +44 -0
  124. package/dist/packages/mpeg4/xvid.wasm +0 -0
  125. package/package.json +18 -57
  126. package/dist/bundles/mediabunny.cjs +0 -26140
  127. package/dist/bundles/mediabunny.min.cjs +0 -147
  128. package/dist/bundles/mediabunny.min.mjs +0 -146
  129. package/dist/mediabunny.d.ts +0 -3319
  130. package/dist/modules/shared/mp3-misc.js +0 -147
  131. package/dist/modules/src/adts/adts-demuxer.js +0 -239
  132. package/dist/modules/src/adts/adts-muxer.js +0 -80
  133. package/dist/modules/src/adts/adts-reader.js +0 -63
  134. package/dist/modules/src/codec-data.js +0 -1730
  135. package/dist/modules/src/codec.js +0 -869
  136. package/dist/modules/src/conversion.js +0 -1459
  137. package/dist/modules/src/custom-coder.js +0 -117
  138. package/dist/modules/src/demuxer.js +0 -12
  139. package/dist/modules/src/encode.js +0 -442
  140. package/dist/modules/src/flac/flac-demuxer.js +0 -504
  141. package/dist/modules/src/flac/flac-misc.js +0 -135
  142. package/dist/modules/src/flac/flac-muxer.js +0 -222
  143. package/dist/modules/src/id3.js +0 -848
  144. package/dist/modules/src/index.js +0 -28
  145. package/dist/modules/src/input-format.js +0 -480
  146. package/dist/modules/src/input-track.js +0 -372
  147. package/dist/modules/src/input.js +0 -188
  148. package/dist/modules/src/isobmff/isobmff-boxes.js +0 -1480
  149. package/dist/modules/src/isobmff/isobmff-demuxer.js +0 -2618
  150. package/dist/modules/src/isobmff/isobmff-misc.js +0 -20
  151. package/dist/modules/src/isobmff/isobmff-muxer.js +0 -966
  152. package/dist/modules/src/isobmff/isobmff-reader.js +0 -72
  153. package/dist/modules/src/matroska/ebml.js +0 -653
  154. package/dist/modules/src/matroska/matroska-demuxer.js +0 -2133
  155. package/dist/modules/src/matroska/matroska-misc.js +0 -20
  156. package/dist/modules/src/matroska/matroska-muxer.js +0 -1017
  157. package/dist/modules/src/media-sink.js +0 -1736
  158. package/dist/modules/src/media-source.js +0 -1825
  159. package/dist/modules/src/metadata.js +0 -193
  160. package/dist/modules/src/misc.js +0 -623
  161. package/dist/modules/src/mp3/mp3-demuxer.js +0 -285
  162. package/dist/modules/src/mp3/mp3-muxer.js +0 -123
  163. package/dist/modules/src/mp3/mp3-reader.js +0 -26
  164. package/dist/modules/src/mp3/mp3-writer.js +0 -78
  165. package/dist/modules/src/muxer.js +0 -50
  166. package/dist/modules/src/node.js +0 -9
  167. package/dist/modules/src/ogg/ogg-demuxer.js +0 -763
  168. package/dist/modules/src/ogg/ogg-misc.js +0 -78
  169. package/dist/modules/src/ogg/ogg-muxer.js +0 -353
  170. package/dist/modules/src/ogg/ogg-reader.js +0 -65
  171. package/dist/modules/src/output-format.js +0 -527
  172. package/dist/modules/src/output.js +0 -300
  173. package/dist/modules/src/packet.js +0 -182
  174. package/dist/modules/src/pcm.js +0 -85
  175. package/dist/modules/src/reader.js +0 -236
  176. package/dist/modules/src/sample.js +0 -1056
  177. package/dist/modules/src/source.js +0 -1182
  178. package/dist/modules/src/subtitles.js +0 -575
  179. package/dist/modules/src/target.js +0 -140
  180. package/dist/modules/src/wave/riff-writer.js +0 -30
  181. package/dist/modules/src/wave/wave-demuxer.js +0 -447
  182. package/dist/modules/src/wave/wave-muxer.js +0 -318
  183. package/dist/modules/src/writer.js +0 -370
  184. package/src/adts/adts-demuxer.ts +0 -331
  185. package/src/adts/adts-muxer.ts +0 -111
  186. package/src/adts/adts-reader.ts +0 -85
  187. package/src/codec-data.ts +0 -2078
  188. package/src/codec.ts +0 -1092
  189. package/src/conversion.ts +0 -2112
  190. package/src/custom-coder.ts +0 -197
  191. package/src/demuxer.ts +0 -24
  192. package/src/encode.ts +0 -739
  193. package/src/flac/flac-demuxer.ts +0 -730
  194. package/src/flac/flac-misc.ts +0 -164
  195. package/src/flac/flac-muxer.ts +0 -320
  196. package/src/id3.ts +0 -925
  197. package/src/index.ts +0 -221
  198. package/src/input-format.ts +0 -541
  199. package/src/input-track.ts +0 -529
  200. package/src/input.ts +0 -235
  201. package/src/isobmff/isobmff-boxes.ts +0 -1719
  202. package/src/isobmff/isobmff-demuxer.ts +0 -3190
  203. package/src/isobmff/isobmff-misc.ts +0 -29
  204. package/src/isobmff/isobmff-muxer.ts +0 -1348
  205. package/src/isobmff/isobmff-reader.ts +0 -91
  206. package/src/matroska/ebml.ts +0 -730
  207. package/src/matroska/matroska-demuxer.ts +0 -2481
  208. package/src/matroska/matroska-misc.ts +0 -29
  209. package/src/matroska/matroska-muxer.ts +0 -1276
  210. package/src/media-sink.ts +0 -2179
  211. package/src/media-source.ts +0 -2243
  212. package/src/metadata.ts +0 -320
  213. package/src/misc.ts +0 -798
  214. package/src/mp3/mp3-demuxer.ts +0 -383
  215. package/src/mp3/mp3-muxer.ts +0 -166
  216. package/src/mp3/mp3-reader.ts +0 -34
  217. package/src/mp3/mp3-writer.ts +0 -120
  218. package/src/muxer.ts +0 -88
  219. package/src/node.ts +0 -11
  220. package/src/ogg/ogg-demuxer.ts +0 -1053
  221. package/src/ogg/ogg-misc.ts +0 -116
  222. package/src/ogg/ogg-muxer.ts +0 -497
  223. package/src/ogg/ogg-reader.ts +0 -93
  224. package/src/output-format.ts +0 -945
  225. package/src/output.ts +0 -488
  226. package/src/packet.ts +0 -263
  227. package/src/pcm.ts +0 -112
  228. package/src/reader.ts +0 -323
  229. package/src/sample.ts +0 -1461
  230. package/src/source.ts +0 -1688
  231. package/src/subtitles.ts +0 -711
  232. package/src/target.ts +0 -204
  233. package/src/tsconfig.json +0 -16
  234. package/src/wave/riff-writer.ts +0 -36
  235. package/src/wave/wave-demuxer.ts +0 -529
  236. package/src/wave/wave-muxer.ts +0 -371
  237. package/src/writer.ts +0 -490
@@ -1,1736 +0,0 @@
1
- /*!
2
- * Copyright (c) 2025-present, Vanilagy and contributors
3
- *
4
- * This Source Code Form is subject to the terms of the Mozilla Public
5
- * License, v. 2.0. If a copy of the MPL was not distributed with this
6
- * file, You can obtain one at https://mozilla.org/MPL/2.0/.
7
- */
8
- import { parsePcmCodec, PCM_AUDIO_CODECS } from './codec.js';
9
- import { deserializeAvcDecoderConfigurationRecord, determineVideoPacketType, extractHevcNalUnits, extractNalUnitTypeForHevc, HevcNalUnitType, parseAvcSps, } from './codec-data.js';
10
- import { customVideoDecoders, customAudioDecoders } from './custom-coder.js';
11
- import { InputDisposedError } from './input.js';
12
- import { InputAudioTrack, InputTrack, InputVideoTrack } from './input-track.js';
13
- import { assert, assertNever, CallSerializer, getInt24, getUint24, insertSorted, isChromium, isFirefox, isNumber, isWebKit, last, mapAsyncGenerator, promiseWithResolvers, toAsyncIterator, toDataView, toUint8Array, validateAnyIterable, } from './misc.js';
14
- import { EncodedPacket } from './packet.js';
15
- import { fromAlaw, fromUlaw } from './pcm.js';
16
- import { AudioSample, clampCropRectangle, validateCropRectangle, VideoSample } from './sample.js';
17
- const validatePacketRetrievalOptions = (options) => {
18
- if (!options || typeof options !== 'object') {
19
- throw new TypeError('options must be an object.');
20
- }
21
- if (options.metadataOnly !== undefined && typeof options.metadataOnly !== 'boolean') {
22
- throw new TypeError('options.metadataOnly, when defined, must be a boolean.');
23
- }
24
- if (options.verifyKeyPackets !== undefined && typeof options.verifyKeyPackets !== 'boolean') {
25
- throw new TypeError('options.verifyKeyPackets, when defined, must be a boolean.');
26
- }
27
- if (options.verifyKeyPackets && options.metadataOnly) {
28
- throw new TypeError('options.verifyKeyPackets and options.metadataOnly cannot be enabled together.');
29
- }
30
- };
31
- const validateTimestamp = (timestamp) => {
32
- if (!isNumber(timestamp)) {
33
- throw new TypeError('timestamp must be a number.'); // It can be non-finite, that's fine
34
- }
35
- };
36
- const maybeFixPacketType = (track, promise, options) => {
37
- if (options.verifyKeyPackets) {
38
- return promise.then(async (packet) => {
39
- if (!packet || packet.type === 'delta') {
40
- return packet;
41
- }
42
- const determinedType = await track.determinePacketType(packet);
43
- if (determinedType) {
44
- // @ts-expect-error Technically readonly
45
- packet.type = determinedType;
46
- }
47
- return packet;
48
- });
49
- }
50
- else {
51
- return promise;
52
- }
53
- };
54
- /**
55
- * Sink for retrieving encoded packets from an input track.
56
- * @group Media sinks
57
- * @public
58
- */
59
- export class EncodedPacketSink {
60
- /** Creates a new {@link EncodedPacketSink} for the given {@link InputTrack}. */
61
- constructor(track) {
62
- if (!(track instanceof InputTrack)) {
63
- throw new TypeError('track must be an InputTrack.');
64
- }
65
- this._track = track;
66
- }
67
- /**
68
- * Retrieves the track's first packet (in decode order), or null if it has no packets. The first packet is very
69
- * likely to be a key packet.
70
- */
71
- getFirstPacket(options = {}) {
72
- validatePacketRetrievalOptions(options);
73
- if (this._track.input._disposed) {
74
- throw new InputDisposedError();
75
- }
76
- return maybeFixPacketType(this._track, this._track._backing.getFirstPacket(options), options);
77
- }
78
- /**
79
- * Retrieves the packet corresponding to the given timestamp, in seconds. More specifically, returns the last packet
80
- * (in presentation order) with a start timestamp less than or equal to the given timestamp. This method can be
81
- * used to retrieve a track's last packet using `getPacket(Infinity)`. The method returns null if the timestamp
82
- * is before the first packet in the track.
83
- *
84
- * @param timestamp - The timestamp used for retrieval, in seconds.
85
- */
86
- getPacket(timestamp, options = {}) {
87
- validateTimestamp(timestamp);
88
- validatePacketRetrievalOptions(options);
89
- if (this._track.input._disposed) {
90
- throw new InputDisposedError();
91
- }
92
- return maybeFixPacketType(this._track, this._track._backing.getPacket(timestamp, options), options);
93
- }
94
- /**
95
- * Retrieves the packet following the given packet (in decode order), or null if the given packet is the
96
- * last packet.
97
- */
98
- getNextPacket(packet, options = {}) {
99
- if (!(packet instanceof EncodedPacket)) {
100
- throw new TypeError('packet must be an EncodedPacket.');
101
- }
102
- validatePacketRetrievalOptions(options);
103
- if (this._track.input._disposed) {
104
- throw new InputDisposedError();
105
- }
106
- return maybeFixPacketType(this._track, this._track._backing.getNextPacket(packet, options), options);
107
- }
108
- /**
109
- * Retrieves the key packet corresponding to the given timestamp, in seconds. More specifically, returns the last
110
- * key packet (in presentation order) with a start timestamp less than or equal to the given timestamp. A key packet
111
- * is a packet that doesn't require previous packets to be decoded. This method can be used to retrieve a track's
112
- * last key packet using `getKeyPacket(Infinity)`. The method returns null if the timestamp is before the first
113
- * key packet in the track.
114
- *
115
- * To ensure that the returned packet is guaranteed to be a real key frame, enable `options.verifyKeyPackets`.
116
- *
117
- * @param timestamp - The timestamp used for retrieval, in seconds.
118
- */
119
- async getKeyPacket(timestamp, options = {}) {
120
- validateTimestamp(timestamp);
121
- validatePacketRetrievalOptions(options);
122
- if (this._track.input._disposed) {
123
- throw new InputDisposedError();
124
- }
125
- if (!options.verifyKeyPackets) {
126
- return this._track._backing.getKeyPacket(timestamp, options);
127
- }
128
- const packet = await this._track._backing.getKeyPacket(timestamp, options);
129
- if (!packet || packet.type === 'delta') {
130
- return packet;
131
- }
132
- const determinedType = await this._track.determinePacketType(packet);
133
- if (determinedType === 'delta') {
134
- // Try returning the previous key packet (in hopes that it's actually a key packet)
135
- return this.getKeyPacket(packet.timestamp - 1 / this._track.timeResolution, options);
136
- }
137
- return packet;
138
- }
139
- /**
140
- * Retrieves the key packet following the given packet (in decode order), or null if the given packet is the last
141
- * key packet.
142
- *
143
- * To ensure that the returned packet is guaranteed to be a real key frame, enable `options.verifyKeyPackets`.
144
- */
145
- async getNextKeyPacket(packet, options = {}) {
146
- if (!(packet instanceof EncodedPacket)) {
147
- throw new TypeError('packet must be an EncodedPacket.');
148
- }
149
- validatePacketRetrievalOptions(options);
150
- if (this._track.input._disposed) {
151
- throw new InputDisposedError();
152
- }
153
- if (!options.verifyKeyPackets) {
154
- return this._track._backing.getNextKeyPacket(packet, options);
155
- }
156
- const nextPacket = await this._track._backing.getNextKeyPacket(packet, options);
157
- if (!nextPacket || nextPacket.type === 'delta') {
158
- return nextPacket;
159
- }
160
- const determinedType = await this._track.determinePacketType(nextPacket);
161
- if (determinedType === 'delta') {
162
- // Try returning the next key packet (in hopes that it's actually a key packet)
163
- return this.getNextKeyPacket(nextPacket, options);
164
- }
165
- return nextPacket;
166
- }
167
- /**
168
- * Creates an async iterator that yields the packets in this track in decode order. To enable fast iteration, this
169
- * method will intelligently preload packets based on the speed of the consumer.
170
- *
171
- * @param startPacket - (optional) The packet from which iteration should begin. This packet will also be yielded.
172
- * @param endTimestamp - (optional) The timestamp at which iteration should end. This packet will _not_ be yielded.
173
- */
174
- packets(startPacket, endPacket, options = {}) {
175
- if (startPacket !== undefined && !(startPacket instanceof EncodedPacket)) {
176
- throw new TypeError('startPacket must be an EncodedPacket.');
177
- }
178
- if (startPacket !== undefined && startPacket.isMetadataOnly && !options?.metadataOnly) {
179
- throw new TypeError('startPacket can only be metadata-only if options.metadataOnly is enabled.');
180
- }
181
- if (endPacket !== undefined && !(endPacket instanceof EncodedPacket)) {
182
- throw new TypeError('endPacket must be an EncodedPacket.');
183
- }
184
- validatePacketRetrievalOptions(options);
185
- if (this._track.input._disposed) {
186
- throw new InputDisposedError();
187
- }
188
- const packetQueue = [];
189
- let { promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers();
190
- let { promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers();
191
- let ended = false;
192
- let terminated = false;
193
- // This stores errors that are "out of band" in the sense that they didn't occur in the normal flow of this
194
- // method but instead in a different context. This error should not go unnoticed and must be bubbled up to
195
- // the consumer.
196
- let outOfBandError = null;
197
- const timestamps = [];
198
- // The queue should always be big enough to hold 1 second worth of packets
199
- const maxQueueSize = () => Math.max(2, timestamps.length);
200
- // The following is the "pump" process that keeps pumping packets into the queue
201
- (async () => {
202
- let packet = startPacket ?? await this.getFirstPacket(options);
203
- while (packet && !terminated && !this._track.input._disposed) {
204
- if (endPacket && packet.sequenceNumber >= endPacket?.sequenceNumber) {
205
- break;
206
- }
207
- if (packetQueue.length > maxQueueSize()) {
208
- ({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers());
209
- await queueDequeue;
210
- continue;
211
- }
212
- packetQueue.push(packet);
213
- onQueueNotEmpty();
214
- ({ promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers());
215
- packet = await this.getNextPacket(packet, options);
216
- }
217
- ended = true;
218
- onQueueNotEmpty();
219
- })().catch((error) => {
220
- if (!outOfBandError) {
221
- outOfBandError = error;
222
- onQueueNotEmpty();
223
- }
224
- });
225
- const track = this._track;
226
- return {
227
- async next() {
228
- while (true) {
229
- if (track.input._disposed) {
230
- throw new InputDisposedError();
231
- }
232
- else if (terminated) {
233
- return { value: undefined, done: true };
234
- }
235
- else if (outOfBandError) {
236
- throw outOfBandError;
237
- }
238
- else if (packetQueue.length > 0) {
239
- const value = packetQueue.shift();
240
- const now = performance.now();
241
- timestamps.push(now);
242
- while (timestamps.length > 0 && now - timestamps[0] >= 1000) {
243
- timestamps.shift();
244
- }
245
- onQueueDequeue();
246
- return { value, done: false };
247
- }
248
- else if (ended) {
249
- return { value: undefined, done: true };
250
- }
251
- else {
252
- await queueNotEmpty;
253
- }
254
- }
255
- },
256
- async return() {
257
- terminated = true;
258
- onQueueDequeue();
259
- onQueueNotEmpty();
260
- return { value: undefined, done: true };
261
- },
262
- async throw(error) {
263
- throw error;
264
- },
265
- [Symbol.asyncIterator]() {
266
- return this;
267
- },
268
- };
269
- }
270
- }
271
- class DecoderWrapper {
272
- constructor(onSample, onError) {
273
- this.onSample = onSample;
274
- this.onError = onError;
275
- }
276
- }
277
- /**
278
- * Base class for decoded media sample sinks.
279
- * @group Media sinks
280
- * @public
281
- */
282
- export class BaseMediaSampleSink {
283
- /** @internal */
284
- mediaSamplesInRange(startTimestamp = 0, endTimestamp = Infinity) {
285
- validateTimestamp(startTimestamp);
286
- validateTimestamp(endTimestamp);
287
- const sampleQueue = [];
288
- let firstSampleQueued = false;
289
- let lastSample = null;
290
- let { promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers();
291
- let { promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers();
292
- let decoderIsFlushed = false;
293
- let ended = false;
294
- let terminated = false;
295
- // This stores errors that are "out of band" in the sense that they didn't occur in the normal flow of this
296
- // method but instead in a different context. This error should not go unnoticed and must be bubbled up to
297
- // the consumer.
298
- let outOfBandError = null;
299
- // The following is the "pump" process that keeps pumping packets into the decoder
300
- (async () => {
301
- const decoderError = new Error();
302
- const decoder = await this._createDecoder((sample) => {
303
- onQueueDequeue();
304
- if (sample.timestamp >= endTimestamp) {
305
- ended = true;
306
- }
307
- if (ended) {
308
- sample.close();
309
- return;
310
- }
311
- if (lastSample) {
312
- if (sample.timestamp > startTimestamp) {
313
- // We don't know ahead of time what the first first is. This is because the first first is the
314
- // last first whose timestamp is less than or equal to the start timestamp. Therefore we need to
315
- // wait for the first first after the start timestamp, and then we'll know that the previous
316
- // first was the first first.
317
- sampleQueue.push(lastSample);
318
- firstSampleQueued = true;
319
- }
320
- else {
321
- lastSample.close();
322
- }
323
- }
324
- if (sample.timestamp >= startTimestamp) {
325
- sampleQueue.push(sample);
326
- firstSampleQueued = true;
327
- }
328
- lastSample = firstSampleQueued ? null : sample;
329
- if (sampleQueue.length > 0) {
330
- onQueueNotEmpty();
331
- ({ promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers());
332
- }
333
- }, (error) => {
334
- if (!outOfBandError) {
335
- error.stack = decoderError.stack; // Provide a more useful stack trace
336
- outOfBandError = error;
337
- onQueueNotEmpty();
338
- }
339
- });
340
- const packetSink = this._createPacketSink();
341
- const keyPacket = await packetSink.getKeyPacket(startTimestamp, { verifyKeyPackets: true })
342
- ?? await packetSink.getFirstPacket();
343
- if (!keyPacket) {
344
- return;
345
- }
346
- let currentPacket = keyPacket;
347
- let endPacket = undefined;
348
- if (endTimestamp < Infinity) {
349
- // When an end timestamp is set, we cannot simply use that for the packet iterator due to out-of-order
350
- // frames (B-frames). Instead, we'll need to keep decoding packets until we get a frame that exceeds
351
- // this end time. However, we can still put a bound on it: Since key frames are by definition never
352
- // out of order, we can stop at the first key frame after the end timestamp.
353
- const packet = await packetSink.getPacket(endTimestamp);
354
- const keyPacket = !packet
355
- ? null
356
- : packet.type === 'key' && packet.timestamp === endTimestamp
357
- ? packet
358
- : await packetSink.getNextKeyPacket(packet, { verifyKeyPackets: true });
359
- if (keyPacket) {
360
- endPacket = keyPacket;
361
- }
362
- }
363
- const packets = packetSink.packets(keyPacket, endPacket);
364
- await packets.next(); // Skip the start packet as we already have it
365
- while (currentPacket && !ended && !this._track.input._disposed) {
366
- const maxQueueSize = computeMaxQueueSize(sampleQueue.length);
367
- if (sampleQueue.length + decoder.getDecodeQueueSize() > maxQueueSize) {
368
- ({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers());
369
- await queueDequeue;
370
- continue;
371
- }
372
- decoder.decode(currentPacket);
373
- const packetResult = await packets.next();
374
- if (packetResult.done) {
375
- break;
376
- }
377
- currentPacket = packetResult.value;
378
- }
379
- await packets.return();
380
- if (!terminated && !this._track.input._disposed) {
381
- await decoder.flush();
382
- }
383
- decoder.close();
384
- if (!firstSampleQueued && lastSample) {
385
- sampleQueue.push(lastSample);
386
- }
387
- decoderIsFlushed = true;
388
- onQueueNotEmpty(); // To unstuck the generator
389
- })().catch((error) => {
390
- if (!outOfBandError) {
391
- outOfBandError = error;
392
- onQueueNotEmpty();
393
- }
394
- });
395
- const track = this._track;
396
- const closeSamples = () => {
397
- lastSample?.close();
398
- for (const sample of sampleQueue) {
399
- sample.close();
400
- }
401
- };
402
- return {
403
- async next() {
404
- while (true) {
405
- if (track.input._disposed) {
406
- closeSamples();
407
- throw new InputDisposedError();
408
- }
409
- else if (terminated) {
410
- return { value: undefined, done: true };
411
- }
412
- else if (outOfBandError) {
413
- closeSamples();
414
- throw outOfBandError;
415
- }
416
- else if (sampleQueue.length > 0) {
417
- const value = sampleQueue.shift();
418
- onQueueDequeue();
419
- return { value, done: false };
420
- }
421
- else if (!decoderIsFlushed) {
422
- await queueNotEmpty;
423
- }
424
- else {
425
- return { value: undefined, done: true };
426
- }
427
- }
428
- },
429
- async return() {
430
- terminated = true;
431
- ended = true;
432
- onQueueDequeue();
433
- onQueueNotEmpty();
434
- closeSamples();
435
- return { value: undefined, done: true };
436
- },
437
- async throw(error) {
438
- throw error;
439
- },
440
- [Symbol.asyncIterator]() {
441
- return this;
442
- },
443
- };
444
- }
445
- /** @internal */
446
- mediaSamplesAtTimestamps(timestamps) {
447
- validateAnyIterable(timestamps);
448
- const timestampIterator = toAsyncIterator(timestamps);
449
- const timestampsOfInterest = [];
450
- const sampleQueue = [];
451
- let { promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers();
452
- let { promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers();
453
- let decoderIsFlushed = false;
454
- let terminated = false;
455
- // This stores errors that are "out of band" in the sense that they didn't occur in the normal flow of this
456
- // method but instead in a different context. This error should not go unnoticed and must be bubbled up to
457
- // the consumer.
458
- let outOfBandError = null;
459
- const pushToQueue = (sample) => {
460
- sampleQueue.push(sample);
461
- onQueueNotEmpty();
462
- ({ promise: queueNotEmpty, resolve: onQueueNotEmpty } = promiseWithResolvers());
463
- };
464
- // The following is the "pump" process that keeps pumping packets into the decoder
465
- (async () => {
466
- const decoderError = new Error();
467
- const decoder = await this._createDecoder((sample) => {
468
- onQueueDequeue();
469
- if (terminated) {
470
- sample.close();
471
- return;
472
- }
473
- let sampleUses = 0;
474
- while (timestampsOfInterest.length > 0
475
- && sample.timestamp - timestampsOfInterest[0] > -1e-10 // Give it a little epsilon
476
- ) {
477
- sampleUses++;
478
- timestampsOfInterest.shift();
479
- }
480
- if (sampleUses > 0) {
481
- for (let i = 0; i < sampleUses; i++) {
482
- // Clone the sample if we need to emit it multiple times
483
- pushToQueue((i < sampleUses - 1 ? sample.clone() : sample));
484
- }
485
- }
486
- else {
487
- sample.close();
488
- }
489
- }, (error) => {
490
- if (!outOfBandError) {
491
- error.stack = decoderError.stack; // Provide a more useful stack trace
492
- outOfBandError = error;
493
- onQueueNotEmpty();
494
- }
495
- });
496
- const packetSink = this._createPacketSink();
497
- let lastPacket = null;
498
- let lastKeyPacket = null;
499
- // The end sequence number (inclusive) in the next batch of packets that will be decoded. The batch starts
500
- // at the last key frame and goes until this sequence number.
501
- let maxSequenceNumber = -1;
502
- const decodePackets = async () => {
503
- assert(lastKeyPacket);
504
- // Start at the current key packet
505
- let currentPacket = lastKeyPacket;
506
- decoder.decode(currentPacket);
507
- while (currentPacket.sequenceNumber < maxSequenceNumber) {
508
- const maxQueueSize = computeMaxQueueSize(sampleQueue.length);
509
- while (sampleQueue.length + decoder.getDecodeQueueSize() > maxQueueSize && !terminated) {
510
- ({ promise: queueDequeue, resolve: onQueueDequeue } = promiseWithResolvers());
511
- await queueDequeue;
512
- }
513
- if (terminated) {
514
- break;
515
- }
516
- const nextPacket = await packetSink.getNextPacket(currentPacket);
517
- assert(nextPacket);
518
- decoder.decode(nextPacket);
519
- currentPacket = nextPacket;
520
- }
521
- maxSequenceNumber = -1;
522
- };
523
- const flushDecoder = async () => {
524
- await decoder.flush();
525
- // We don't expect this list to have any elements in it anymore, but in case it does, let's emit
526
- // nulls for every remaining element, then clear it.
527
- for (let i = 0; i < timestampsOfInterest.length; i++) {
528
- pushToQueue(null);
529
- }
530
- timestampsOfInterest.length = 0;
531
- };
532
- for await (const timestamp of timestampIterator) {
533
- validateTimestamp(timestamp);
534
- if (terminated || this._track.input._disposed) {
535
- break;
536
- }
537
- const targetPacket = await packetSink.getPacket(timestamp);
538
- const keyPacket = targetPacket && await packetSink.getKeyPacket(timestamp, { verifyKeyPackets: true });
539
- if (!keyPacket) {
540
- if (maxSequenceNumber !== -1) {
541
- await decodePackets();
542
- await flushDecoder();
543
- }
544
- pushToQueue(null);
545
- lastPacket = null;
546
- continue;
547
- }
548
- // Check if the key packet has changed or if we're going back in time
549
- if (lastPacket
550
- && (keyPacket.sequenceNumber !== lastKeyPacket.sequenceNumber
551
- || targetPacket.timestamp < lastPacket.timestamp)) {
552
- await decodePackets();
553
- await flushDecoder(); // Always flush here, improves decoder compatibility
554
- }
555
- timestampsOfInterest.push(targetPacket.timestamp);
556
- maxSequenceNumber = Math.max(targetPacket.sequenceNumber, maxSequenceNumber);
557
- lastPacket = targetPacket;
558
- lastKeyPacket = keyPacket;
559
- }
560
- if (!terminated && !this._track.input._disposed) {
561
- if (maxSequenceNumber !== -1) {
562
- // We still need to decode packets
563
- await decodePackets();
564
- }
565
- await flushDecoder();
566
- }
567
- decoder.close();
568
- decoderIsFlushed = true;
569
- onQueueNotEmpty(); // To unstuck the generator
570
- })().catch((error) => {
571
- if (!outOfBandError) {
572
- outOfBandError = error;
573
- onQueueNotEmpty();
574
- }
575
- });
576
- const track = this._track;
577
- const closeSamples = () => {
578
- for (const sample of sampleQueue) {
579
- sample?.close();
580
- }
581
- };
582
- return {
583
- async next() {
584
- while (true) {
585
- if (track.input._disposed) {
586
- closeSamples();
587
- throw new InputDisposedError();
588
- }
589
- else if (terminated) {
590
- return { value: undefined, done: true };
591
- }
592
- else if (outOfBandError) {
593
- closeSamples();
594
- throw outOfBandError;
595
- }
596
- else if (sampleQueue.length > 0) {
597
- const value = sampleQueue.shift();
598
- assert(value !== undefined);
599
- onQueueDequeue();
600
- return { value, done: false };
601
- }
602
- else if (!decoderIsFlushed) {
603
- await queueNotEmpty;
604
- }
605
- else {
606
- return { value: undefined, done: true };
607
- }
608
- }
609
- },
610
- async return() {
611
- terminated = true;
612
- onQueueDequeue();
613
- onQueueNotEmpty();
614
- closeSamples();
615
- return { value: undefined, done: true };
616
- },
617
- async throw(error) {
618
- throw error;
619
- },
620
- [Symbol.asyncIterator]() {
621
- return this;
622
- },
623
- };
624
- }
625
- }
626
- const computeMaxQueueSize = (decodedSampleQueueSize) => {
627
- // If we have decoded samples lying around, limit the total queue size to a small value (decoded samples can use up
628
- // a lot of memory). If not, we're fine with a much bigger queue of encoded packets waiting to be decoded. In fact,
629
- // some decoders only start flushing out decoded chunks when the packet queue is large enough.
630
- return decodedSampleQueueSize === 0 ? 40 : 8;
631
- };
632
- class VideoDecoderWrapper extends DecoderWrapper {
633
- constructor(onSample, onError, codec, decoderConfig, rotation, timeResolution) {
634
- super(onSample, onError);
635
- this.codec = codec;
636
- this.decoderConfig = decoderConfig;
637
- this.rotation = rotation;
638
- this.timeResolution = timeResolution;
639
- this.decoder = null;
640
- this.customDecoder = null;
641
- this.customDecoderCallSerializer = new CallSerializer();
642
- this.customDecoderQueueSize = 0;
643
- this.inputTimestamps = []; // Timestamps input into the decoder, sorted.
644
- this.sampleQueue = []; // Safari-specific thing, check usage.
645
- this.currentPacketIndex = 0;
646
- this.raslSkipped = false; // For HEVC stuff
647
- // Alpha stuff
648
- this.alphaDecoder = null;
649
- this.alphaHadKeyframe = false;
650
- this.colorQueue = [];
651
- this.alphaQueue = [];
652
- this.merger = null;
653
- this.mergerCreationFailed = false;
654
- this.decodedAlphaChunkCount = 0;
655
- this.alphaDecoderQueueSize = 0;
656
- /** Each value is the number of decoded alpha chunks at which a null alpha frame should be added. */
657
- this.nullAlphaFrameQueue = [];
658
- this.currentAlphaPacketIndex = 0;
659
- this.alphaRaslSkipped = false; // For HEVC stuff
660
- const MatchingCustomDecoder = customVideoDecoders.find(x => x.supports(codec, decoderConfig));
661
- if (MatchingCustomDecoder) {
662
- // @ts-expect-error "Can't create instance of abstract class 🤓"
663
- this.customDecoder = new MatchingCustomDecoder();
664
- // @ts-expect-error It's technically readonly
665
- this.customDecoder.codec = codec;
666
- // @ts-expect-error It's technically readonly
667
- this.customDecoder.config = decoderConfig;
668
- // @ts-expect-error It's technically readonly
669
- this.customDecoder.onSample = (sample) => {
670
- if (!(sample instanceof VideoSample)) {
671
- throw new TypeError('The argument passed to onSample must be a VideoSample.');
672
- }
673
- this.finalizeAndEmitSample(sample);
674
- };
675
- void this.customDecoderCallSerializer.call(() => this.customDecoder.init());
676
- }
677
- else {
678
- const colorHandler = (frame) => {
679
- if (this.alphaQueue.length > 0) {
680
- // Even when no alpha data is present (most of the time), there will be nulls in this queue
681
- const alphaFrame = this.alphaQueue.shift();
682
- assert(alphaFrame !== undefined);
683
- this.mergeAlpha(frame, alphaFrame);
684
- }
685
- else {
686
- this.colorQueue.push(frame);
687
- }
688
- };
689
- if (codec === 'avc' && this.decoderConfig.description && isChromium()) {
690
- // Chromium has/had a bug with playing interlaced AVC (https://issues.chromium.org/issues/456919096)
691
- // which can be worked around by requesting that software decoding be used. So, here we peek into the
692
- // AVC description, if present, and switch to software decoding if we find interlaced content.
693
- const record = deserializeAvcDecoderConfigurationRecord(toUint8Array(this.decoderConfig.description));
694
- if (record && record.sequenceParameterSets.length > 0) {
695
- const sps = parseAvcSps(record.sequenceParameterSets[0]);
696
- if (sps && sps.frameMbsOnlyFlag === 0) {
697
- this.decoderConfig = {
698
- ...this.decoderConfig,
699
- hardwareAcceleration: 'prefer-software',
700
- };
701
- }
702
- }
703
- }
704
- this.decoder = new VideoDecoder({
705
- output: (frame) => {
706
- try {
707
- colorHandler(frame);
708
- }
709
- catch (error) {
710
- this.onError(error);
711
- }
712
- },
713
- error: onError,
714
- });
715
- this.decoder.configure(this.decoderConfig);
716
- }
717
- }
718
- getDecodeQueueSize() {
719
- if (this.customDecoder) {
720
- return this.customDecoderQueueSize;
721
- }
722
- else {
723
- assert(this.decoder);
724
- return Math.max(this.decoder.decodeQueueSize, this.alphaDecoder?.decodeQueueSize ?? 0);
725
- }
726
- }
727
- decode(packet) {
728
- if (this.codec === 'hevc' && this.currentPacketIndex > 0 && !this.raslSkipped) {
729
- if (this.hasHevcRaslPicture(packet.data)) {
730
- return; // Drop
731
- }
732
- this.raslSkipped = true;
733
- }
734
- this.currentPacketIndex++;
735
- if (this.customDecoder) {
736
- this.customDecoderQueueSize++;
737
- void this.customDecoderCallSerializer
738
- .call(() => this.customDecoder.decode(packet))
739
- .then(() => this.customDecoderQueueSize--);
740
- }
741
- else {
742
- assert(this.decoder);
743
- if (!isWebKit()) {
744
- insertSorted(this.inputTimestamps, packet.timestamp, x => x);
745
- }
746
- this.decoder.decode(packet.toEncodedVideoChunk());
747
- this.decodeAlphaData(packet);
748
- }
749
- }
750
- decodeAlphaData(packet) {
751
- if (!packet.sideData.alpha || this.mergerCreationFailed) {
752
- // No alpha side data in the packet, most common case
753
- this.pushNullAlphaFrame();
754
- return;
755
- }
756
- if (!this.merger) {
757
- try {
758
- this.merger = new ColorAlphaMerger();
759
- }
760
- catch (error) {
761
- console.error('Due to an error, only color data will be decoded.', error);
762
- this.mergerCreationFailed = true;
763
- this.decodeAlphaData(packet); // Go again
764
- return;
765
- }
766
- }
767
- // Check if we need to set up the alpha decoder
768
- if (!this.alphaDecoder) {
769
- const alphaHandler = (frame) => {
770
- this.alphaDecoderQueueSize--;
771
- if (this.colorQueue.length > 0) {
772
- const colorFrame = this.colorQueue.shift();
773
- assert(colorFrame !== undefined);
774
- this.mergeAlpha(colorFrame, frame);
775
- }
776
- else {
777
- this.alphaQueue.push(frame);
778
- }
779
- // Check if any null frames have been queued for this point
780
- this.decodedAlphaChunkCount++;
781
- while (this.nullAlphaFrameQueue.length > 0
782
- && this.nullAlphaFrameQueue[0] === this.decodedAlphaChunkCount) {
783
- this.nullAlphaFrameQueue.shift();
784
- if (this.colorQueue.length > 0) {
785
- const colorFrame = this.colorQueue.shift();
786
- assert(colorFrame !== undefined);
787
- this.mergeAlpha(colorFrame, null);
788
- }
789
- else {
790
- this.alphaQueue.push(null);
791
- }
792
- }
793
- };
794
- this.alphaDecoder = new VideoDecoder({
795
- output: (frame) => {
796
- try {
797
- alphaHandler(frame);
798
- }
799
- catch (error) {
800
- this.onError(error);
801
- }
802
- },
803
- error: this.onError,
804
- });
805
- this.alphaDecoder.configure(this.decoderConfig);
806
- }
807
- const type = determineVideoPacketType(this.codec, this.decoderConfig, packet.sideData.alpha);
808
- // Alpha packets might follow a different key frame rhythm than the main packets. Therefore, before we start
809
- // decoding, we must first find a packet that's actually a key frame. Until then, we treat the image as opaque.
810
- if (!this.alphaHadKeyframe) {
811
- this.alphaHadKeyframe = type === 'key';
812
- }
813
- if (this.alphaHadKeyframe) {
814
- // Same RASL skipping logic as for color, unlikely to be hit (since who uses HEVC with separate alpha??) but
815
- // here for symmetry.
816
- if (this.codec === 'hevc' && this.currentAlphaPacketIndex > 0 && !this.alphaRaslSkipped) {
817
- if (this.hasHevcRaslPicture(packet.sideData.alpha)) {
818
- this.pushNullAlphaFrame();
819
- return;
820
- }
821
- this.alphaRaslSkipped = true;
822
- }
823
- this.currentAlphaPacketIndex++;
824
- this.alphaDecoder.decode(packet.alphaToEncodedVideoChunk(type ?? packet.type));
825
- this.alphaDecoderQueueSize++;
826
- }
827
- else {
828
- this.pushNullAlphaFrame();
829
- }
830
- }
831
- pushNullAlphaFrame() {
832
- if (this.alphaDecoderQueueSize === 0) {
833
- // Easy
834
- this.alphaQueue.push(null);
835
- }
836
- else {
837
- // There are still alpha chunks being decoded, so pushing `null` immediately would result in out-of-order
838
- // data and be incorrect. Instead, we need to enqueue a "null frame" for when the current decoder workload
839
- // has finished.
840
- this.nullAlphaFrameQueue.push(this.decodedAlphaChunkCount + this.alphaDecoderQueueSize);
841
- }
842
- }
843
- /**
844
- * If we're using HEVC, we need to make sure to skip any RASL slices that follow a non-IDR key frame such as
845
- * CRA_NUT. This is because RASL slices cannot be decoded without data before the CRA_NUT. Browsers behave
846
- * differently here: Chromium drops the packets, Safari throws a decoder error. Either way, it's not good
847
- * and causes bugs upstream. So, let's take the dropping into our own hands.
848
- */
849
- hasHevcRaslPicture(packetData) {
850
- const nalUnits = extractHevcNalUnits(packetData, this.decoderConfig);
851
- return nalUnits.some((x) => {
852
- const type = extractNalUnitTypeForHevc(x);
853
- return type === HevcNalUnitType.RASL_N || type === HevcNalUnitType.RASL_R;
854
- });
855
- }
856
- /** Handler for the WebCodecs VideoDecoder for ironing out browser differences. */
857
- sampleHandler(sample) {
858
- if (isWebKit()) {
859
- // For correct B-frame handling, we don't just hand over the frames directly but instead add them to
860
- // a queue, because we want to ensure frames are emitted in presentation order. We flush the queue
861
- // each time we receive a frame with a timestamp larger than the highest we've seen so far, as we
862
- // can sure that is not a B-frame. Typically, WebCodecs automatically guarantees that frames are
863
- // emitted in presentation order, but Safari doesn't always follow this rule.
864
- if (this.sampleQueue.length > 0 && (sample.timestamp >= last(this.sampleQueue).timestamp)) {
865
- for (const sample of this.sampleQueue) {
866
- this.finalizeAndEmitSample(sample);
867
- }
868
- this.sampleQueue.length = 0;
869
- }
870
- insertSorted(this.sampleQueue, sample, x => x.timestamp);
871
- }
872
- else {
873
- // Assign it the next earliest timestamp from the input. We do this because browsers, by spec, are
874
- // required to emit decoded frames in presentation order *while* retaining the timestamp of their
875
- // originating EncodedVideoChunk. For files with B-frames but no out-of-order timestamps (like a
876
- // missing ctts box, for example), this causes a mismatch. We therefore fix the timestamps and
877
- // ensure they are sorted by doing this.
878
- const timestamp = this.inputTimestamps.shift();
879
- // There's no way we'd have more decoded frames than encoded packets we passed in. Actually, the
880
- // correspondence should be 1:1.
881
- assert(timestamp !== undefined);
882
- sample.setTimestamp(timestamp);
883
- this.finalizeAndEmitSample(sample);
884
- }
885
- }
886
- finalizeAndEmitSample(sample) {
887
- // Round the timestamps to the time resolution
888
- sample.setTimestamp(Math.round(sample.timestamp * this.timeResolution) / this.timeResolution);
889
- sample.setDuration(Math.round(sample.duration * this.timeResolution) / this.timeResolution);
890
- sample.setRotation(this.rotation);
891
- this.onSample(sample);
892
- }
893
- mergeAlpha(color, alpha) {
894
- if (!alpha) {
895
- // Nothing needs to be merged
896
- const finalSample = new VideoSample(color);
897
- this.sampleHandler(finalSample);
898
- return;
899
- }
900
- assert(this.merger);
901
- this.merger.update(color, alpha);
902
- color.close();
903
- alpha.close();
904
- const finalFrame = new VideoFrame(this.merger.canvas, {
905
- timestamp: color.timestamp,
906
- duration: color.duration ?? undefined,
907
- });
908
- const finalSample = new VideoSample(finalFrame);
909
- this.sampleHandler(finalSample);
910
- }
911
- async flush() {
912
- if (this.customDecoder) {
913
- await this.customDecoderCallSerializer.call(() => this.customDecoder.flush());
914
- }
915
- else {
916
- assert(this.decoder);
917
- await Promise.all([
918
- this.decoder.flush(),
919
- this.alphaDecoder?.flush(),
920
- ]);
921
- this.colorQueue.forEach(x => x.close());
922
- this.colorQueue.length = 0;
923
- this.alphaQueue.forEach(x => x?.close());
924
- this.alphaQueue.length = 0;
925
- this.alphaHadKeyframe = false;
926
- this.decodedAlphaChunkCount = 0;
927
- this.alphaDecoderQueueSize = 0;
928
- this.nullAlphaFrameQueue.length = 0;
929
- this.currentAlphaPacketIndex = 0;
930
- this.alphaRaslSkipped = false;
931
- }
932
- if (isWebKit()) {
933
- for (const sample of this.sampleQueue) {
934
- this.finalizeAndEmitSample(sample);
935
- }
936
- this.sampleQueue.length = 0;
937
- }
938
- this.currentPacketIndex = 0;
939
- this.raslSkipped = false;
940
- }
941
- close() {
942
- if (this.customDecoder) {
943
- void this.customDecoderCallSerializer.call(() => this.customDecoder.close());
944
- }
945
- else {
946
- assert(this.decoder);
947
- this.decoder.close();
948
- this.alphaDecoder?.close();
949
- this.colorQueue.forEach(x => x.close());
950
- this.colorQueue.length = 0;
951
- this.alphaQueue.forEach(x => x?.close());
952
- this.alphaQueue.length = 0;
953
- this.merger?.close();
954
- }
955
- for (const sample of this.sampleQueue) {
956
- sample.close();
957
- }
958
- this.sampleQueue.length = 0;
959
- }
960
- }
961
- /** Utility class that merges together color and alpha information using simple WebGL 2 shaders. */
962
- class ColorAlphaMerger {
963
- constructor() {
964
- // Canvas will be resized later
965
- if (typeof OffscreenCanvas !== 'undefined') {
966
- // Prefer OffscreenCanvas for Worker environments
967
- this.canvas = new OffscreenCanvas(300, 150);
968
- }
969
- else {
970
- this.canvas = document.createElement('canvas');
971
- }
972
- const gl = this.canvas.getContext('webgl2', {
973
- premultipliedAlpha: false,
974
- }); // Casting because of some TypeScript weirdness
975
- if (!gl) {
976
- throw new Error('Couldn\'t acquire WebGL 2 context.');
977
- }
978
- this.gl = gl;
979
- this.program = this.createProgram();
980
- this.vao = this.createVAO();
981
- this.colorTexture = this.createTexture();
982
- this.alphaTexture = this.createTexture();
983
- this.gl.useProgram(this.program);
984
- this.gl.uniform1i(this.gl.getUniformLocation(this.program, 'u_colorTexture'), 0);
985
- this.gl.uniform1i(this.gl.getUniformLocation(this.program, 'u_alphaTexture'), 1);
986
- }
987
- createProgram() {
988
- const vertexShader = this.createShader(this.gl.VERTEX_SHADER, `#version 300 es
989
- in vec2 a_position;
990
- in vec2 a_texCoord;
991
- out vec2 v_texCoord;
992
-
993
- void main() {
994
- gl_Position = vec4(a_position, 0.0, 1.0);
995
- v_texCoord = a_texCoord;
996
- }
997
- `);
998
- const fragmentShader = this.createShader(this.gl.FRAGMENT_SHADER, `#version 300 es
999
- precision highp float;
1000
-
1001
- uniform sampler2D u_colorTexture;
1002
- uniform sampler2D u_alphaTexture;
1003
- in vec2 v_texCoord;
1004
- out vec4 fragColor;
1005
-
1006
- void main() {
1007
- vec3 color = texture(u_colorTexture, v_texCoord).rgb;
1008
- float alpha = texture(u_alphaTexture, v_texCoord).r;
1009
- fragColor = vec4(color, alpha);
1010
- }
1011
- `);
1012
- const program = this.gl.createProgram();
1013
- this.gl.attachShader(program, vertexShader);
1014
- this.gl.attachShader(program, fragmentShader);
1015
- this.gl.linkProgram(program);
1016
- return program;
1017
- }
1018
- createShader(type, source) {
1019
- const shader = this.gl.createShader(type);
1020
- this.gl.shaderSource(shader, source);
1021
- this.gl.compileShader(shader);
1022
- return shader;
1023
- }
1024
- createVAO() {
1025
- const vao = this.gl.createVertexArray();
1026
- this.gl.bindVertexArray(vao);
1027
- const vertices = new Float32Array([
1028
- -1, -1, 0, 1,
1029
- 1, -1, 1, 1,
1030
- -1, 1, 0, 0,
1031
- 1, 1, 1, 0,
1032
- ]);
1033
- const buffer = this.gl.createBuffer();
1034
- this.gl.bindBuffer(this.gl.ARRAY_BUFFER, buffer);
1035
- this.gl.bufferData(this.gl.ARRAY_BUFFER, vertices, this.gl.STATIC_DRAW);
1036
- const positionLocation = this.gl.getAttribLocation(this.program, 'a_position');
1037
- const texCoordLocation = this.gl.getAttribLocation(this.program, 'a_texCoord');
1038
- this.gl.enableVertexAttribArray(positionLocation);
1039
- this.gl.vertexAttribPointer(positionLocation, 2, this.gl.FLOAT, false, 16, 0);
1040
- this.gl.enableVertexAttribArray(texCoordLocation);
1041
- this.gl.vertexAttribPointer(texCoordLocation, 2, this.gl.FLOAT, false, 16, 8);
1042
- return vao;
1043
- }
1044
- createTexture() {
1045
- const texture = this.gl.createTexture();
1046
- this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
1047
- this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
1048
- this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
1049
- this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR);
1050
- this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.LINEAR);
1051
- return texture;
1052
- }
1053
- update(color, alpha) {
1054
- if (color.displayWidth !== this.canvas.width || color.displayHeight !== this.canvas.height) {
1055
- this.canvas.width = color.displayWidth;
1056
- this.canvas.height = color.displayHeight;
1057
- }
1058
- this.gl.activeTexture(this.gl.TEXTURE0);
1059
- this.gl.bindTexture(this.gl.TEXTURE_2D, this.colorTexture);
1060
- this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, color);
1061
- this.gl.activeTexture(this.gl.TEXTURE1);
1062
- this.gl.bindTexture(this.gl.TEXTURE_2D, this.alphaTexture);
1063
- this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, alpha);
1064
- this.gl.viewport(0, 0, this.canvas.width, this.canvas.height);
1065
- this.gl.clear(this.gl.COLOR_BUFFER_BIT);
1066
- this.gl.bindVertexArray(this.vao);
1067
- this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4);
1068
- }
1069
- close() {
1070
- this.gl.getExtension('WEBGL_lose_context')?.loseContext();
1071
- this.gl = null;
1072
- }
1073
- }
1074
- /**
1075
- * A sink that retrieves decoded video samples (video frames) from a video track.
1076
- * @group Media sinks
1077
- * @public
1078
- */
1079
- export class VideoSampleSink extends BaseMediaSampleSink {
1080
- /** Creates a new {@link VideoSampleSink} for the given {@link InputVideoTrack}. */
1081
- constructor(videoTrack) {
1082
- if (!(videoTrack instanceof InputVideoTrack)) {
1083
- throw new TypeError('videoTrack must be an InputVideoTrack.');
1084
- }
1085
- super();
1086
- this._track = videoTrack;
1087
- }
1088
- /** @internal */
1089
- async _createDecoder(onSample, onError) {
1090
- if (!(await this._track.canDecode())) {
1091
- throw new Error('This video track cannot be decoded by this browser. Make sure to check decodability before using'
1092
- + ' a track.');
1093
- }
1094
- const codec = this._track.codec;
1095
- const rotation = this._track.rotation;
1096
- const decoderConfig = await this._track.getDecoderConfig();
1097
- const timeResolution = this._track.timeResolution;
1098
- assert(codec && decoderConfig);
1099
- return new VideoDecoderWrapper(onSample, onError, codec, decoderConfig, rotation, timeResolution);
1100
- }
1101
- /** @internal */
1102
- _createPacketSink() {
1103
- return new EncodedPacketSink(this._track);
1104
- }
1105
- /**
1106
- * Retrieves the video sample (frame) corresponding to the given timestamp, in seconds. More specifically, returns
1107
- * the last video sample (in presentation order) with a start timestamp less than or equal to the given timestamp.
1108
- * Returns null if the timestamp is before the track's first timestamp.
1109
- *
1110
- * @param timestamp - The timestamp used for retrieval, in seconds.
1111
- */
1112
- async getSample(timestamp) {
1113
- validateTimestamp(timestamp);
1114
- for await (const sample of this.mediaSamplesAtTimestamps([timestamp])) {
1115
- return sample;
1116
- }
1117
- throw new Error('Internal error: Iterator returned nothing.');
1118
- }
1119
- /**
1120
- * Creates an async iterator that yields the video samples (frames) of this track in presentation order. This method
1121
- * will intelligently pre-decode a few frames ahead to enable fast iteration.
1122
- *
1123
- * @param startTimestamp - The timestamp in seconds at which to start yielding samples (inclusive).
1124
- * @param endTimestamp - The timestamp in seconds at which to stop yielding samples (exclusive).
1125
- */
1126
- samples(startTimestamp = 0, endTimestamp = Infinity) {
1127
- return this.mediaSamplesInRange(startTimestamp, endTimestamp);
1128
- }
1129
- /**
1130
- * Creates an async iterator that yields a video sample (frame) for each timestamp in the argument. This method
1131
- * uses an optimized decoding pipeline if these timestamps are monotonically sorted, decoding each packet at most
1132
- * once, and is therefore more efficient than manually getting the sample for every timestamp. The iterator may
1133
- * yield null if no frame is available for a given timestamp.
1134
- *
1135
- * @param timestamps - An iterable or async iterable of timestamps in seconds.
1136
- */
1137
- samplesAtTimestamps(timestamps) {
1138
- return this.mediaSamplesAtTimestamps(timestamps);
1139
- }
1140
- }
1141
- /**
1142
- * A sink that renders video samples (frames) of the given video track to canvases. This is often more useful than
1143
- * directly retrieving frames, as it comes with common preprocessing steps such as resizing or applying rotation
1144
- * metadata.
1145
- *
1146
- * This sink will yield `HTMLCanvasElement`s when in a DOM context, and `OffscreenCanvas`es otherwise.
1147
- *
1148
- * @group Media sinks
1149
- * @public
1150
- */
1151
- export class CanvasSink {
1152
- /** Creates a new {@link CanvasSink} for the given {@link InputVideoTrack}. */
1153
- constructor(videoTrack, options = {}) {
1154
- /** @internal */
1155
- this._nextCanvasIndex = 0;
1156
- if (!(videoTrack instanceof InputVideoTrack)) {
1157
- throw new TypeError('videoTrack must be an InputVideoTrack.');
1158
- }
1159
- if (options && typeof options !== 'object') {
1160
- throw new TypeError('options must be an object.');
1161
- }
1162
- if (options.alpha !== undefined && typeof options.alpha !== 'boolean') {
1163
- throw new TypeError('options.alpha, when provided, must be a boolean.');
1164
- }
1165
- if (options.width !== undefined && (!Number.isInteger(options.width) || options.width <= 0)) {
1166
- throw new TypeError('options.width, when defined, must be a positive integer.');
1167
- }
1168
- if (options.height !== undefined && (!Number.isInteger(options.height) || options.height <= 0)) {
1169
- throw new TypeError('options.height, when defined, must be a positive integer.');
1170
- }
1171
- if (options.fit !== undefined && !['fill', 'contain', 'cover'].includes(options.fit)) {
1172
- throw new TypeError('options.fit, when provided, must be one of "fill", "contain", or "cover".');
1173
- }
1174
- if (options.width !== undefined
1175
- && options.height !== undefined
1176
- && options.fit === undefined) {
1177
- throw new TypeError('When both options.width and options.height are provided, options.fit must also be provided.');
1178
- }
1179
- if (options.rotation !== undefined && ![0, 90, 180, 270].includes(options.rotation)) {
1180
- throw new TypeError('options.rotation, when provided, must be 0, 90, 180 or 270.');
1181
- }
1182
- if (options.crop !== undefined) {
1183
- validateCropRectangle(options.crop, 'options.');
1184
- }
1185
- if (options.poolSize !== undefined
1186
- && (typeof options.poolSize !== 'number' || !Number.isInteger(options.poolSize) || options.poolSize < 0)) {
1187
- throw new TypeError('poolSize must be a non-negative integer.');
1188
- }
1189
- const rotation = options.rotation ?? videoTrack.rotation;
1190
- const [rotatedWidth, rotatedHeight] = rotation % 180 === 0
1191
- ? [videoTrack.codedWidth, videoTrack.codedHeight]
1192
- : [videoTrack.codedHeight, videoTrack.codedWidth];
1193
- const crop = options.crop;
1194
- if (crop) {
1195
- clampCropRectangle(crop, rotatedWidth, rotatedHeight);
1196
- }
1197
- let [width, height] = crop
1198
- ? [crop.width, crop.height]
1199
- : [rotatedWidth, rotatedHeight];
1200
- const originalAspectRatio = width / height;
1201
- // If width and height aren't defined together, deduce the missing value using the aspect ratio
1202
- if (options.width !== undefined && options.height === undefined) {
1203
- width = options.width;
1204
- height = Math.round(width / originalAspectRatio);
1205
- }
1206
- else if (options.width === undefined && options.height !== undefined) {
1207
- height = options.height;
1208
- width = Math.round(height * originalAspectRatio);
1209
- }
1210
- else if (options.width !== undefined && options.height !== undefined) {
1211
- width = options.width;
1212
- height = options.height;
1213
- }
1214
- this._videoTrack = videoTrack;
1215
- this._alpha = options.alpha ?? false;
1216
- this._width = width;
1217
- this._height = height;
1218
- this._rotation = rotation;
1219
- this._crop = crop;
1220
- this._fit = options.fit ?? 'fill';
1221
- this._videoSampleSink = new VideoSampleSink(videoTrack);
1222
- this._canvasPool = Array.from({ length: options.poolSize ?? 0 }, () => null);
1223
- }
1224
- /** @internal */
1225
- _videoSampleToWrappedCanvas(sample) {
1226
- let canvas = this._canvasPool[this._nextCanvasIndex];
1227
- let canvasIsNew = false;
1228
- if (!canvas) {
1229
- if (typeof document !== 'undefined') {
1230
- // Prefer an HTMLCanvasElement
1231
- canvas = document.createElement('canvas');
1232
- canvas.width = this._width;
1233
- canvas.height = this._height;
1234
- }
1235
- else {
1236
- canvas = new OffscreenCanvas(this._width, this._height);
1237
- }
1238
- if (this._canvasPool.length > 0) {
1239
- this._canvasPool[this._nextCanvasIndex] = canvas;
1240
- }
1241
- canvasIsNew = true;
1242
- }
1243
- if (this._canvasPool.length > 0) {
1244
- this._nextCanvasIndex = (this._nextCanvasIndex + 1) % this._canvasPool.length;
1245
- }
1246
- const context = canvas.getContext('2d', {
1247
- alpha: this._alpha || isFirefox(), // Firefox has VideoFrame glitches with opaque canvases
1248
- });
1249
- assert(context);
1250
- context.resetTransform();
1251
- if (!canvasIsNew) {
1252
- if (!this._alpha && isFirefox()) {
1253
- context.fillStyle = 'black';
1254
- context.fillRect(0, 0, this._width, this._height);
1255
- }
1256
- else {
1257
- context.clearRect(0, 0, this._width, this._height);
1258
- }
1259
- }
1260
- sample.drawWithFit(context, {
1261
- fit: this._fit,
1262
- rotation: this._rotation,
1263
- crop: this._crop,
1264
- });
1265
- const result = {
1266
- canvas,
1267
- timestamp: sample.timestamp,
1268
- duration: sample.duration,
1269
- };
1270
- sample.close();
1271
- return result;
1272
- }
1273
- /**
1274
- * Retrieves a canvas with the video frame corresponding to the given timestamp, in seconds. More specifically,
1275
- * returns the last video frame (in presentation order) with a start timestamp less than or equal to the given
1276
- * timestamp. Returns null if the timestamp is before the track's first timestamp.
1277
- *
1278
- * @param timestamp - The timestamp used for retrieval, in seconds.
1279
- */
1280
- async getCanvas(timestamp) {
1281
- validateTimestamp(timestamp);
1282
- const sample = await this._videoSampleSink.getSample(timestamp);
1283
- return sample && this._videoSampleToWrappedCanvas(sample);
1284
- }
1285
- /**
1286
- * Creates an async iterator that yields canvases with the video frames of this track in presentation order. This
1287
- * method will intelligently pre-decode a few frames ahead to enable fast iteration.
1288
- *
1289
- * @param startTimestamp - The timestamp in seconds at which to start yielding canvases (inclusive).
1290
- * @param endTimestamp - The timestamp in seconds at which to stop yielding canvases (exclusive).
1291
- */
1292
- canvases(startTimestamp = 0, endTimestamp = Infinity) {
1293
- return mapAsyncGenerator(this._videoSampleSink.samples(startTimestamp, endTimestamp), sample => this._videoSampleToWrappedCanvas(sample));
1294
- }
1295
- /**
1296
- * Creates an async iterator that yields a canvas for each timestamp in the argument. This method uses an optimized
1297
- * decoding pipeline if these timestamps are monotonically sorted, decoding each packet at most once, and is
1298
- * therefore more efficient than manually getting the canvas for every timestamp. The iterator may yield null if
1299
- * no frame is available for a given timestamp.
1300
- *
1301
- * @param timestamps - An iterable or async iterable of timestamps in seconds.
1302
- */
1303
- canvasesAtTimestamps(timestamps) {
1304
- return mapAsyncGenerator(this._videoSampleSink.samplesAtTimestamps(timestamps), sample => sample && this._videoSampleToWrappedCanvas(sample));
1305
- }
1306
- }
1307
- class AudioDecoderWrapper extends DecoderWrapper {
1308
- constructor(onSample, onError, codec, decoderConfig) {
1309
- super(onSample, onError);
1310
- this.decoder = null;
1311
- this.customDecoder = null;
1312
- this.customDecoderCallSerializer = new CallSerializer();
1313
- this.customDecoderQueueSize = 0;
1314
- // Internal state to accumulate a precise current timestamp based on audio durations, not the (potentially
1315
- // inaccurate) packet timestamps.
1316
- this.currentTimestamp = null;
1317
- const sampleHandler = (sample) => {
1318
- if (this.currentTimestamp === null
1319
- || Math.abs(sample.timestamp - this.currentTimestamp) >= sample.duration) {
1320
- // We need to sync with the sample timestamp again
1321
- this.currentTimestamp = sample.timestamp;
1322
- }
1323
- const preciseTimestamp = this.currentTimestamp;
1324
- this.currentTimestamp += sample.duration;
1325
- if (sample.numberOfFrames === 0) {
1326
- // We skip zero-data (empty) AudioSamples. These are sometimes emitted, for example, by Firefox when it
1327
- // decodes Vorbis (at the start).
1328
- sample.close();
1329
- return;
1330
- }
1331
- // Round the timestamp to the sample rate
1332
- const sampleRate = decoderConfig.sampleRate;
1333
- sample.setTimestamp(Math.round(preciseTimestamp * sampleRate) / sampleRate);
1334
- onSample(sample);
1335
- };
1336
- const MatchingCustomDecoder = customAudioDecoders.find(x => x.supports(codec, decoderConfig));
1337
- if (MatchingCustomDecoder) {
1338
- // @ts-expect-error "Can't create instance of abstract class 🤓"
1339
- this.customDecoder = new MatchingCustomDecoder();
1340
- // @ts-expect-error It's technically readonly
1341
- this.customDecoder.codec = codec;
1342
- // @ts-expect-error It's technically readonly
1343
- this.customDecoder.config = decoderConfig;
1344
- // @ts-expect-error It's technically readonly
1345
- this.customDecoder.onSample = (sample) => {
1346
- if (!(sample instanceof AudioSample)) {
1347
- throw new TypeError('The argument passed to onSample must be an AudioSample.');
1348
- }
1349
- sampleHandler(sample);
1350
- };
1351
- void this.customDecoderCallSerializer.call(() => this.customDecoder.init());
1352
- }
1353
- else {
1354
- this.decoder = new AudioDecoder({
1355
- output: (data) => {
1356
- try {
1357
- sampleHandler(new AudioSample(data));
1358
- }
1359
- catch (error) {
1360
- this.onError(error);
1361
- }
1362
- },
1363
- error: onError,
1364
- });
1365
- this.decoder.configure(decoderConfig);
1366
- }
1367
- }
1368
- getDecodeQueueSize() {
1369
- if (this.customDecoder) {
1370
- return this.customDecoderQueueSize;
1371
- }
1372
- else {
1373
- assert(this.decoder);
1374
- return this.decoder.decodeQueueSize;
1375
- }
1376
- }
1377
- decode(packet) {
1378
- if (this.customDecoder) {
1379
- this.customDecoderQueueSize++;
1380
- void this.customDecoderCallSerializer
1381
- .call(() => this.customDecoder.decode(packet))
1382
- .then(() => this.customDecoderQueueSize--);
1383
- }
1384
- else {
1385
- assert(this.decoder);
1386
- this.decoder.decode(packet.toEncodedAudioChunk());
1387
- }
1388
- }
1389
- flush() {
1390
- if (this.customDecoder) {
1391
- return this.customDecoderCallSerializer.call(() => this.customDecoder.flush());
1392
- }
1393
- else {
1394
- assert(this.decoder);
1395
- return this.decoder.flush();
1396
- }
1397
- }
1398
- close() {
1399
- if (this.customDecoder) {
1400
- void this.customDecoderCallSerializer.call(() => this.customDecoder.close());
1401
- }
1402
- else {
1403
- assert(this.decoder);
1404
- this.decoder.close();
1405
- }
1406
- }
1407
- }
1408
- // There are a lot of PCM variants not natively supported by the browser and by AudioData. Therefore we need a simple
1409
- // decoder that maps any input PCM format into a PCM format supported by the browser.
1410
- class PcmAudioDecoderWrapper extends DecoderWrapper {
1411
- constructor(onSample, onError, decoderConfig) {
1412
- super(onSample, onError);
1413
- this.decoderConfig = decoderConfig;
1414
- // Internal state to accumulate a precise current timestamp based on audio durations, not the (potentially
1415
- // inaccurate) packet timestamps.
1416
- this.currentTimestamp = null;
1417
- assert(PCM_AUDIO_CODECS.includes(decoderConfig.codec));
1418
- this.codec = decoderConfig.codec;
1419
- const { dataType, sampleSize, littleEndian } = parsePcmCodec(this.codec);
1420
- this.inputSampleSize = sampleSize;
1421
- switch (sampleSize) {
1422
- case 1:
1423
- {
1424
- if (dataType === 'unsigned') {
1425
- this.readInputValue = (view, byteOffset) => view.getUint8(byteOffset) - 2 ** 7;
1426
- }
1427
- else if (dataType === 'signed') {
1428
- this.readInputValue = (view, byteOffset) => view.getInt8(byteOffset);
1429
- }
1430
- else if (dataType === 'ulaw') {
1431
- this.readInputValue = (view, byteOffset) => fromUlaw(view.getUint8(byteOffset));
1432
- }
1433
- else if (dataType === 'alaw') {
1434
- this.readInputValue = (view, byteOffset) => fromAlaw(view.getUint8(byteOffset));
1435
- }
1436
- else {
1437
- assert(false);
1438
- }
1439
- }
1440
- ;
1441
- break;
1442
- case 2:
1443
- {
1444
- if (dataType === 'unsigned') {
1445
- this.readInputValue = (view, byteOffset) => view.getUint16(byteOffset, littleEndian) - 2 ** 15;
1446
- }
1447
- else if (dataType === 'signed') {
1448
- this.readInputValue = (view, byteOffset) => view.getInt16(byteOffset, littleEndian);
1449
- }
1450
- else {
1451
- assert(false);
1452
- }
1453
- }
1454
- ;
1455
- break;
1456
- case 3:
1457
- {
1458
- if (dataType === 'unsigned') {
1459
- this.readInputValue = (view, byteOffset) => getUint24(view, byteOffset, littleEndian) - 2 ** 23;
1460
- }
1461
- else if (dataType === 'signed') {
1462
- this.readInputValue = (view, byteOffset) => getInt24(view, byteOffset, littleEndian);
1463
- }
1464
- else {
1465
- assert(false);
1466
- }
1467
- }
1468
- ;
1469
- break;
1470
- case 4:
1471
- {
1472
- if (dataType === 'unsigned') {
1473
- this.readInputValue = (view, byteOffset) => view.getUint32(byteOffset, littleEndian) - 2 ** 31;
1474
- }
1475
- else if (dataType === 'signed') {
1476
- this.readInputValue = (view, byteOffset) => view.getInt32(byteOffset, littleEndian);
1477
- }
1478
- else if (dataType === 'float') {
1479
- this.readInputValue = (view, byteOffset) => view.getFloat32(byteOffset, littleEndian);
1480
- }
1481
- else {
1482
- assert(false);
1483
- }
1484
- }
1485
- ;
1486
- break;
1487
- case 8:
1488
- {
1489
- if (dataType === 'float') {
1490
- this.readInputValue = (view, byteOffset) => view.getFloat64(byteOffset, littleEndian);
1491
- }
1492
- else {
1493
- assert(false);
1494
- }
1495
- }
1496
- ;
1497
- break;
1498
- default:
1499
- {
1500
- assertNever(sampleSize);
1501
- assert(false);
1502
- }
1503
- ;
1504
- }
1505
- switch (sampleSize) {
1506
- case 1:
1507
- {
1508
- if (dataType === 'ulaw' || dataType === 'alaw') {
1509
- this.outputSampleSize = 2;
1510
- this.outputFormat = 's16';
1511
- this.writeOutputValue = (view, byteOffset, value) => view.setInt16(byteOffset, value, true);
1512
- }
1513
- else {
1514
- this.outputSampleSize = 1;
1515
- this.outputFormat = 'u8';
1516
- this.writeOutputValue = (view, byteOffset, value) => view.setUint8(byteOffset, value + 2 ** 7);
1517
- }
1518
- }
1519
- ;
1520
- break;
1521
- case 2:
1522
- {
1523
- this.outputSampleSize = 2;
1524
- this.outputFormat = 's16';
1525
- this.writeOutputValue = (view, byteOffset, value) => view.setInt16(byteOffset, value, true);
1526
- }
1527
- ;
1528
- break;
1529
- case 3:
1530
- {
1531
- this.outputSampleSize = 4;
1532
- this.outputFormat = 's32';
1533
- // From https://www.w3.org/TR/webcodecs:
1534
- // AudioData containing 24-bit samples SHOULD store those samples in s32 or f32. When samples are
1535
- // stored in s32, each sample MUST be left-shifted by 8 bits.
1536
- this.writeOutputValue = (view, byteOffset, value) => view.setInt32(byteOffset, value << 8, true);
1537
- }
1538
- ;
1539
- break;
1540
- case 4:
1541
- {
1542
- this.outputSampleSize = 4;
1543
- if (dataType === 'float') {
1544
- this.outputFormat = 'f32';
1545
- this.writeOutputValue = (view, byteOffset, value) => view.setFloat32(byteOffset, value, true);
1546
- }
1547
- else {
1548
- this.outputFormat = 's32';
1549
- this.writeOutputValue = (view, byteOffset, value) => view.setInt32(byteOffset, value, true);
1550
- }
1551
- }
1552
- ;
1553
- break;
1554
- case 8:
1555
- {
1556
- this.outputSampleSize = 4;
1557
- this.outputFormat = 'f32';
1558
- this.writeOutputValue = (view, byteOffset, value) => view.setFloat32(byteOffset, value, true);
1559
- }
1560
- ;
1561
- break;
1562
- default:
1563
- {
1564
- assertNever(sampleSize);
1565
- assert(false);
1566
- }
1567
- ;
1568
- }
1569
- ;
1570
- }
1571
- getDecodeQueueSize() {
1572
- return 0;
1573
- }
1574
- decode(packet) {
1575
- const inputView = toDataView(packet.data);
1576
- const numberOfFrames = packet.byteLength / this.decoderConfig.numberOfChannels / this.inputSampleSize;
1577
- const outputBufferSize = numberOfFrames * this.decoderConfig.numberOfChannels * this.outputSampleSize;
1578
- const outputBuffer = new ArrayBuffer(outputBufferSize);
1579
- const outputView = new DataView(outputBuffer);
1580
- for (let i = 0; i < numberOfFrames * this.decoderConfig.numberOfChannels; i++) {
1581
- const inputIndex = i * this.inputSampleSize;
1582
- const outputIndex = i * this.outputSampleSize;
1583
- const value = this.readInputValue(inputView, inputIndex);
1584
- this.writeOutputValue(outputView, outputIndex, value);
1585
- }
1586
- const preciseDuration = numberOfFrames / this.decoderConfig.sampleRate;
1587
- if (this.currentTimestamp === null || Math.abs(packet.timestamp - this.currentTimestamp) >= preciseDuration) {
1588
- // We need to sync with the packet timestamp again
1589
- this.currentTimestamp = packet.timestamp;
1590
- }
1591
- const preciseTimestamp = this.currentTimestamp;
1592
- this.currentTimestamp += preciseDuration;
1593
- const audioSample = new AudioSample({
1594
- format: this.outputFormat,
1595
- data: outputBuffer,
1596
- numberOfChannels: this.decoderConfig.numberOfChannels,
1597
- sampleRate: this.decoderConfig.sampleRate,
1598
- numberOfFrames,
1599
- timestamp: preciseTimestamp,
1600
- });
1601
- this.onSample(audioSample);
1602
- }
1603
- async flush() {
1604
- // Do nothing
1605
- }
1606
- close() {
1607
- // Do nothing
1608
- }
1609
- }
1610
- /**
1611
- * Sink for retrieving decoded audio samples from an audio track.
1612
- * @group Media sinks
1613
- * @public
1614
- */
1615
- export class AudioSampleSink extends BaseMediaSampleSink {
1616
- /** Creates a new {@link AudioSampleSink} for the given {@link InputAudioTrack}. */
1617
- constructor(audioTrack) {
1618
- if (!(audioTrack instanceof InputAudioTrack)) {
1619
- throw new TypeError('audioTrack must be an InputAudioTrack.');
1620
- }
1621
- super();
1622
- this._track = audioTrack;
1623
- }
1624
- /** @internal */
1625
- async _createDecoder(onSample, onError) {
1626
- if (!(await this._track.canDecode())) {
1627
- throw new Error('This audio track cannot be decoded by this browser. Make sure to check decodability before using'
1628
- + ' a track.');
1629
- }
1630
- const codec = this._track.codec;
1631
- const decoderConfig = await this._track.getDecoderConfig();
1632
- assert(codec && decoderConfig);
1633
- if (PCM_AUDIO_CODECS.includes(decoderConfig.codec)) {
1634
- return new PcmAudioDecoderWrapper(onSample, onError, decoderConfig);
1635
- }
1636
- else {
1637
- return new AudioDecoderWrapper(onSample, onError, codec, decoderConfig);
1638
- }
1639
- }
1640
- /** @internal */
1641
- _createPacketSink() {
1642
- return new EncodedPacketSink(this._track);
1643
- }
1644
- /**
1645
- * Retrieves the audio sample corresponding to the given timestamp, in seconds. More specifically, returns
1646
- * the last audio sample (in presentation order) with a start timestamp less than or equal to the given timestamp.
1647
- * Returns null if the timestamp is before the track's first timestamp.
1648
- *
1649
- * @param timestamp - The timestamp used for retrieval, in seconds.
1650
- */
1651
- async getSample(timestamp) {
1652
- validateTimestamp(timestamp);
1653
- for await (const sample of this.mediaSamplesAtTimestamps([timestamp])) {
1654
- return sample;
1655
- }
1656
- throw new Error('Internal error: Iterator returned nothing.');
1657
- }
1658
- /**
1659
- * Creates an async iterator that yields the audio samples of this track in presentation order. This method
1660
- * will intelligently pre-decode a few samples ahead to enable fast iteration.
1661
- *
1662
- * @param startTimestamp - The timestamp in seconds at which to start yielding samples (inclusive).
1663
- * @param endTimestamp - The timestamp in seconds at which to stop yielding samples (exclusive).
1664
- */
1665
- samples(startTimestamp = 0, endTimestamp = Infinity) {
1666
- return this.mediaSamplesInRange(startTimestamp, endTimestamp);
1667
- }
1668
- /**
1669
- * Creates an async iterator that yields an audio sample for each timestamp in the argument. This method
1670
- * uses an optimized decoding pipeline if these timestamps are monotonically sorted, decoding each packet at most
1671
- * once, and is therefore more efficient than manually getting the sample for every timestamp. The iterator may
1672
- * yield null if no sample is available for a given timestamp.
1673
- *
1674
- * @param timestamps - An iterable or async iterable of timestamps in seconds.
1675
- */
1676
- samplesAtTimestamps(timestamps) {
1677
- return this.mediaSamplesAtTimestamps(timestamps);
1678
- }
1679
- }
1680
- /**
1681
- * A sink that retrieves decoded audio samples from an audio track and converts them to `AudioBuffer` instances. This is
1682
- * often more useful than directly retrieving audio samples, as audio buffers can be directly used with the
1683
- * Web Audio API.
1684
- * @group Media sinks
1685
- * @public
1686
- */
1687
- export class AudioBufferSink {
1688
- /** Creates a new {@link AudioBufferSink} for the given {@link InputAudioTrack}. */
1689
- constructor(audioTrack) {
1690
- if (!(audioTrack instanceof InputAudioTrack)) {
1691
- throw new TypeError('audioTrack must be an InputAudioTrack.');
1692
- }
1693
- this._audioSampleSink = new AudioSampleSink(audioTrack);
1694
- }
1695
- /** @internal */
1696
- _audioSampleToWrappedArrayBuffer(sample) {
1697
- return {
1698
- buffer: sample.toAudioBuffer(),
1699
- timestamp: sample.timestamp,
1700
- duration: sample.duration,
1701
- };
1702
- }
1703
- /**
1704
- * Retrieves the audio buffer corresponding to the given timestamp, in seconds. More specifically, returns
1705
- * the last audio buffer (in presentation order) with a start timestamp less than or equal to the given timestamp.
1706
- * Returns null if the timestamp is before the track's first timestamp.
1707
- *
1708
- * @param timestamp - The timestamp used for retrieval, in seconds.
1709
- */
1710
- async getBuffer(timestamp) {
1711
- validateTimestamp(timestamp);
1712
- const data = await this._audioSampleSink.getSample(timestamp);
1713
- return data && this._audioSampleToWrappedArrayBuffer(data);
1714
- }
1715
- /**
1716
- * Creates an async iterator that yields audio buffers of this track in presentation order. This method
1717
- * will intelligently pre-decode a few buffers ahead to enable fast iteration.
1718
- *
1719
- * @param startTimestamp - The timestamp in seconds at which to start yielding buffers (inclusive).
1720
- * @param endTimestamp - The timestamp in seconds at which to stop yielding buffers (exclusive).
1721
- */
1722
- buffers(startTimestamp = 0, endTimestamp = Infinity) {
1723
- return mapAsyncGenerator(this._audioSampleSink.samples(startTimestamp, endTimestamp), data => this._audioSampleToWrappedArrayBuffer(data));
1724
- }
1725
- /**
1726
- * Creates an async iterator that yields an audio buffer for each timestamp in the argument. This method
1727
- * uses an optimized decoding pipeline if these timestamps are monotonically sorted, decoding each packet at most
1728
- * once, and is therefore more efficient than manually getting the buffer for every timestamp. The iterator may
1729
- * yield null if no buffer is available for a given timestamp.
1730
- *
1731
- * @param timestamps - An iterable or async iterable of timestamps in seconds.
1732
- */
1733
- buffersAtTimestamps(timestamps) {
1734
- return mapAsyncGenerator(this._audioSampleSink.samplesAtTimestamps(timestamps), data => data && this._audioSampleToWrappedArrayBuffer(data));
1735
- }
1736
- }