@appium/media-proxy 0.0.1 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/CHANGELOG.md +33 -0
  2. package/LICENSE +201 -0
  3. package/README.md +183 -2
  4. package/THIRD_PARTY_NOTICES.md +29 -0
  5. package/binding.gyp +90 -0
  6. package/lib/src/background.d.ts +5 -0
  7. package/lib/src/background.d.ts.map +1 -0
  8. package/lib/src/background.js +16 -0
  9. package/lib/src/background.js.map +1 -0
  10. package/lib/src/capabilities.d.ts +9 -0
  11. package/lib/src/capabilities.d.ts.map +1 -0
  12. package/lib/src/capabilities.js +11 -0
  13. package/lib/src/capabilities.js.map +1 -0
  14. package/lib/src/errors.d.ts +26 -0
  15. package/lib/src/errors.d.ts.map +1 -0
  16. package/lib/src/errors.js +40 -0
  17. package/lib/src/errors.js.map +1 -0
  18. package/lib/src/framing/apst.d.ts +43 -0
  19. package/lib/src/framing/apst.d.ts.map +1 -0
  20. package/lib/src/framing/apst.js +126 -0
  21. package/lib/src/framing/apst.js.map +1 -0
  22. package/lib/src/framing/mjpeg-http.d.ts +30 -0
  23. package/lib/src/framing/mjpeg-http.d.ts.map +1 -0
  24. package/lib/src/framing/mjpeg-http.js +148 -0
  25. package/lib/src/framing/mjpeg-http.js.map +1 -0
  26. package/lib/src/framing/mjpeg-multipart.d.ts +11 -0
  27. package/lib/src/framing/mjpeg-multipart.d.ts.map +1 -0
  28. package/lib/src/framing/mjpeg-multipart.js +26 -0
  29. package/lib/src/framing/mjpeg-multipart.js.map +1 -0
  30. package/lib/src/index.d.ts +10 -0
  31. package/lib/src/index.d.ts.map +1 -0
  32. package/lib/src/index.js +7 -0
  33. package/lib/src/index.js.map +1 -0
  34. package/lib/src/native.d.ts +12 -0
  35. package/lib/src/native.d.ts.map +1 -0
  36. package/lib/src/native.js +36 -0
  37. package/lib/src/native.js.map +1 -0
  38. package/lib/src/pkg-root.d.ts +6 -0
  39. package/lib/src/pkg-root.d.ts.map +1 -0
  40. package/lib/src/pkg-root.js +12 -0
  41. package/lib/src/pkg-root.js.map +1 -0
  42. package/lib/src/queue.d.ts +34 -0
  43. package/lib/src/queue.d.ts.map +1 -0
  44. package/lib/src/queue.js +123 -0
  45. package/lib/src/queue.js.map +1 -0
  46. package/lib/src/transcoder.d.ts +81 -0
  47. package/lib/src/transcoder.d.ts.map +1 -0
  48. package/lib/src/transcoder.js +422 -0
  49. package/lib/src/transcoder.js.map +1 -0
  50. package/lib/src/types.d.ts +147 -0
  51. package/lib/src/types.d.ts.map +1 -0
  52. package/lib/src/types.js +2 -0
  53. package/lib/src/types.js.map +1 -0
  54. package/package.json +53 -4
  55. package/prebuilds/darwin-arm64/@appium+media-proxy.node +0 -0
  56. package/prebuilds/darwin-arm64/ffmpeg-LICENSE-LGPL-2.1.txt +502 -0
  57. package/prebuilds/darwin-arm64/ffmpeg-build-info.txt +3 -0
  58. package/prebuilds/darwin-arm64/libavcodec-appium.62.dylib +0 -0
  59. package/prebuilds/darwin-arm64/libavutil-appium.60.dylib +0 -0
  60. package/prebuilds/darwin-arm64/libswscale-appium.9.dylib +0 -0
  61. package/prebuilds/darwin-x64/@appium+media-proxy.node +0 -0
  62. package/prebuilds/darwin-x64/ffmpeg-LICENSE-LGPL-2.1.txt +502 -0
  63. package/prebuilds/darwin-x64/ffmpeg-build-info.txt +3 -0
  64. package/prebuilds/darwin-x64/libavcodec-appium.62.dylib +0 -0
  65. package/prebuilds/darwin-x64/libavutil-appium.60.dylib +0 -0
  66. package/prebuilds/darwin-x64/libswscale-appium.9.dylib +0 -0
  67. package/prebuilds/linux-arm64/@appium+media-proxy.node +0 -0
  68. package/prebuilds/linux-arm64/ffmpeg-LICENSE-LGPL-2.1.txt +502 -0
  69. package/prebuilds/linux-arm64/ffmpeg-build-info.txt +3 -0
  70. package/prebuilds/linux-arm64/libavcodec-appium.so.62 +0 -0
  71. package/prebuilds/linux-arm64/libavutil-appium.so.60 +0 -0
  72. package/prebuilds/linux-arm64/libswscale-appium.so.9 +0 -0
  73. package/prebuilds/linux-x64/@appium+media-proxy.node +0 -0
  74. package/prebuilds/linux-x64/ffmpeg-LICENSE-LGPL-2.1.txt +502 -0
  75. package/prebuilds/linux-x64/ffmpeg-build-info.txt +3 -0
  76. package/prebuilds/linux-x64/libavcodec-appium.so.62 +0 -0
  77. package/prebuilds/linux-x64/libavutil-appium.so.60 +0 -0
  78. package/prebuilds/linux-x64/libswscale-appium.so.9 +0 -0
  79. package/prebuilds/win32-arm64/@appium+media-proxy.node +0 -0
  80. package/prebuilds/win32-arm64/avcodec-appium-62.dll +0 -0
  81. package/prebuilds/win32-arm64/avutil-appium-60.dll +0 -0
  82. package/prebuilds/win32-arm64/ffmpeg-LICENSE-LGPL-2.1.txt +502 -0
  83. package/prebuilds/win32-arm64/ffmpeg-build-info.txt +3 -0
  84. package/prebuilds/win32-arm64/swscale-appium-9.dll +0 -0
  85. package/prebuilds/win32-x64/@appium+media-proxy.node +0 -0
  86. package/prebuilds/win32-x64/avcodec-appium-62.dll +0 -0
  87. package/prebuilds/win32-x64/avutil-appium-60.dll +0 -0
  88. package/prebuilds/win32-x64/ffmpeg-LICENSE-LGPL-2.1.txt +502 -0
  89. package/prebuilds/win32-x64/ffmpeg-build-info.txt +3 -0
  90. package/prebuilds/win32-x64/swscale-appium-9.dll +0 -0
  91. package/scripts/install.mjs +29 -0
  92. package/src/background.ts +15 -0
  93. package/src/capabilities.ts +12 -0
  94. package/src/errors.ts +45 -0
  95. package/src/framing/apst.ts +158 -0
  96. package/src/framing/mjpeg-http.ts +186 -0
  97. package/src/framing/mjpeg-multipart.ts +38 -0
  98. package/src/index.ts +43 -0
  99. package/src/native/addon.cc +402 -0
  100. package/src/native/audio_pipeline.cc +426 -0
  101. package/src/native/audio_pipeline.h +63 -0
  102. package/src/native/ffmpeg_util.h +37 -0
  103. package/src/native/h264_encoder.cc +133 -0
  104. package/src/native/h264_encoder.h +48 -0
  105. package/src/native/mediafoundation_encoder.cc +442 -0
  106. package/src/native/mediafoundation_encoder.h +14 -0
  107. package/src/native/napi_util.h +20 -0
  108. package/src/native/openh264_encoder.cc +285 -0
  109. package/src/native/openh264_encoder.h +14 -0
  110. package/src/native/pipeline.cc +555 -0
  111. package/src/native/pipeline.h +108 -0
  112. package/src/native/vendor/openh264/LICENSE +23 -0
  113. package/src/native/vendor/openh264/codec_api.h +592 -0
  114. package/src/native/vendor/openh264/codec_app_def.h +820 -0
  115. package/src/native/vendor/openh264/codec_def.h +216 -0
  116. package/src/native/vendor/openh264/codec_ver.h +15 -0
  117. package/src/native/videotoolbox_encoder.cc +314 -0
  118. package/src/native/videotoolbox_encoder.h +13 -0
  119. package/src/native.ts +51 -0
  120. package/src/pkg-root.ts +14 -0
  121. package/src/queue.ts +133 -0
  122. package/src/transcoder.ts +480 -0
  123. package/src/types.ts +158 -0
@@ -0,0 +1,426 @@
1
+ #include "audio_pipeline.h"
2
+
3
+ #include <algorithm>
4
+ #include <array>
5
+ #include <cmath>
6
+ #include <cstring>
7
+ #include <exception>
8
+ #include <optional>
9
+ #include <string>
10
+ #include <utility>
11
+ #include <vector>
12
+
13
+ extern "C" {
14
+ #include <libavcodec/avcodec.h>
15
+ #include <libavutil/audio_fifo.h>
16
+ #include <libavutil/channel_layout.h>
17
+ #include <libavutil/error.h>
18
+ #include <libavutil/frame.h>
19
+ #include <libavutil/mem.h>
20
+ #include <libavutil/samplefmt.h>
21
+ }
22
+
23
+ #include "ffmpeg_util.h"
24
+
25
+ namespace media_proxy {
26
+
27
+ namespace {
28
+
29
+ constexpr int64_t kMicrosPerSecond = 1000000;
30
+ // RTP timestamps are exact, so anything beyond this between units is a real gap (lost packets).
31
+ constexpr int64_t kGapToleranceMicros = 5000;
32
+ // Longer gaps restart the timeline instead of being padded with silence.
33
+ constexpr int64_t kMaxSilenceMicros = 1000000;
34
+ constexpr int kMaxChannels = 6;
35
+ constexpr size_t kAdtsHeaderSize = 7;
36
+ constexpr int kAdtsLcProfile = 1; // object type 2 (AAC-LC) minus one
37
+ constexpr std::array<int, 13> kAdtsSampleRates = {96000, 88200, 64000, 48000, 44100, 32000, 24000,
38
+ 22050, 16000, 12000, 11025, 8000, 7350};
39
+
40
+ struct FifoDeleter {
41
+ void operator()(AVAudioFifo* fifo) const { av_audio_fifo_free(fifo); }
42
+ };
43
+ using Fifo = std::unique_ptr<AVAudioFifo, FifoDeleter>;
44
+
45
+ constexpr int kEscapeObjectType = 31;
46
+ constexpr int kEscapeSampleRate = 15;
47
+ constexpr std::array<int, 8> kChannelsByConfig = {0, 1, 2, 3, 4, 5, 6, 8};
48
+
49
+ // Reads the leading fields of an MPEG-4 AudioSpecificConfig: the decoder is not told them by a demuxer here.
50
+ struct AscFormat {
51
+ int sampleRate = 0;
52
+ int channels = 0;
53
+ };
54
+
55
+ std::optional<AscFormat> ParseAsc(const std::vector<uint8_t>& asc) {
56
+ size_t bit = 0;
57
+ bool ok = true;
58
+ const auto read = [&](int count) {
59
+ int value = 0;
60
+ for (int i = 0; i < count; ++i, ++bit) {
61
+ if (bit / 8 >= asc.size()) {
62
+ ok = false;
63
+ return 0;
64
+ }
65
+ value = (value << 1) | ((asc[bit / 8] >> (7 - (bit % 8))) & 1);
66
+ }
67
+ return value;
68
+ };
69
+ if (read(5) == kEscapeObjectType) {
70
+ read(6);
71
+ }
72
+ const int rateIndex = read(4);
73
+ int rate = 0;
74
+ if (rateIndex == kEscapeSampleRate) {
75
+ rate = read(24);
76
+ } else if (rateIndex < static_cast<int>(kAdtsSampleRates.size())) {
77
+ rate = kAdtsSampleRates[rateIndex];
78
+ }
79
+ const int config = read(4);
80
+ if (!ok || rate <= 0 || config < 1 || config >= static_cast<int>(kChannelsByConfig.size())) {
81
+ return std::nullopt;
82
+ }
83
+ return AscFormat{rate, kChannelsByConfig[config]};
84
+ }
85
+
86
+ int64_t SamplesToMicros(int64_t samples, int rate) { return ((samples * kMicrosPerSecond) + (rate / 2)) / rate; }
87
+
88
+ int AdtsRateIndex(int rate) {
89
+ for (size_t i = 0; i < kAdtsSampleRates.size(); ++i) {
90
+ if (kAdtsSampleRates[i] == rate) {
91
+ return static_cast<int>(i);
92
+ }
93
+ }
94
+ return -1;
95
+ }
96
+
97
+ std::array<uint8_t, kAdtsHeaderSize> AdtsHeader(size_t payloadSize, int rateIndex, int channels) {
98
+ const size_t length = kAdtsHeaderSize + payloadSize;
99
+ return {
100
+ 0xFF, // syncword
101
+ 0xF1, // MPEG-4, layer 0, no CRC
102
+ static_cast<uint8_t>((kAdtsLcProfile << 6) | (rateIndex << 2) | ((channels >> 2) & 1)),
103
+ static_cast<uint8_t>(((channels & 3) << 6) | ((length >> 11) & 3)),
104
+ static_cast<uint8_t>((length >> 3) & 0xFF),
105
+ static_cast<uint8_t>(((length & 7) << 5) | 0x1F), // buffer fullness: VBR
106
+ 0xFC,
107
+ };
108
+ }
109
+
110
+ } // namespace
111
+
112
+ // FFmpeg state; touched only by the worker thread after construction.
113
+ class AudioPipeline::Impl {
114
+ public:
115
+ Impl(const AudioConfig& config, UnitCallback onUnit) : config_(config), onUnit_(std::move(onUnit)) {
116
+ const AVCodec* decoder = avcodec_find_decoder(AV_CODEC_ID_AAC);
117
+ encoder_ = avcodec_find_encoder(AV_CODEC_ID_AAC);
118
+ if (decoder == nullptr || encoder_ == nullptr) {
119
+ throw PipelineError{kEncoderUnavailableError, "The AAC decoder or encoder is missing from this build"};
120
+ }
121
+ if (config.audioSpecificConfig.empty()) {
122
+ throw PipelineError{"MediaProxyOptionError", "audioSpecificConfig must not be empty"};
123
+ }
124
+ const std::optional<AscFormat> format = ParseAsc(config.audioSpecificConfig);
125
+ if (!format) {
126
+ throw PipelineError{"MediaProxyOptionError", "audioSpecificConfig is not a usable AudioSpecificConfig"};
127
+ }
128
+ decoder_.reset(avcodec_alloc_context3(decoder));
129
+ decoder_->sample_rate = format->sampleRate;
130
+ av_channel_layout_default(&decoder_->ch_layout, format->channels);
131
+ const size_t size = config.audioSpecificConfig.size();
132
+ decoder_->extradata = static_cast<uint8_t*>(av_mallocz(size + AV_INPUT_BUFFER_PADDING_SIZE));
133
+ if (decoder_->extradata == nullptr) {
134
+ throw PipelineError{"MediaProxyError", "Out of memory"};
135
+ }
136
+ std::memcpy(decoder_->extradata, config.audioSpecificConfig.data(), size);
137
+ decoder_->extradata_size = static_cast<int>(size);
138
+ decoder_->flags |= AV_CODEC_FLAG_LOW_DELAY;
139
+ if (const int ret = avcodec_open2(decoder_.get(), decoder, nullptr); ret < 0) {
140
+ throw Fatal("Could not open the AAC decoder", ret);
141
+ }
142
+ packet_.reset(av_packet_alloc());
143
+ decoded_.reset(av_frame_alloc());
144
+ chunk_.reset(av_frame_alloc());
145
+ }
146
+
147
+ void Process(const Input& input) {
148
+ if (input.data.empty()) {
149
+ return;
150
+ }
151
+ if (!AlignTimeline(input.timestampMicros)) {
152
+ return;
153
+ }
154
+ if (const int ret = av_new_packet(packet_.get(), static_cast<int>(input.data.size())); ret < 0) {
155
+ throw Fatal("Could not allocate a packet", ret);
156
+ }
157
+ std::copy(input.data.begin(), input.data.end(), packet_->data);
158
+ const int ret = avcodec_send_packet(decoder_.get(), packet_.get());
159
+ av_packet_unref(packet_.get());
160
+ // A corrupt unit is skipped; the gap it leaves is padded with silence by the next one.
161
+ if (ret < 0 && ret != AVERROR_INVALIDDATA && ret != AVERROR(EAGAIN)) {
162
+ throw Fatal("AAC decode failed", ret);
163
+ }
164
+ DrainDecoder();
165
+ EncodeAvailable();
166
+ }
167
+
168
+ void Flush() {
169
+ avcodec_send_packet(decoder_.get(), nullptr);
170
+ DrainDecoder();
171
+ EncodeAvailable();
172
+ if (fifo_ && av_audio_fifo_size(fifo_.get()) > 0) {
173
+ WriteSilence(chunkSize_ - av_audio_fifo_size(fifo_.get())); // pad the tail to a whole AAC frame
174
+ EncodeAvailable();
175
+ }
176
+ if (encoderCtx_) {
177
+ avcodec_send_frame(encoderCtx_.get(), nullptr);
178
+ DrainEncoder();
179
+ }
180
+ }
181
+
182
+ private:
183
+ [[nodiscard]] int64_t QueuedSamples() const { return readSamples_ + (fifo_ ? av_audio_fifo_size(fifo_.get()) : 0); }
184
+
185
+ // Keeps the output on the input's clock: pads gaps with silence, restarts after a long one, drops late repeats.
186
+ bool AlignTimeline(int64_t timestampMicros) {
187
+ if (!anchored_ || sampleRate_ == 0) {
188
+ anchorMicros_ = timestampMicros;
189
+ anchored_ = true;
190
+ return true;
191
+ }
192
+ const int64_t gap = timestampMicros - (anchorMicros_ + SamplesToMicros(QueuedSamples(), sampleRate_));
193
+ if (gap < -kGapToleranceMicros) {
194
+ return false;
195
+ }
196
+ if (gap > kMaxSilenceMicros) {
197
+ av_audio_fifo_reset(fifo_.get());
198
+ anchorMicros_ = timestampMicros;
199
+ readSamples_ = 0;
200
+ frameStarts_.clear();
201
+ } else if (gap > kGapToleranceMicros) {
202
+ WriteSilence(gap * sampleRate_ / kMicrosPerSecond);
203
+ }
204
+ return true;
205
+ }
206
+
207
+ void DrainDecoder() {
208
+ for (;;) {
209
+ const int ret = avcodec_receive_frame(decoder_.get(), decoded_.get());
210
+ if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
211
+ return;
212
+ }
213
+ if (ret == AVERROR_INVALIDDATA) {
214
+ continue;
215
+ }
216
+ if (ret < 0) {
217
+ throw Fatal("AAC decode failed", ret);
218
+ }
219
+ OpenEncoder(*decoded_);
220
+ if (const int written =
221
+ av_audio_fifo_write(fifo_.get(), reinterpret_cast<void**>(decoded_->extended_data), decoded_->nb_samples);
222
+ written < decoded_->nb_samples) {
223
+ throw Fatal("Could not buffer decoded audio", written < 0 ? written : AVERROR(ENOMEM));
224
+ }
225
+ av_frame_unref(decoded_.get());
226
+ }
227
+ }
228
+
229
+ void OpenEncoder(const AVFrame& first) {
230
+ if (encoderCtx_) {
231
+ return;
232
+ }
233
+ const int channels = first.ch_layout.nb_channels;
234
+ const int rateIndex = AdtsRateIndex(first.sample_rate);
235
+ if (first.format != AV_SAMPLE_FMT_FLTP || channels < 1 || channels > kMaxChannels || rateIndex < 0) {
236
+ throw PipelineError{"MediaProxyError", "Unsupported audio format: " + std::to_string(first.sample_rate) +
237
+ " Hz, " + std::to_string(channels) + " channels"};
238
+ }
239
+ encoderCtx_.reset(avcodec_alloc_context3(encoder_));
240
+ encoderCtx_->sample_fmt = AV_SAMPLE_FMT_FLTP;
241
+ encoderCtx_->sample_rate = first.sample_rate;
242
+ encoderCtx_->time_base = {1, first.sample_rate};
243
+ encoderCtx_->bit_rate = static_cast<int64_t>(config_.bitratePerChannel) * channels;
244
+ encoderCtx_->profile = AV_PROFILE_AAC_LOW;
245
+ av_channel_layout_default(&encoderCtx_->ch_layout, channels);
246
+ if (const int ret = avcodec_open2(encoderCtx_.get(), encoder_, nullptr); ret < 0) {
247
+ throw Fatal("Could not open the AAC encoder", ret);
248
+ }
249
+ sampleRate_ = first.sample_rate;
250
+ channels_ = channels;
251
+ rateIndex_ = rateIndex;
252
+ chunkSize_ = encoderCtx_->frame_size;
253
+ fifo_.reset(av_audio_fifo_alloc(AV_SAMPLE_FMT_FLTP, channels, chunkSize_));
254
+ if (!fifo_) {
255
+ throw PipelineError{"MediaProxyError", "Out of memory"};
256
+ }
257
+ }
258
+
259
+ void WriteSilence(int64_t samples) {
260
+ constexpr int kSilenceChunk = 4096;
261
+ const std::vector<float> zeros(kSilenceChunk, 0.0F);
262
+ std::vector<void*> planes(channels_, const_cast<float*>(zeros.data()));
263
+ while (samples > 0) {
264
+ const int count = static_cast<int>(std::min<int64_t>(samples, kSilenceChunk));
265
+ if (av_audio_fifo_write(fifo_.get(), planes.data(), count) < count) {
266
+ throw PipelineError{"MediaProxyError", "Could not buffer silence"};
267
+ }
268
+ samples -= count;
269
+ }
270
+ }
271
+
272
+ void EncodeAvailable() {
273
+ while (fifo_ && av_audio_fifo_size(fifo_.get()) >= chunkSize_) {
274
+ av_frame_unref(chunk_.get());
275
+ chunk_->nb_samples = chunkSize_;
276
+ chunk_->format = AV_SAMPLE_FMT_FLTP;
277
+ chunk_->sample_rate = sampleRate_;
278
+ av_channel_layout_copy(&chunk_->ch_layout, &encoderCtx_->ch_layout);
279
+ if (const int ret = av_frame_get_buffer(chunk_.get(), 0); ret < 0) {
280
+ throw Fatal("Could not allocate an audio frame", ret);
281
+ }
282
+ av_audio_fifo_read(fifo_.get(), reinterpret_cast<void**>(chunk_->extended_data), chunkSize_);
283
+ chunk_->pts = readSamples_;
284
+ frameStarts_.push_back(anchorMicros_ + SamplesToMicros(readSamples_, sampleRate_));
285
+ readSamples_ += chunkSize_;
286
+ if (const int ret = avcodec_send_frame(encoderCtx_.get(), chunk_.get()); ret < 0) {
287
+ throw Fatal("AAC encode failed", ret);
288
+ }
289
+ DrainEncoder();
290
+ }
291
+ }
292
+
293
+ // Each packet takes the start time of the frame it was encoded from (so it includes the encoder's own delay).
294
+ void DrainEncoder() {
295
+ for (;;) {
296
+ const int ret = avcodec_receive_packet(encoderCtx_.get(), packet_.get());
297
+ if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
298
+ return;
299
+ }
300
+ if (ret < 0) {
301
+ throw Fatal("AAC encode failed", ret);
302
+ }
303
+ EncodedUnit unit;
304
+ unit.isKeyFrame = true;
305
+ const auto header = AdtsHeader(static_cast<size_t>(packet_->size), rateIndex_, channels_);
306
+ unit.data.reserve(header.size() + static_cast<size_t>(packet_->size));
307
+ unit.data.assign(header.begin(), header.end());
308
+ unit.data.insert(unit.data.end(), packet_->data, packet_->data + packet_->size);
309
+ if (!frameStarts_.empty()) {
310
+ lastStart_ = frameStarts_.front();
311
+ frameStarts_.pop_front();
312
+ } else {
313
+ lastStart_ += SamplesToMicros(chunkSize_, sampleRate_); // the encoder's delayed tail on flush
314
+ }
315
+ unit.timestampMicros = lastStart_;
316
+ av_packet_unref(packet_.get());
317
+ onUnit_(std::move(unit));
318
+ }
319
+ }
320
+
321
+ AudioConfig config_;
322
+ UnitCallback onUnit_;
323
+ const AVCodec* encoder_ = nullptr;
324
+ CodecContext decoder_;
325
+ CodecContext encoderCtx_;
326
+ Packet packet_;
327
+ Frame decoded_;
328
+ Frame chunk_;
329
+ Fifo fifo_;
330
+ std::deque<int64_t> frameStarts_;
331
+ int64_t anchorMicros_ = 0;
332
+ int64_t readSamples_ = 0;
333
+ int64_t lastStart_ = 0;
334
+ bool anchored_ = false;
335
+ int sampleRate_ = 0;
336
+ int channels_ = 0;
337
+ int rateIndex_ = 0;
338
+ int chunkSize_ = 0;
339
+ };
340
+
341
+ AudioPipeline::AudioPipeline(const AudioConfig& config, UnitCallback onUnit, ErrorCallback onError)
342
+ : onError_(std::move(onError)), impl_(std::make_unique<Impl>(config, std::move(onUnit))) {
343
+ worker_ = std::thread([this] { Run(); });
344
+ }
345
+
346
+ AudioPipeline::~AudioPipeline() {
347
+ {
348
+ const std::scoped_lock<std::mutex> lock(mutex_);
349
+ aborting_ = true;
350
+ }
351
+ wake_.notify_all();
352
+ if (worker_.joinable()) {
353
+ worker_.join();
354
+ }
355
+ }
356
+
357
+ void AudioPipeline::Push(std::vector<uint8_t> accessUnit, int64_t timestampMicros) {
358
+ {
359
+ const std::scoped_lock<std::mutex> lock(mutex_);
360
+ if (finishing_ || aborting_) {
361
+ return;
362
+ }
363
+ if (queue_.size() >= kMaxQueuedUnits) {
364
+ queue_.pop_front();
365
+ }
366
+ queue_.push_back({std::move(accessUnit), timestampMicros});
367
+ }
368
+ wake_.notify_one();
369
+ }
370
+
371
+ void AudioPipeline::Finish(DoneCallback onDone) {
372
+ {
373
+ const std::scoped_lock<std::mutex> lock(mutex_);
374
+ onDone_ = std::move(onDone);
375
+ finishing_ = true;
376
+ }
377
+ wake_.notify_all();
378
+ }
379
+
380
+ void AudioPipeline::Run() {
381
+ bool failed = false;
382
+ // Nothing may escape the worker: an uncaught exception would terminate the whole process.
383
+ const auto attempt = [&](const auto& body) {
384
+ try {
385
+ body();
386
+ } catch (PipelineError& err) {
387
+ failed = true;
388
+ onError_(std::move(err));
389
+ } catch (const std::exception& err) {
390
+ failed = true;
391
+ onError_({"MediaProxyError", err.what()});
392
+ }
393
+ };
394
+ for (;;) {
395
+ Input input;
396
+ {
397
+ std::unique_lock<std::mutex> lock(mutex_);
398
+ wake_.wait(lock, [this] { return aborting_ || finishing_ || !queue_.empty(); });
399
+ if (aborting_) {
400
+ return;
401
+ }
402
+ if (queue_.empty()) {
403
+ break;
404
+ }
405
+ input = std::move(queue_.front());
406
+ queue_.pop_front();
407
+ }
408
+ if (!failed) {
409
+ attempt([&] { impl_->Process(input); });
410
+ }
411
+ }
412
+ if (!failed) {
413
+ attempt([&] { impl_->Flush(); });
414
+ }
415
+ impl_.reset();
416
+ DoneCallback done;
417
+ {
418
+ const std::scoped_lock<std::mutex> lock(mutex_);
419
+ done = std::move(onDone_);
420
+ }
421
+ if (done) {
422
+ done();
423
+ }
424
+ }
425
+
426
+ } // namespace media_proxy
@@ -0,0 +1,63 @@
1
+ #pragma once
2
+
3
+ #include <condition_variable>
4
+ #include <cstdint>
5
+ #include <deque>
6
+ #include <functional>
7
+ #include <memory>
8
+ #include <mutex>
9
+ #include <thread>
10
+ #include <vector>
11
+
12
+ #include "pipeline.h"
13
+
14
+ namespace media_proxy {
15
+
16
+ struct AudioConfig {
17
+ // MPEG-4 AudioSpecificConfig of the input (AAC-ELD from the device).
18
+ std::vector<uint8_t> audioSpecificConfig;
19
+ // Output AAC-LC bitrate per channel, in bits/sec.
20
+ double bitratePerChannel = 64000;
21
+ };
22
+
23
+ // AAC-ELD access units in, ADTS-framed AAC-LC units out, on a dedicated worker thread. Pure C++/FFmpeg: no N-API.
24
+ class AudioPipeline {
25
+ public:
26
+ using UnitCallback = std::function<void(EncodedUnit&&)>;
27
+ using ErrorCallback = std::function<void(PipelineError&&)>;
28
+ using DoneCallback = std::function<void()>;
29
+
30
+ // Throws PipelineError when the AAC decoder/encoder is missing from this build or the config is rejected.
31
+ AudioPipeline(const AudioConfig& config, UnitCallback onUnit, ErrorCallback onError);
32
+ ~AudioPipeline();
33
+ AudioPipeline(const AudioPipeline&) = delete;
34
+ AudioPipeline& operator=(const AudioPipeline&) = delete;
35
+
36
+ // Non-blocking; beyond kMaxQueuedUnits the oldest unit is dropped (the gap is later filled with silence).
37
+ void Push(std::vector<uint8_t> accessUnit, int64_t timestampMicros);
38
+ // Drains queued input, flushes the codecs, then calls `onDone` from the worker thread (once).
39
+ void Finish(DoneCallback onDone);
40
+
41
+ static constexpr size_t kMaxQueuedUnits = 500;
42
+
43
+ private:
44
+ struct Input {
45
+ std::vector<uint8_t> data;
46
+ int64_t timestampMicros = 0;
47
+ };
48
+ class Impl;
49
+
50
+ void Run();
51
+
52
+ std::mutex mutex_;
53
+ std::condition_variable wake_;
54
+ std::deque<Input> queue_;
55
+ bool finishing_ = false;
56
+ bool aborting_ = false;
57
+ DoneCallback onDone_;
58
+ ErrorCallback onError_;
59
+ std::unique_ptr<Impl> impl_;
60
+ std::thread worker_;
61
+ };
62
+
63
+ } // namespace media_proxy
@@ -0,0 +1,37 @@
1
+ #pragma once
2
+
3
+ #include <array>
4
+ #include <memory>
5
+ #include <string>
6
+
7
+ extern "C" {
8
+ #include <libavcodec/avcodec.h>
9
+ #include <libavutil/error.h>
10
+ #include <libavutil/frame.h>
11
+ }
12
+
13
+ #include "pipeline.h"
14
+
15
+ namespace media_proxy {
16
+
17
+ struct CodecContextDeleter {
18
+ void operator()(AVCodecContext* ctx) const { avcodec_free_context(&ctx); }
19
+ };
20
+ struct FrameDeleter {
21
+ void operator()(AVFrame* frame) const { av_frame_free(&frame); }
22
+ };
23
+ struct PacketDeleter {
24
+ void operator()(AVPacket* pkt) const { av_packet_free(&pkt); }
25
+ };
26
+
27
+ using CodecContext = std::unique_ptr<AVCodecContext, CodecContextDeleter>;
28
+ using Frame = std::unique_ptr<AVFrame, FrameDeleter>;
29
+ using Packet = std::unique_ptr<AVPacket, PacketDeleter>;
30
+
31
+ inline PipelineError Fatal(const std::string& what, int code) {
32
+ std::array<char, AV_ERROR_MAX_STRING_SIZE> text{};
33
+ av_strerror(code, text.data(), text.size());
34
+ return {"MediaProxyError", what + ": " + text.data()};
35
+ }
36
+
37
+ } // namespace media_proxy
@@ -0,0 +1,133 @@
1
+ #include "h264_encoder.h"
2
+
3
+ #include <array>
4
+ #include <atomic>
5
+ #include <cstdlib>
6
+ #include <string>
7
+ #include <vector>
8
+
9
+ #include "openh264_encoder.h"
10
+ #ifdef __APPLE__
11
+ #include "videotoolbox_encoder.h"
12
+ #elif defined(_WIN32)
13
+ #include "mediafoundation_encoder.h"
14
+ #endif
15
+
16
+ namespace media_proxy {
17
+
18
+ namespace {
19
+
20
+ using Factory = std::unique_ptr<H264Encoder> (*)(const H264Params&, const H264Encoder::Sink&);
21
+
22
+ struct Backend {
23
+ const char* name;
24
+ Factory create;
25
+ std::atomic<bool> usable;
26
+ };
27
+
28
+ // Best first: platform hardware, then OpenH264 (only once its library has been installed).
29
+ std::array<Backend, 2> g_backends{{
30
+ #ifdef __APPLE__
31
+ {"h264_videotoolbox", CreateVideoToolboxEncoder, false},
32
+ #elif defined(_WIN32)
33
+ {"h264_mf", CreateMediaFoundationEncoder, false},
34
+ #else
35
+ {"", nullptr, false},
36
+ #endif
37
+ {"libopenh264", CreateOpenH264Encoder, false},
38
+ }};
39
+
40
+ // Opening a throwaway session is the only reliable test (VMs and some machines have no usable encoder). Only a
41
+ // success is cached: a failure may be transient (media-service reset), so the next call tries again.
42
+ bool Usable(Backend& backend, std::string* reason = nullptr) {
43
+ if (backend.create == nullptr) {
44
+ return false;
45
+ }
46
+ if (backend.usable.load()) {
47
+ return true;
48
+ }
49
+ constexpr H264Params kProbe{320, 240, 30, 1'000'000, 60};
50
+ try {
51
+ backend.usable.store(backend.create(kProbe, [](EncodedUnit&&) {}) != nullptr);
52
+ } catch (const PipelineError& err) {
53
+ if (reason != nullptr) {
54
+ *reason = std::string(backend.name) + ": " + err.message;
55
+ }
56
+ return false;
57
+ }
58
+ return backend.usable.load();
59
+ }
60
+
61
+ constexpr const char* kOverrideVariable = "MEDIA_PROXY_H264_ENCODER";
62
+
63
+ // The override restricts every lookup to one backend (diagnostics).
64
+ bool Selected(const Backend& backend) {
65
+ const char* forced = std::getenv(kOverrideVariable);
66
+ return backend.create != nullptr && (forced == nullptr || std::string(forced) == backend.name);
67
+ }
68
+
69
+ constexpr const char* kDocsUrl =
70
+ "https://github.com/appium/appium-ios/tree/main/packages/media-proxy#software-h264-openh264";
71
+
72
+ PipelineError Unavailable(std::vector<std::string> reasons) {
73
+ const char* forced = std::getenv(kOverrideVariable);
74
+ if (forced != nullptr) {
75
+ std::string known;
76
+ for (const Backend& backend : g_backends) {
77
+ known += backend.create == nullptr ? "" : (known.empty() ? "" : ", ") + std::string(backend.name);
78
+ }
79
+ reasons.insert(reasons.begin(), std::string(kOverrideVariable) + "='" + forced +
80
+ "' restricts the choice to that encoder (known: " + known + ")");
81
+ }
82
+ std::string message = "No H.264 encoder is available";
83
+ for (size_t i = 0; i < reasons.size(); ++i) {
84
+ message += (i == 0 ? " (" : "; ") + reasons[i] + (i + 1 == reasons.size() ? ")" : "");
85
+ }
86
+ message += ". Use codec 'mjpeg', or install Cisco's OpenH264 and set MEDIA_PROXY_OPENH264_PATH to the library; see ";
87
+ message += kDocsUrl;
88
+ return {kEncoderUnavailableError, message};
89
+ }
90
+
91
+ } // namespace
92
+
93
+ std::vector<std::string> H264EncoderNames() {
94
+ std::vector<std::string> names;
95
+ for (Backend& backend : g_backends) {
96
+ if (Selected(backend) && Usable(backend)) {
97
+ names.emplace_back(backend.name);
98
+ }
99
+ }
100
+ return names;
101
+ }
102
+
103
+ void RequireH264Encoder() {
104
+ std::vector<std::string> reasons;
105
+ for (Backend& backend : g_backends) {
106
+ std::string reason;
107
+ if (!Selected(backend)) {
108
+ continue;
109
+ }
110
+ if (Usable(backend, &reason)) {
111
+ return;
112
+ }
113
+ reasons.push_back(std::move(reason));
114
+ }
115
+ throw Unavailable(std::move(reasons));
116
+ }
117
+
118
+ std::unique_ptr<H264Encoder> CreateH264Encoder(const H264Params& params, const H264Encoder::Sink& sink) {
119
+ std::vector<std::string> reasons;
120
+ for (Backend& backend : g_backends) {
121
+ if (!Selected(backend)) {
122
+ continue;
123
+ }
124
+ try {
125
+ return backend.create(params, sink);
126
+ } catch (const PipelineError& err) {
127
+ reasons.push_back(std::string(backend.name) + ": " + err.message);
128
+ }
129
+ }
130
+ throw Unavailable(reasons);
131
+ }
132
+
133
+ } // namespace media_proxy
@@ -0,0 +1,48 @@
1
+ #pragma once
2
+
3
+ #include <cstdint>
4
+ #include <functional>
5
+ #include <memory>
6
+ #include <string>
7
+ #include <vector>
8
+
9
+ #include "pipeline.h"
10
+
11
+ struct AVFrame;
12
+
13
+ namespace media_proxy {
14
+
15
+ struct H264Params {
16
+ int width = 0;
17
+ int height = 0;
18
+ double fps = 60;
19
+ int64_t bitrate = 0;
20
+ int gopFrames = 120;
21
+ };
22
+
23
+ // Hardware H.264 encoder fed NV12 (limited range) frames. Output units (Annex-B, 4-byte start codes, parameter
24
+ // sets inline on key frames, no B-frames) are handed to the sink synchronously from Encode()/Flush().
25
+ class H264Encoder {
26
+ public:
27
+ using Sink = std::function<void(EncodedUnit&&)>;
28
+
29
+ virtual ~H264Encoder() = default;
30
+ H264Encoder() = default;
31
+ H264Encoder(const H264Encoder&) = delete;
32
+ H264Encoder& operator=(const H264Encoder&) = delete;
33
+
34
+ virtual void Encode(const AVFrame& nv12, bool forceKeyFrame) = 0;
35
+ virtual void Flush() = 0;
36
+ };
37
+
38
+ // Names of the H.264 encoders this platform's build can offer, best first (empty: none).
39
+ std::vector<std::string> H264EncoderNames();
40
+
41
+ // Throws PipelineError (MediaProxyEncoderUnavailableError) unless some encoder opens; the message says why each
42
+ // backend failed and how to install OpenH264.
43
+ void RequireH264Encoder();
44
+
45
+ // Opens the best available encoder. Throws PipelineError (MediaProxyEncoderUnavailableError) if none opens.
46
+ std::unique_ptr<H264Encoder> CreateH264Encoder(const H264Params& params, const H264Encoder::Sink& sink);
47
+
48
+ } // namespace media_proxy