@appium/media-proxy 0.0.1 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/CHANGELOG.md +33 -0
  2. package/LICENSE +201 -0
  3. package/README.md +183 -2
  4. package/THIRD_PARTY_NOTICES.md +29 -0
  5. package/binding.gyp +90 -0
  6. package/lib/src/background.d.ts +5 -0
  7. package/lib/src/background.d.ts.map +1 -0
  8. package/lib/src/background.js +16 -0
  9. package/lib/src/background.js.map +1 -0
  10. package/lib/src/capabilities.d.ts +9 -0
  11. package/lib/src/capabilities.d.ts.map +1 -0
  12. package/lib/src/capabilities.js +11 -0
  13. package/lib/src/capabilities.js.map +1 -0
  14. package/lib/src/errors.d.ts +26 -0
  15. package/lib/src/errors.d.ts.map +1 -0
  16. package/lib/src/errors.js +40 -0
  17. package/lib/src/errors.js.map +1 -0
  18. package/lib/src/framing/apst.d.ts +43 -0
  19. package/lib/src/framing/apst.d.ts.map +1 -0
  20. package/lib/src/framing/apst.js +126 -0
  21. package/lib/src/framing/apst.js.map +1 -0
  22. package/lib/src/framing/mjpeg-http.d.ts +30 -0
  23. package/lib/src/framing/mjpeg-http.d.ts.map +1 -0
  24. package/lib/src/framing/mjpeg-http.js +148 -0
  25. package/lib/src/framing/mjpeg-http.js.map +1 -0
  26. package/lib/src/framing/mjpeg-multipart.d.ts +11 -0
  27. package/lib/src/framing/mjpeg-multipart.d.ts.map +1 -0
  28. package/lib/src/framing/mjpeg-multipart.js +26 -0
  29. package/lib/src/framing/mjpeg-multipart.js.map +1 -0
  30. package/lib/src/index.d.ts +10 -0
  31. package/lib/src/index.d.ts.map +1 -0
  32. package/lib/src/index.js +7 -0
  33. package/lib/src/index.js.map +1 -0
  34. package/lib/src/native.d.ts +12 -0
  35. package/lib/src/native.d.ts.map +1 -0
  36. package/lib/src/native.js +36 -0
  37. package/lib/src/native.js.map +1 -0
  38. package/lib/src/pkg-root.d.ts +6 -0
  39. package/lib/src/pkg-root.d.ts.map +1 -0
  40. package/lib/src/pkg-root.js +12 -0
  41. package/lib/src/pkg-root.js.map +1 -0
  42. package/lib/src/queue.d.ts +34 -0
  43. package/lib/src/queue.d.ts.map +1 -0
  44. package/lib/src/queue.js +123 -0
  45. package/lib/src/queue.js.map +1 -0
  46. package/lib/src/transcoder.d.ts +81 -0
  47. package/lib/src/transcoder.d.ts.map +1 -0
  48. package/lib/src/transcoder.js +422 -0
  49. package/lib/src/transcoder.js.map +1 -0
  50. package/lib/src/types.d.ts +147 -0
  51. package/lib/src/types.d.ts.map +1 -0
  52. package/lib/src/types.js +2 -0
  53. package/lib/src/types.js.map +1 -0
  54. package/package.json +53 -4
  55. package/prebuilds/darwin-arm64/@appium+media-proxy.node +0 -0
  56. package/prebuilds/darwin-arm64/ffmpeg-LICENSE-LGPL-2.1.txt +502 -0
  57. package/prebuilds/darwin-arm64/ffmpeg-build-info.txt +3 -0
  58. package/prebuilds/darwin-arm64/libavcodec-appium.62.dylib +0 -0
  59. package/prebuilds/darwin-arm64/libavutil-appium.60.dylib +0 -0
  60. package/prebuilds/darwin-arm64/libswscale-appium.9.dylib +0 -0
  61. package/prebuilds/darwin-x64/@appium+media-proxy.node +0 -0
  62. package/prebuilds/darwin-x64/ffmpeg-LICENSE-LGPL-2.1.txt +502 -0
  63. package/prebuilds/darwin-x64/ffmpeg-build-info.txt +3 -0
  64. package/prebuilds/darwin-x64/libavcodec-appium.62.dylib +0 -0
  65. package/prebuilds/darwin-x64/libavutil-appium.60.dylib +0 -0
  66. package/prebuilds/darwin-x64/libswscale-appium.9.dylib +0 -0
  67. package/prebuilds/linux-arm64/@appium+media-proxy.node +0 -0
  68. package/prebuilds/linux-arm64/ffmpeg-LICENSE-LGPL-2.1.txt +502 -0
  69. package/prebuilds/linux-arm64/ffmpeg-build-info.txt +3 -0
  70. package/prebuilds/linux-arm64/libavcodec-appium.so.62 +0 -0
  71. package/prebuilds/linux-arm64/libavutil-appium.so.60 +0 -0
  72. package/prebuilds/linux-arm64/libswscale-appium.so.9 +0 -0
  73. package/prebuilds/linux-x64/@appium+media-proxy.node +0 -0
  74. package/prebuilds/linux-x64/ffmpeg-LICENSE-LGPL-2.1.txt +502 -0
  75. package/prebuilds/linux-x64/ffmpeg-build-info.txt +3 -0
  76. package/prebuilds/linux-x64/libavcodec-appium.so.62 +0 -0
  77. package/prebuilds/linux-x64/libavutil-appium.so.60 +0 -0
  78. package/prebuilds/linux-x64/libswscale-appium.so.9 +0 -0
  79. package/prebuilds/win32-arm64/@appium+media-proxy.node +0 -0
  80. package/prebuilds/win32-arm64/avcodec-appium-62.dll +0 -0
  81. package/prebuilds/win32-arm64/avutil-appium-60.dll +0 -0
  82. package/prebuilds/win32-arm64/ffmpeg-LICENSE-LGPL-2.1.txt +502 -0
  83. package/prebuilds/win32-arm64/ffmpeg-build-info.txt +3 -0
  84. package/prebuilds/win32-arm64/swscale-appium-9.dll +0 -0
  85. package/prebuilds/win32-x64/@appium+media-proxy.node +0 -0
  86. package/prebuilds/win32-x64/avcodec-appium-62.dll +0 -0
  87. package/prebuilds/win32-x64/avutil-appium-60.dll +0 -0
  88. package/prebuilds/win32-x64/ffmpeg-LICENSE-LGPL-2.1.txt +502 -0
  89. package/prebuilds/win32-x64/ffmpeg-build-info.txt +3 -0
  90. package/prebuilds/win32-x64/swscale-appium-9.dll +0 -0
  91. package/scripts/install.mjs +29 -0
  92. package/src/background.ts +15 -0
  93. package/src/capabilities.ts +12 -0
  94. package/src/errors.ts +45 -0
  95. package/src/framing/apst.ts +158 -0
  96. package/src/framing/mjpeg-http.ts +186 -0
  97. package/src/framing/mjpeg-multipart.ts +38 -0
  98. package/src/index.ts +43 -0
  99. package/src/native/addon.cc +402 -0
  100. package/src/native/audio_pipeline.cc +426 -0
  101. package/src/native/audio_pipeline.h +63 -0
  102. package/src/native/ffmpeg_util.h +37 -0
  103. package/src/native/h264_encoder.cc +133 -0
  104. package/src/native/h264_encoder.h +48 -0
  105. package/src/native/mediafoundation_encoder.cc +442 -0
  106. package/src/native/mediafoundation_encoder.h +14 -0
  107. package/src/native/napi_util.h +20 -0
  108. package/src/native/openh264_encoder.cc +285 -0
  109. package/src/native/openh264_encoder.h +14 -0
  110. package/src/native/pipeline.cc +555 -0
  111. package/src/native/pipeline.h +108 -0
  112. package/src/native/vendor/openh264/LICENSE +23 -0
  113. package/src/native/vendor/openh264/codec_api.h +592 -0
  114. package/src/native/vendor/openh264/codec_app_def.h +820 -0
  115. package/src/native/vendor/openh264/codec_def.h +216 -0
  116. package/src/native/vendor/openh264/codec_ver.h +15 -0
  117. package/src/native/videotoolbox_encoder.cc +314 -0
  118. package/src/native/videotoolbox_encoder.h +13 -0
  119. package/src/native.ts +51 -0
  120. package/src/pkg-root.ts +14 -0
  121. package/src/queue.ts +133 -0
  122. package/src/transcoder.ts +480 -0
  123. package/src/types.ts +158 -0
@@ -0,0 +1,555 @@
1
+ #include "pipeline.h"
2
+
3
+ #include <algorithm>
4
+ #include <array>
5
+ #include <atomic>
6
+ #include <cmath>
7
+ #include <cstdlib>
8
+ #include <exception>
9
+ #include <mutex>
10
+ #include <utility>
11
+
12
+ extern "C" {
13
+ #include <libavcodec/avcodec.h>
14
+ #include <libavutil/error.h>
15
+ #include <libavutil/frame.h>
16
+ #include <libavutil/log.h>
17
+ #include <libavutil/opt.h>
18
+ #include <libswscale/swscale.h>
19
+ }
20
+
21
+ #include "ffmpeg_util.h"
22
+ #include "h264_encoder.h"
23
+
24
+ namespace media_proxy {
25
+
26
+ namespace {
27
+
28
+ constexpr int kMaxCodecThreads = 4;
29
+ constexpr int kMaxJpegQscale = 31;
30
+ constexpr double kMaxGopFrames = 1 << 20; // keeps the int conversion defined for absurd fps
31
+ constexpr double kGopSeconds = 2; // coresim's default key-frame interval
32
+ constexpr int kMinJpegQscale = 2;
33
+ // Slot-grid frame limiter (see Impl::AcceptFrame): timestamp jitter tolerated, as a fraction of 1/fps.
34
+ constexpr int kToleranceDivisor = 4;
35
+ // A slot further ahead of the stream than this many intervals means the timeline jumped back.
36
+ constexpr int64_t kMaxSlotLeadIntervals = 2;
37
+
38
+ struct SwsDeleter {
39
+ void operator()(SwsContext* sws) const { sws_free_context(&sws); }
40
+ };
41
+
42
+ using Sws = std::unique_ptr<SwsContext, SwsDeleter>;
43
+
44
+ int CodecThreads() {
45
+ const auto cores = static_cast<int>(std::thread::hardware_concurrency());
46
+ return std::clamp(cores, 1, kMaxCodecThreads);
47
+ }
48
+
49
+ // libjpeg-style quality (1-100) to the MJPEG encoder's qscale (2 best .. 31 worst).
50
+ int JpegQscale(double quality) {
51
+ const double t = (100.0 - std::clamp(quality, 1.0, 100.0)) / 99.0;
52
+ return kMinJpegQscale + static_cast<int>(std::lround(t * (kMaxJpegQscale - kMinJpegQscale)));
53
+ }
54
+
55
+ // Copies one plane turned clockwise by `Degrees`. `T` is one sample: 1 byte, or 2 for NV12's interleaved UV. Works in
56
+ // blocks so that both the reads and the writes stay in the cache.
57
+ template <typename T, int Degrees>
58
+ void RotatePlane(const uint8_t* src, int srcStride, int width, int height, uint8_t* dst, int dstStride) {
59
+ constexpr int kBlock = 32;
60
+ for (int blockY = 0; blockY < height; blockY += kBlock) {
61
+ const int endY = std::min(blockY + kBlock, height);
62
+ for (int blockX = 0; blockX < width; blockX += kBlock) {
63
+ const int endX = std::min(blockX + kBlock, width);
64
+ for (int y = blockY; y < endY; ++y) {
65
+ const T* row = reinterpret_cast<const T*>(src + (static_cast<ptrdiff_t>(y) * srcStride));
66
+ for (int x = blockX; x < endX; ++x) {
67
+ int dstX = 0;
68
+ int dstY = 0;
69
+ if constexpr (Degrees == 90) {
70
+ dstX = height - 1 - y;
71
+ dstY = x;
72
+ } else if constexpr (Degrees == 180) {
73
+ dstX = width - 1 - x;
74
+ dstY = height - 1 - y;
75
+ } else {
76
+ dstX = y;
77
+ dstY = width - 1 - x;
78
+ }
79
+ reinterpret_cast<T*>(dst + (static_cast<ptrdiff_t>(dstY) * dstStride))[dstX] = row[x];
80
+ }
81
+ }
82
+ }
83
+ }
84
+ }
85
+
86
+ template <typename T>
87
+ void RotatePlane(const uint8_t* src, int srcStride, int width, int height, uint8_t* dst, int dstStride, int degrees) {
88
+ switch (degrees) {
89
+ case 90:
90
+ RotatePlane<T, 90>(src, srcStride, width, height, dst, dstStride);
91
+ break;
92
+ case 180:
93
+ RotatePlane<T, 180>(src, srcStride, width, height, dst, dstStride);
94
+ break;
95
+ default:
96
+ RotatePlane<T, 270>(src, srcStride, width, height, dst, dstStride);
97
+ break;
98
+ }
99
+ }
100
+
101
+ // Turns `src` clockwise by `degrees` (90, 180 or 270) into `dst`, which must be allocated for the turned size. Only
102
+ // the two formats the encoders take are supported.
103
+ void RotateFrame(const AVFrame& src, AVFrame& dst, int degrees) {
104
+ const int width = src.width;
105
+ const int height = src.height;
106
+ if (src.format == AV_PIX_FMT_NV12) {
107
+ RotatePlane<uint8_t>(src.data[0], src.linesize[0], width, height, dst.data[0], dst.linesize[0], degrees);
108
+ // Interleaved UV: one sample is a pair
109
+ RotatePlane<uint16_t>(src.data[1], src.linesize[1], width / 2, height / 2, dst.data[1], dst.linesize[1], degrees);
110
+ return;
111
+ }
112
+ RotatePlane<uint8_t>(src.data[0], src.linesize[0], width, height, dst.data[0], dst.linesize[0], degrees);
113
+ for (int plane = 1; plane <= 2; ++plane) {
114
+ RotatePlane<uint8_t>(src.data[plane], src.linesize[plane], width / 2, height / 2, dst.data[plane],
115
+ dst.linesize[plane], degrees);
116
+ }
117
+ }
118
+
119
+ } // namespace
120
+
121
+ // FFmpeg state; touched only by the worker thread after construction.
122
+ class Pipeline::Impl {
123
+ public:
124
+ Impl(const PipelineConfig& config, UnitCallback onUnit, std::atomic<bool>& keyFrameRequested, std::atomic<bool>& warm,
125
+ std::atomic<int>& rotation)
126
+ : config_(config),
127
+ isH264_(config.outputCodec == "h264"),
128
+ keyFrameRequested_(keyFrameRequested),
129
+ warm_(warm),
130
+ rotation_(rotation),
131
+ appliedRotation_(config.rotation),
132
+ onUnit_(std::move(onUnit)),
133
+ intervalMicros_(std::max<int64_t>(1, std::llround(1e6 / std::max(config.fps, 1.0)))),
134
+ toleranceMicros_(intervalMicros_ / kToleranceDivisor) {
135
+ if (!isH264_ && config.outputCodec != "mjpeg") {
136
+ throw PipelineError{"MediaProxyOptionError", "outputCodec must be 'h264' or 'mjpeg'"};
137
+ }
138
+ const AVCodec* decoder = avcodec_find_decoder(AV_CODEC_ID_HEVC);
139
+ if (isH264_) {
140
+ RequireH264Encoder();
141
+ } else {
142
+ mjpeg_ = avcodec_find_encoder(AV_CODEC_ID_MJPEG);
143
+ }
144
+ if (decoder == nullptr || (!isH264_ && mjpeg_ == nullptr)) {
145
+ throw PipelineError{kEncoderUnavailableError,
146
+ "The HEVC decoder or the output encoder is missing from this build"};
147
+ }
148
+ ConfigureLogging();
149
+ decoder_.reset(avcodec_alloc_context3(decoder));
150
+ decoder_->flags |= AV_CODEC_FLAG_LOW_DELAY;
151
+ decoder_->thread_type = FF_THREAD_SLICE;
152
+ decoder_->thread_count = CodecThreads();
153
+ if (const int ret = avcodec_open2(decoder_.get(), decoder, nullptr); ret < 0) {
154
+ throw Fatal("Could not open the HEVC decoder", ret);
155
+ }
156
+ packet_.reset(av_packet_alloc());
157
+ decoded_.reset(av_frame_alloc());
158
+ converted_.reset(av_frame_alloc());
159
+ rotated_.reset(av_frame_alloc());
160
+ sws_.reset(sws_alloc_context());
161
+ }
162
+
163
+ void Decode(const Input& input) {
164
+ if (const int ret = av_new_packet(packet_.get(), static_cast<int>(input.data.size())); ret < 0) {
165
+ throw Fatal("Could not allocate a packet", ret);
166
+ }
167
+ std::copy(input.data.begin(), input.data.end(), packet_->data);
168
+ packet_->pts = input.timestampMicros;
169
+ packet_->dts = input.timestampMicros;
170
+ const int ret = avcodec_send_packet(decoder_.get(), packet_.get());
171
+ av_packet_unref(packet_.get());
172
+ // Corrupt or undecodable units (e.g. before the first IRAP) are skipped, not fatal.
173
+ if (ret < 0 && ret != AVERROR_INVALIDDATA && ret != AVERROR(EAGAIN)) {
174
+ throw Fatal("HEVC decode failed", ret);
175
+ }
176
+ DrainDecoder();
177
+ }
178
+
179
+ [[nodiscard]] bool EncoderOpen() const { return h264_ || mjpegCtx_; }
180
+
181
+ void Flush() {
182
+ avcodec_send_packet(decoder_.get(), nullptr);
183
+ DrainDecoder();
184
+ FlushEncoder();
185
+ }
186
+
187
+ private:
188
+ // MEDIA_PROXY_FFMPEG_LOG=<av log level, e.g. 32 for info> re-enables FFmpeg's own logging for debugging.
189
+ static void ConfigureLogging() {
190
+ static std::once_flag once;
191
+ std::call_once(once, [] {
192
+ const char* text = std::getenv("MEDIA_PROXY_FFMPEG_LOG");
193
+ char* end = nullptr;
194
+ const long parsed = text != nullptr ? std::strtol(text, &end, 10) : AV_LOG_QUIET;
195
+ av_log_set_level(end == text ? AV_LOG_QUIET : static_cast<int>(parsed));
196
+ });
197
+ }
198
+
199
+ // fps is an upper bound: output frames fill slots of a 1/fps grid and each frame consumes one, so the
200
+ // long-run rate never exceeds fps while the tolerance absorbs timestamp jitter.
201
+ bool AcceptFrame(int64_t pts) {
202
+ if (nextSlotMicros_ < 0 || nextSlotMicros_ - pts > kMaxSlotLeadIntervals * intervalMicros_) {
203
+ nextSlotMicros_ = pts;
204
+ }
205
+ if (pts + toleranceMicros_ < nextSlotMicros_) {
206
+ return false;
207
+ }
208
+ nextSlotMicros_ += intervalMicros_;
209
+ while (nextSlotMicros_ - toleranceMicros_ <= pts) {
210
+ nextSlotMicros_ += intervalMicros_; // a gap in the input: don't bank slots for a later burst
211
+ }
212
+ return true;
213
+ }
214
+
215
+ void DrainDecoder() {
216
+ for (;;) {
217
+ const int ret = avcodec_receive_frame(decoder_.get(), decoded_.get());
218
+ if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
219
+ return;
220
+ }
221
+ if (ret == AVERROR_INVALIDDATA) {
222
+ continue;
223
+ }
224
+ if (ret < 0) {
225
+ throw Fatal("HEVC decode failed", ret);
226
+ }
227
+ Encode(decoded_.get());
228
+ av_frame_unref(decoded_.get());
229
+ }
230
+ }
231
+
232
+ void Encode(AVFrame* decoded) {
233
+ const int64_t pts = decoded->pts;
234
+ if (!AcceptFrame(pts)) {
235
+ return;
236
+ }
237
+ ApplyCrop(decoded);
238
+ const int rotation = rotation_.load();
239
+ if (rotation != appliedRotation_) {
240
+ appliedRotation_ = rotation;
241
+ // The picture changes shape: pictures coded before this one are of no use to a decoder that joins now
242
+ keyFrameRequested_.store(true);
243
+ }
244
+ const bool turned = rotation == 90 || rotation == 270;
245
+ const int width = ScaledSize(decoded->width);
246
+ const int height = ScaledSize(decoded->height);
247
+ const int outWidth = turned ? height : width;
248
+ const int outHeight = turned ? width : height;
249
+ OpenEncoder(outWidth, outHeight);
250
+
251
+ PrepareConverted(converted_.get(), width, height);
252
+ // Untagged input (the device doesn't tag its stream) is taken as the BT.709 the output is labelled with.
253
+ if (isH264_ && decoded->colorspace == AVCOL_SPC_UNSPECIFIED) {
254
+ decoded->colorspace = AVCOL_SPC_BT709;
255
+ }
256
+ if (const int ret = sws_scale_frame(sws_.get(), converted_.get(), decoded); ret < 0) {
257
+ throw Fatal("Could not convert the decoded frame", ret);
258
+ }
259
+ AVFrame* out = converted_.get();
260
+ if (rotation != 0) {
261
+ PrepareConverted(rotated_.get(), outWidth, outHeight);
262
+ RotateFrame(*converted_, *rotated_, rotation);
263
+ out = rotated_.get();
264
+ }
265
+ out->pts = pts;
266
+ if (isH264_) {
267
+ EncodeH264(outWidth, outHeight, *out);
268
+ return;
269
+ }
270
+ out->quality = mjpegCtx_->global_quality;
271
+ if (const int ret = avcodec_send_frame(mjpegCtx_.get(), out); ret < 0) {
272
+ throw Fatal("MJPEG encode failed", ret);
273
+ }
274
+ DrainEncoder();
275
+ }
276
+
277
+ // Keeps the top-left part of the picture, without copying: only the size changes, the rows keep their stride.
278
+ void ApplyCrop(AVFrame* frame) const {
279
+ if (config_.cropWidth <= 0 || config_.cropHeight <= 0) {
280
+ return;
281
+ }
282
+ frame->crop_right = static_cast<size_t>(std::max(0, frame->width - config_.cropWidth));
283
+ frame->crop_bottom = static_cast<size_t>(std::max(0, frame->height - config_.cropHeight));
284
+ av_frame_apply_cropping(frame, AV_FRAME_CROP_UNALIGNED);
285
+ }
286
+
287
+ // A hardware session can die mid-stream (media-service reset): reopen once and retry; the new session starts
288
+ // with a key frame. A forced key frame that comes out missing (the encoder dropped it) is requested again.
289
+ void EncodeH264(int width, int height, const AVFrame& frame) {
290
+ const bool forced = keyFrameRequested_.exchange(false);
291
+ sawKeyFrame_ = false;
292
+ try {
293
+ h264_->Encode(frame, forced);
294
+ } catch (const PipelineError&) {
295
+ h264_.reset();
296
+ OpenEncoder(width, height);
297
+ h264_->Encode(frame, true);
298
+ }
299
+ if (forced && !sawKeyFrame_) {
300
+ keyFrameRequested_.store(true);
301
+ }
302
+ }
303
+
304
+ // Keeps the aspect ratio; both sides stay even for 4:2:0.
305
+ [[nodiscard]] int ScaledSize(int side) const {
306
+ const auto scaled = static_cast<int>(std::lround(side * config_.scale / 100.0));
307
+ return std::max(2, scaled & ~1);
308
+ }
309
+
310
+ void OpenEncoder(int width, int height) {
311
+ if (isH264_) {
312
+ if (!h264_ || width != openWidth_ || height != openHeight_) {
313
+ warm_.store(false);
314
+ FlushEncoder();
315
+ h264_.reset();
316
+ H264Params params;
317
+ params.width = width;
318
+ params.height = height;
319
+ params.fps = config_.fps;
320
+ params.bitrate = static_cast<int64_t>(config_.bitrate);
321
+ params.gopFrames = static_cast<int>(std::lround(std::clamp(config_.fps * kGopSeconds, 1.0, kMaxGopFrames)));
322
+ h264_ = CreateH264Encoder(params, [this](EncodedUnit&& unit) {
323
+ sawKeyFrame_ = sawKeyFrame_ || unit.isKeyFrame;
324
+ onUnit_(std::move(unit));
325
+ });
326
+ openWidth_ = width;
327
+ openHeight_ = height;
328
+ }
329
+ return;
330
+ }
331
+ if (mjpegCtx_ && mjpegCtx_->width == width && mjpegCtx_->height == height) {
332
+ return;
333
+ }
334
+ warm_.store(false);
335
+ FlushEncoder();
336
+ mjpegCtx_.reset(avcodec_alloc_context3(mjpeg_));
337
+ mjpegCtx_->width = width;
338
+ mjpegCtx_->height = height;
339
+ mjpegCtx_->pix_fmt = AV_PIX_FMT_YUVJ420P;
340
+ mjpegCtx_->color_range = AVCOL_RANGE_JPEG;
341
+ mjpegCtx_->time_base = {1, 1000000};
342
+ mjpegCtx_->framerate = av_d2q(config_.fps, 1000000);
343
+ mjpegCtx_->flags |= AV_CODEC_FLAG_QSCALE;
344
+ mjpegCtx_->global_quality = FF_QP2LAMBDA * JpegQscale(config_.quality);
345
+ mjpegCtx_->thread_type = FF_THREAD_SLICE;
346
+ mjpegCtx_->thread_count = CodecThreads();
347
+ // Default Huffman tables: the optimal ones need a second pass over every frame.
348
+ av_opt_set(mjpegCtx_->priv_data, "huffman", "default", 0);
349
+ if (const int ret = avcodec_open2(mjpegCtx_.get(), mjpeg_, nullptr); ret < 0) {
350
+ throw Fatal("Could not open the MJPEG encoder", ret);
351
+ }
352
+ }
353
+
354
+ [[nodiscard]] AVPixelFormat EncoderPixelFormat() const { return isH264_ ? AV_PIX_FMT_NV12 : AV_PIX_FMT_YUVJ420P; }
355
+
356
+ // Reuses the buffer of `frame` unless the encoder still holds a reference to it (or the size changed).
357
+ void PrepareConverted(AVFrame* frame, int width, int height) const {
358
+ if (frame->buf[0] != nullptr && frame->width == width && frame->height == height &&
359
+ av_frame_is_writable(frame) != 0) {
360
+ return;
361
+ }
362
+ av_frame_unref(frame);
363
+ frame->format = EncoderPixelFormat();
364
+ frame->width = width;
365
+ frame->height = height;
366
+ frame->color_range = isH264_ ? AVCOL_RANGE_MPEG : AVCOL_RANGE_JPEG;
367
+ // The H.264 output is tagged BT.709; an unspecified target would make swscale convert to BT.601.
368
+ frame->colorspace = isH264_ ? AVCOL_SPC_BT709 : AVCOL_SPC_UNSPECIFIED;
369
+ if (const int ret = av_frame_get_buffer(frame, 0); ret < 0) {
370
+ throw Fatal("Could not allocate the output frame", ret);
371
+ }
372
+ }
373
+
374
+ void FlushEncoder() {
375
+ if (h264_) {
376
+ h264_->Flush();
377
+ } else if (mjpegCtx_) {
378
+ avcodec_send_frame(mjpegCtx_.get(), nullptr);
379
+ DrainEncoder();
380
+ }
381
+ }
382
+
383
+ void DrainEncoder() {
384
+ for (;;) {
385
+ const int ret = avcodec_receive_packet(mjpegCtx_.get(), packet_.get());
386
+ if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
387
+ return;
388
+ }
389
+ if (ret < 0) {
390
+ throw Fatal("MJPEG encode failed", ret);
391
+ }
392
+ EncodedUnit unit;
393
+ unit.data.assign(packet_->data, packet_->data + packet_->size);
394
+ unit.isKeyFrame = (packet_->flags & AV_PKT_FLAG_KEY) != 0;
395
+ unit.timestampMicros = packet_->pts;
396
+ av_packet_unref(packet_.get());
397
+ onUnit_(std::move(unit));
398
+ }
399
+ }
400
+
401
+ PipelineConfig config_;
402
+ bool isH264_;
403
+ std::atomic<bool>& keyFrameRequested_;
404
+ std::atomic<bool>& warm_;
405
+ std::atomic<int>& rotation_;
406
+ int appliedRotation_;
407
+ UnitCallback onUnit_;
408
+ int64_t intervalMicros_;
409
+ int64_t toleranceMicros_;
410
+ int64_t nextSlotMicros_ = -1;
411
+ const AVCodec* mjpeg_ = nullptr;
412
+ std::unique_ptr<H264Encoder> h264_;
413
+ bool sawKeyFrame_ = false;
414
+ int openWidth_ = 0;
415
+ int openHeight_ = 0;
416
+ CodecContext decoder_;
417
+ CodecContext mjpegCtx_;
418
+ Packet packet_;
419
+ Frame decoded_;
420
+ Frame converted_;
421
+ Frame rotated_;
422
+ Sws sws_;
423
+ };
424
+
425
+ Capabilities Pipeline::Probe() {
426
+ Capabilities caps;
427
+ if (const AVCodec* decoder = avcodec_find_decoder(AV_CODEC_ID_HEVC); decoder != nullptr) {
428
+ caps.decoders.emplace_back(decoder->name);
429
+ }
430
+ if (const AVCodec* encoder = avcodec_find_encoder(AV_CODEC_ID_MJPEG); encoder != nullptr) {
431
+ caps.encoders.emplace_back(encoder->name);
432
+ }
433
+ if (const AVCodec* decoder = avcodec_find_decoder(AV_CODEC_ID_AAC); decoder != nullptr) {
434
+ caps.decoders.emplace_back(decoder->name);
435
+ }
436
+ if (const AVCodec* encoder = avcodec_find_encoder(AV_CODEC_ID_AAC); encoder != nullptr) {
437
+ caps.encoders.emplace_back(encoder->name);
438
+ }
439
+ for (std::string& name : H264EncoderNames()) {
440
+ caps.encoders.push_back(std::move(name));
441
+ }
442
+ return caps;
443
+ }
444
+
445
+ Pipeline::Pipeline(const PipelineConfig& config, UnitCallback onUnit, ErrorCallback onError,
446
+ std::atomic<bool>& keyFrameRequested)
447
+ : rotation_(config.rotation),
448
+ onError_(std::move(onError)),
449
+ impl_(std::make_unique<Impl>(config, std::move(onUnit), keyFrameRequested, warm_, rotation_)) {
450
+ worker_ = std::thread([this] { Run(); });
451
+ }
452
+
453
+ Pipeline::~Pipeline() {
454
+ {
455
+ const std::scoped_lock<std::mutex> lock(mutex_);
456
+ aborting_ = true;
457
+ }
458
+ wake_.notify_all();
459
+ if (worker_.joinable()) {
460
+ worker_.join();
461
+ }
462
+ }
463
+
464
+ void Pipeline::Push(std::vector<uint8_t> accessUnit, int64_t timestampMicros, bool isKeyFrame) {
465
+ {
466
+ const std::scoped_lock<std::mutex> lock(mutex_);
467
+ if (finishing_ || aborting_) {
468
+ return;
469
+ }
470
+ if (awaitingKeyFrame_) {
471
+ if (!isKeyFrame) {
472
+ return;
473
+ }
474
+ awaitingKeyFrame_ = false;
475
+ }
476
+ // While an encoder is opening (first frame, rotation) nothing is on screen yet: queue more and catch up fast.
477
+ if (queue_.size() >= (warm_.load() ? kMaxQueuedUnits : kMaxWarmupUnits)) {
478
+ // Falling behind: drop the backlog and resume at the next key frame (P-frames can't decode alone).
479
+ queue_.clear();
480
+ if (!isKeyFrame) {
481
+ awaitingKeyFrame_ = true;
482
+ return;
483
+ }
484
+ }
485
+ queue_.push_back({std::move(accessUnit), timestampMicros, isKeyFrame});
486
+ }
487
+ wake_.notify_one();
488
+ }
489
+
490
+ void Pipeline::Finish(DoneCallback onDone) {
491
+ {
492
+ const std::scoped_lock<std::mutex> lock(mutex_);
493
+ onDone_ = std::move(onDone);
494
+ finishing_ = true;
495
+ }
496
+ wake_.notify_all();
497
+ }
498
+
499
+ void Pipeline::Run() {
500
+ bool failed = false;
501
+ const auto fail = [&](PipelineError&& err) {
502
+ failed = true;
503
+ onError_(std::move(err));
504
+ };
505
+ // Nothing may escape the worker: an uncaught exception would terminate the whole process.
506
+ const auto attempt = [&](const auto& body) {
507
+ try {
508
+ body();
509
+ } catch (PipelineError& err) {
510
+ fail(std::move(err));
511
+ } catch (const std::exception& err) {
512
+ fail({"MediaProxyError", err.what()});
513
+ }
514
+ };
515
+ for (;;) {
516
+ Input input;
517
+ {
518
+ std::unique_lock<std::mutex> lock(mutex_);
519
+ wake_.wait(lock, [this] { return aborting_ || finishing_ || !queue_.empty(); });
520
+ if (aborting_) {
521
+ return;
522
+ }
523
+ if (queue_.empty()) {
524
+ break;
525
+ }
526
+ input = std::move(queue_.front());
527
+ queue_.pop_front();
528
+ }
529
+ if (failed) {
530
+ continue;
531
+ }
532
+ attempt([&] { impl_->Decode(input); });
533
+ {
534
+ // Caught up again (the backlog from an encoder start-up is gone): back to the tight steady-state limit.
535
+ const std::scoped_lock<std::mutex> lock(mutex_);
536
+ if (impl_->EncoderOpen() && queue_.size() < kMaxQueuedUnits) {
537
+ warm_.store(true);
538
+ }
539
+ }
540
+ }
541
+ if (!failed) {
542
+ attempt([&] { impl_->Flush(); });
543
+ }
544
+ impl_.reset(); // free the codecs (and their threads) now, not when the owner is garbage collected
545
+ DoneCallback done;
546
+ {
547
+ const std::scoped_lock<std::mutex> lock(mutex_);
548
+ done = std::move(onDone_);
549
+ }
550
+ if (done) {
551
+ done();
552
+ }
553
+ }
554
+
555
+ } // namespace media_proxy
@@ -0,0 +1,108 @@
1
+ #pragma once
2
+
3
+ #include <array>
4
+ #include <atomic>
5
+ #include <condition_variable>
6
+ #include <cstdint>
7
+ #include <deque>
8
+ #include <functional>
9
+ #include <memory>
10
+ #include <mutex>
11
+ #include <string>
12
+ #include <thread>
13
+ #include <vector>
14
+
15
+ namespace media_proxy {
16
+
17
+ // Typed error name (src/errors.ts) and the Annex-B start code shared by the native sources.
18
+ inline constexpr const char* kEncoderUnavailableError = "MediaProxyEncoderUnavailableError";
19
+ inline constexpr std::array<uint8_t, 4> kAnnexBStartCode = {0, 0, 0, 1};
20
+
21
+ struct PipelineConfig {
22
+ std::string outputCodec; // "h264" | "mjpeg"
23
+ // Doubles: the TS layer accepts fractional values (e.g. fps 29.97, scale 33.3).
24
+ double fps = 60;
25
+ double bitrate = 0;
26
+ double quality = 80;
27
+ double scale = 100;
28
+ // The visible part of the decoded picture, taken from its top-left corner; 0 keeps the whole picture. The device
29
+ // pads its stream to a multiple of 16 with black.
30
+ int cropWidth = 0;
31
+ int cropHeight = 0;
32
+ // Clockwise degrees (0, 90, 180 or 270) that make the decoded picture upright; changeable with SetRotation().
33
+ int rotation = 0;
34
+ };
35
+
36
+ struct EncodedUnit {
37
+ std::vector<uint8_t> data;
38
+ bool isKeyFrame = false;
39
+ int64_t timestampMicros = 0;
40
+ };
41
+
42
+ // Error names match the typed errors in src/errors.ts.
43
+ struct PipelineError {
44
+ std::string name;
45
+ std::string message;
46
+ };
47
+
48
+ // Codecs this build can actually use, probed from the loaded FFmpeg.
49
+ struct Capabilities {
50
+ std::vector<std::string> decoders;
51
+ std::vector<std::string> encoders;
52
+ };
53
+
54
+ // HEVC access units in, encoded units out, on a dedicated worker thread. Pure C++/FFmpeg: no N-API.
55
+ class Pipeline {
56
+ public:
57
+ static Capabilities Probe();
58
+
59
+ using UnitCallback = std::function<void(EncodedUnit&&)>;
60
+ using ErrorCallback = std::function<void(PipelineError&&)>;
61
+ using DoneCallback = std::function<void()>;
62
+
63
+ // Throws PipelineError when the requested encoder/decoder is unavailable. Setting `keyFrameRequested` (from any
64
+ // thread, must outlive the pipeline) makes the H.264 encoder emit a key frame next; it is ignored for MJPEG.
65
+ Pipeline(const PipelineConfig& config, UnitCallback onUnit, ErrorCallback onError,
66
+ std::atomic<bool>& keyFrameRequested);
67
+ ~Pipeline();
68
+ Pipeline(const Pipeline&) = delete;
69
+ Pipeline& operator=(const Pipeline&) = delete;
70
+
71
+ // Changes the rotation (see PipelineConfig) from any thread; takes effect with the next frame and, as the picture
72
+ // changes shape, makes the encoder start a new stream with a key frame.
73
+ void SetRotation(int degrees) { rotation_.store(degrees); }
74
+
75
+ // One access unit with 4-byte start codes. Non-blocking; overflow drops to the next key frame.
76
+ void Push(std::vector<uint8_t> accessUnit, int64_t timestampMicros, bool isKeyFrame);
77
+ // Drains queued input, flushes the codecs, then calls `onDone` from the worker thread (once).
78
+ void Finish(DoneCallback onDone);
79
+
80
+ static constexpr size_t kMaxQueuedUnits = 8;
81
+ // Limit until the backlog behind a (re)opening encoder has drained (start-up, rotation).
82
+ static constexpr size_t kMaxWarmupUnits = 120;
83
+
84
+ private:
85
+ struct Input {
86
+ std::vector<uint8_t> data;
87
+ int64_t timestampMicros = 0;
88
+ bool isKeyFrame = false;
89
+ };
90
+ class Impl;
91
+
92
+ void Run();
93
+
94
+ std::mutex mutex_;
95
+ std::condition_variable wake_;
96
+ std::deque<Input> queue_;
97
+ bool awaitingKeyFrame_ = false;
98
+ bool finishing_ = false;
99
+ bool aborting_ = false;
100
+ std::atomic<bool> warm_{false}; // false while an encoder opens and the backlog behind it drains
101
+ std::atomic<int> rotation_;
102
+ DoneCallback onDone_;
103
+ ErrorCallback onError_;
104
+ std::unique_ptr<Impl> impl_;
105
+ std::thread worker_;
106
+ };
107
+
108
+ } // namespace media_proxy
@@ -0,0 +1,23 @@
1
+ Copyright (c) 2013, Cisco Systems
2
+ All rights reserved.
3
+
4
+ Redistribution and use in source and binary forms, with or without modification,
5
+ are permitted provided that the following conditions are met:
6
+
7
+ * Redistributions of source code must retain the above copyright notice, this
8
+ list of conditions and the following disclaimer.
9
+
10
+ * Redistributions in binary form must reproduce the above copyright notice, this
11
+ list of conditions and the following disclaimer in the documentation and/or
12
+ other materials provided with the distribution.
13
+
14
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
15
+ ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
16
+ WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
17
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
18
+ ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
19
+ (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
20
+ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
21
+ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
23
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.