@appium/media-proxy 0.0.0-stage → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/CHANGELOG.md +33 -0
  2. package/LICENSE +201 -0
  3. package/README.md +184 -2
  4. package/THIRD_PARTY_NOTICES.md +29 -0
  5. package/binding.gyp +90 -0
  6. package/lib/src/background.d.ts +5 -0
  7. package/lib/src/background.d.ts.map +1 -0
  8. package/lib/src/background.js +16 -0
  9. package/lib/src/background.js.map +1 -0
  10. package/lib/src/capabilities.d.ts +9 -0
  11. package/lib/src/capabilities.d.ts.map +1 -0
  12. package/lib/src/capabilities.js +11 -0
  13. package/lib/src/capabilities.js.map +1 -0
  14. package/lib/src/errors.d.ts +26 -0
  15. package/lib/src/errors.d.ts.map +1 -0
  16. package/lib/src/errors.js +40 -0
  17. package/lib/src/errors.js.map +1 -0
  18. package/lib/src/framing/apst.d.ts +43 -0
  19. package/lib/src/framing/apst.d.ts.map +1 -0
  20. package/lib/src/framing/apst.js +126 -0
  21. package/lib/src/framing/apst.js.map +1 -0
  22. package/lib/src/framing/mjpeg-http.d.ts +30 -0
  23. package/lib/src/framing/mjpeg-http.d.ts.map +1 -0
  24. package/lib/src/framing/mjpeg-http.js +148 -0
  25. package/lib/src/framing/mjpeg-http.js.map +1 -0
  26. package/lib/src/framing/mjpeg-multipart.d.ts +11 -0
  27. package/lib/src/framing/mjpeg-multipart.d.ts.map +1 -0
  28. package/lib/src/framing/mjpeg-multipart.js +26 -0
  29. package/lib/src/framing/mjpeg-multipart.js.map +1 -0
  30. package/lib/src/index.d.ts +10 -0
  31. package/lib/src/index.d.ts.map +1 -0
  32. package/lib/src/index.js +7 -0
  33. package/lib/src/index.js.map +1 -0
  34. package/lib/src/native.d.ts +12 -0
  35. package/lib/src/native.d.ts.map +1 -0
  36. package/lib/src/native.js +36 -0
  37. package/lib/src/native.js.map +1 -0
  38. package/lib/src/pkg-root.d.ts +6 -0
  39. package/lib/src/pkg-root.d.ts.map +1 -0
  40. package/lib/src/pkg-root.js +12 -0
  41. package/lib/src/pkg-root.js.map +1 -0
  42. package/lib/src/queue.d.ts +34 -0
  43. package/lib/src/queue.d.ts.map +1 -0
  44. package/lib/src/queue.js +123 -0
  45. package/lib/src/queue.js.map +1 -0
  46. package/lib/src/transcoder.d.ts +81 -0
  47. package/lib/src/transcoder.d.ts.map +1 -0
  48. package/lib/src/transcoder.js +422 -0
  49. package/lib/src/transcoder.js.map +1 -0
  50. package/lib/src/types.d.ts +147 -0
  51. package/lib/src/types.d.ts.map +1 -0
  52. package/lib/src/types.js +2 -0
  53. package/lib/src/types.js.map +1 -0
  54. package/package.json +65 -4
  55. package/prebuilds/darwin-arm64/@appium+media-proxy.node +0 -0
  56. package/prebuilds/darwin-arm64/ffmpeg-LICENSE-LGPL-2.1.txt +502 -0
  57. package/prebuilds/darwin-arm64/ffmpeg-build-info.txt +3 -0
  58. package/prebuilds/darwin-arm64/libavcodec-appium.62.dylib +0 -0
  59. package/prebuilds/darwin-arm64/libavutil-appium.60.dylib +0 -0
  60. package/prebuilds/darwin-arm64/libswscale-appium.9.dylib +0 -0
  61. package/prebuilds/darwin-x64/@appium+media-proxy.node +0 -0
  62. package/prebuilds/darwin-x64/ffmpeg-LICENSE-LGPL-2.1.txt +502 -0
  63. package/prebuilds/darwin-x64/ffmpeg-build-info.txt +3 -0
  64. package/prebuilds/darwin-x64/libavcodec-appium.62.dylib +0 -0
  65. package/prebuilds/darwin-x64/libavutil-appium.60.dylib +0 -0
  66. package/prebuilds/darwin-x64/libswscale-appium.9.dylib +0 -0
  67. package/prebuilds/linux-arm64/@appium+media-proxy.node +0 -0
  68. package/prebuilds/linux-arm64/ffmpeg-LICENSE-LGPL-2.1.txt +502 -0
  69. package/prebuilds/linux-arm64/ffmpeg-build-info.txt +3 -0
  70. package/prebuilds/linux-arm64/libavcodec-appium.so.62 +0 -0
  71. package/prebuilds/linux-arm64/libavutil-appium.so.60 +0 -0
  72. package/prebuilds/linux-arm64/libswscale-appium.so.9 +0 -0
  73. package/prebuilds/linux-x64/@appium+media-proxy.node +0 -0
  74. package/prebuilds/linux-x64/ffmpeg-LICENSE-LGPL-2.1.txt +502 -0
  75. package/prebuilds/linux-x64/ffmpeg-build-info.txt +3 -0
  76. package/prebuilds/linux-x64/libavcodec-appium.so.62 +0 -0
  77. package/prebuilds/linux-x64/libavutil-appium.so.60 +0 -0
  78. package/prebuilds/linux-x64/libswscale-appium.so.9 +0 -0
  79. package/prebuilds/win32-arm64/@appium+media-proxy.node +0 -0
  80. package/prebuilds/win32-arm64/avcodec-appium-62.dll +0 -0
  81. package/prebuilds/win32-arm64/avutil-appium-60.dll +0 -0
  82. package/prebuilds/win32-arm64/ffmpeg-LICENSE-LGPL-2.1.txt +502 -0
  83. package/prebuilds/win32-arm64/ffmpeg-build-info.txt +3 -0
  84. package/prebuilds/win32-arm64/swscale-appium-9.dll +0 -0
  85. package/prebuilds/win32-x64/@appium+media-proxy.node +0 -0
  86. package/prebuilds/win32-x64/avcodec-appium-62.dll +0 -0
  87. package/prebuilds/win32-x64/avutil-appium-60.dll +0 -0
  88. package/prebuilds/win32-x64/ffmpeg-LICENSE-LGPL-2.1.txt +502 -0
  89. package/prebuilds/win32-x64/ffmpeg-build-info.txt +3 -0
  90. package/prebuilds/win32-x64/swscale-appium-9.dll +0 -0
  91. package/scripts/install.mjs +29 -0
  92. package/src/background.ts +15 -0
  93. package/src/capabilities.ts +12 -0
  94. package/src/errors.ts +45 -0
  95. package/src/framing/apst.ts +158 -0
  96. package/src/framing/mjpeg-http.ts +186 -0
  97. package/src/framing/mjpeg-multipart.ts +38 -0
  98. package/src/index.ts +43 -0
  99. package/src/native/addon.cc +402 -0
  100. package/src/native/audio_pipeline.cc +426 -0
  101. package/src/native/audio_pipeline.h +63 -0
  102. package/src/native/ffmpeg_util.h +37 -0
  103. package/src/native/h264_encoder.cc +133 -0
  104. package/src/native/h264_encoder.h +48 -0
  105. package/src/native/mediafoundation_encoder.cc +442 -0
  106. package/src/native/mediafoundation_encoder.h +14 -0
  107. package/src/native/napi_util.h +20 -0
  108. package/src/native/openh264_encoder.cc +285 -0
  109. package/src/native/openh264_encoder.h +14 -0
  110. package/src/native/pipeline.cc +555 -0
  111. package/src/native/pipeline.h +108 -0
  112. package/src/native/vendor/openh264/LICENSE +23 -0
  113. package/src/native/vendor/openh264/codec_api.h +592 -0
  114. package/src/native/vendor/openh264/codec_app_def.h +820 -0
  115. package/src/native/vendor/openh264/codec_def.h +216 -0
  116. package/src/native/vendor/openh264/codec_ver.h +15 -0
  117. package/src/native/videotoolbox_encoder.cc +314 -0
  118. package/src/native/videotoolbox_encoder.h +13 -0
  119. package/src/native.ts +51 -0
  120. package/src/pkg-root.ts +14 -0
  121. package/src/queue.ts +133 -0
  122. package/src/transcoder.ts +480 -0
  123. package/src/types.ts +158 -0
@@ -0,0 +1,216 @@
1
+ /*!
2
+ * \copy
3
+ * Copyright (c) 2013, Cisco Systems
4
+ * All rights reserved.
5
+ *
6
+ * Redistribution and use in source and binary forms, with or without
7
+ * modification, are permitted provided that the following conditions
8
+ * are met:
9
+ *
10
+ * * Redistributions of source code must retain the above copyright
11
+ * notice, this list of conditions and the following disclaimer.
12
+ *
13
+ * * Redistributions in binary form must reproduce the above copyright
14
+ * notice, this list of conditions and the following disclaimer in
15
+ * the documentation and/or other materials provided with the
16
+ * distribution.
17
+ *
18
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
21
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
22
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
23
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
24
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
26
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29
+ * POSSIBILITY OF SUCH DAMAGE.
30
+ *
31
+ */
32
+
33
+ #ifndef WELS_VIDEO_CODEC_DEFINITION_H__
34
+ #define WELS_VIDEO_CODEC_DEFINITION_H__
35
+
36
+ /**
37
+ * @file codec_def.h
38
+ */
39
+
40
+ /**
41
+ * @brief Enumerate the type of video format
42
+ */
43
+ typedef enum {
44
+ videoFormatRGB = 1, ///< rgb color formats
45
+ videoFormatRGBA = 2,
46
+ videoFormatRGB555 = 3,
47
+ videoFormatRGB565 = 4,
48
+ videoFormatBGR = 5,
49
+ videoFormatBGRA = 6,
50
+ videoFormatABGR = 7,
51
+ videoFormatARGB = 8,
52
+
53
+ videoFormatYUY2 = 20, ///< yuv color formats
54
+ videoFormatYVYU = 21,
55
+ videoFormatUYVY = 22,
56
+ videoFormatI420 = 23, ///< the same as IYUV
57
+ videoFormatYV12 = 24,
58
+ videoFormatInternal = 25, ///< only used in SVC decoder testbed
59
+
60
+ videoFormatNV12 = 26, ///< new format for output by DXVA decoding
61
+
62
+ videoFormatVFlip = 0x80000000
63
+ } EVideoFormatType;
64
+
65
+ /**
66
+ * @brief Enumerate video frame type
67
+ */
68
+ typedef enum {
69
+ videoFrameTypeInvalid, ///< encoder not ready or parameters are invalidate
70
+ videoFrameTypeIDR, ///< IDR frame in H.264
71
+ videoFrameTypeI, ///< I frame type
72
+ videoFrameTypeP, ///< P frame type
73
+ videoFrameTypeSkip, ///< skip the frame based encoder kernel
74
+ videoFrameTypeIPMixed ///< a frame where I and P slices are mixing, not supported yet
75
+ } EVideoFrameType;
76
+
77
+ /**
78
+ * @brief Enumerate return type
79
+ */
80
+ typedef enum {
81
+ cmResultSuccess, ///< successful
82
+ cmInitParaError, ///< parameters are invalid
83
+ cmUnknownReason,
84
+ cmMallocMemeError, ///< malloc a memory error
85
+ cmInitExpected, ///< initial action is expected
86
+ cmUnsupportedData
87
+ } CM_RETURN;
88
+
89
+ /**
90
+ * @brief Enumulate the nal unit type
91
+ */
92
+ enum ENalUnitType {
93
+ NAL_UNKNOWN = 0,
94
+ NAL_SLICE = 1,
95
+ NAL_SLICE_DPA = 2,
96
+ NAL_SLICE_DPB = 3,
97
+ NAL_SLICE_DPC = 4,
98
+ NAL_SLICE_IDR = 5, ///< ref_idc != 0
99
+ NAL_SEI = 6, ///< ref_idc == 0
100
+ NAL_SPS = 7,
101
+ NAL_PPS = 8
102
+ ///< ref_idc == 0 for 6,9,10,11,12
103
+ };
104
+
105
+ /**
106
+ * @brief NRI: eNalRefIdc
107
+ */
108
+ enum ENalPriority {
109
+ NAL_PRIORITY_DISPOSABLE = 0,
110
+ NAL_PRIORITY_LOW = 1,
111
+ NAL_PRIORITY_HIGH = 2,
112
+ NAL_PRIORITY_HIGHEST = 3
113
+ };
114
+
115
+ #define IS_PARAMETER_SET_NAL(eNalRefIdc, eNalType) \
116
+ ( (eNalRefIdc == NAL_PRIORITY_HIGHEST) && (eNalType == (NAL_SPS|NAL_PPS) || eNalType == NAL_SPS) )
117
+
118
+ #define IS_IDR_NAL(eNalRefIdc, eNalType) \
119
+ ( (eNalRefIdc == NAL_PRIORITY_HIGHEST) && (eNalType == NAL_SLICE_IDR) )
120
+
121
+ #define FRAME_NUM_PARAM_SET (-1)
122
+ #define FRAME_NUM_IDR 0
123
+
124
+ /**
125
+ * @brief eDeblockingIdc
126
+ */
127
+ enum {
128
+ DEBLOCKING_IDC_0 = 0,
129
+ DEBLOCKING_IDC_1 = 1,
130
+ DEBLOCKING_IDC_2 = 2
131
+ };
132
+ #define DEBLOCKING_OFFSET (6)
133
+ #define DEBLOCKING_OFFSET_MINUS (-6)
134
+
135
+ /* Error Tools definition */
136
+ typedef unsigned short ERR_TOOL;
137
+
138
+ /**
139
+ @brief to do
140
+ */
141
+ enum {
142
+ ET_NONE = 0x00, ///< NONE Error Tools
143
+ ET_IP_SCALE = 0x01, ///< IP Scalable
144
+ ET_FMO = 0x02, ///< Flexible Macroblock Ordering
145
+ ET_IR_R1 = 0x04, ///< Intra Refresh in predifined 2% MB
146
+ ET_IR_R2 = 0x08, ///< Intra Refresh in predifined 5% MB
147
+ ET_IR_R3 = 0x10, ///< Intra Refresh in predifined 10% MB
148
+ ET_FEC_HALF = 0x20, ///< Forward Error Correction in 50% redundency mode
149
+ ET_FEC_FULL = 0x40, ///< Forward Error Correction in 100% redundency mode
150
+ ET_RFS = 0x80 ///< Reference Frame Selection
151
+ };
152
+
153
+ /**
154
+ * @brief Information of coded Slice(=NAL)(s)
155
+ */
156
+ typedef struct SliceInformation {
157
+ unsigned char* pBufferOfSlices; ///< base buffer of coded slice(s)
158
+ int iCodedSliceCount; ///< number of coded slices
159
+ unsigned int* pLengthOfSlices; ///< array of slices length accordingly by number of slice
160
+ int iFecType; ///< FEC type[0, 50%FEC, 100%FEC]
161
+ unsigned char uiSliceIdx; ///< index of slice in frame [FMO: 0,..,uiSliceCount-1; No FMO: 0]
162
+ unsigned char uiSliceCount; ///< count number of slice in frame [FMO: 2-8; No FMO: 1]
163
+ char iFrameIndex; ///< index of frame[-1, .., idr_interval-1]
164
+ unsigned char uiNalRefIdc; ///< NRI, priority level of slice(NAL)
165
+ unsigned char uiNalType; ///< NAL type
166
+ unsigned char
167
+ uiContainingFinalNal; ///< whether final NAL is involved in buffer of coded slices, flag used in Pause feature in T27
168
+ } SliceInfo, *PSliceInfo;
169
+
170
+ /**
171
+ * @brief thresholds of the initial, maximal and minimal rate
172
+ */
173
+ typedef struct {
174
+ int iWidth; ///< frame width
175
+ int iHeight; ///< frame height
176
+ int iThresholdOfInitRate; ///< threshold of initial rate
177
+ int iThresholdOfMaxRate; ///< threshold of maximal rate
178
+ int iThresholdOfMinRate; ///< threshold of minimal rate
179
+ int iMinThresholdFrameRate; ///< min frame rate min
180
+ int iSkipFrameRate; ///< skip to frame rate min
181
+ int iSkipFrameStep; ///< how many frames to skip
182
+ } SRateThresholds, *PRateThresholds;
183
+
184
+ /**
185
+ * @brief Structure for decoder memery
186
+ */
187
+ typedef struct TagSysMemBuffer {
188
+ int iWidth; ///< width of decoded pic for display
189
+ int iHeight; ///< height of decoded pic for display
190
+ int iFormat; ///< type is "EVideoFormatType"
191
+ int iStride[2]; ///< stride of 2 component
192
+ } SSysMEMBuffer;
193
+
194
+ /**
195
+ * @brief Buffer info
196
+ */
197
+ typedef struct TagBufferInfo {
198
+ int iBufferStatus; ///< 0: one frame data is not ready; 1: one frame data is ready
199
+ unsigned long long uiInBsTimeStamp; ///< input BS timestamp
200
+ unsigned long long uiOutYuvTimeStamp; ///< output YUV timestamp, when bufferstatus is 1
201
+ union {
202
+ SSysMEMBuffer sSystemBuffer; ///< memory info for one picture
203
+ } UsrData; ///< output buffer info
204
+ unsigned char* pDst[3]; //point to picture YUV data
205
+ } SBufferInfo;
206
+
207
+
208
+ /**
209
+ * @brief In a GOP, multiple of the key frame number, derived from
210
+ * the number of layers(index or array below)
211
+ */
212
+ static const char kiKeyNumMultiple[] = {
213
+ 1, 1, 2, 4, 8, 16,
214
+ };
215
+
216
+ #endif//WELS_VIDEO_CODEC_DEFINITION_H__
@@ -0,0 +1,15 @@
1
+ //The current file is auto-generated by script: generate_codec_ver.sh
2
+ #ifndef CODEC_VER_H
3
+ #define CODEC_VER_H
4
+
5
+ #include "codec_app_def.h"
6
+
7
+ static const OpenH264Version g_stCodecVersion = {2, 6, 0, 2502};
8
+ static const char* const g_strCodecVer = "OpenH264 version:2.6.0.2502";
9
+
10
+ #define OPENH264_MAJOR (2)
11
+ #define OPENH264_MINOR (6)
12
+ #define OPENH264_REVISION (0)
13
+ #define OPENH264_RESERVED (2502)
14
+
15
+ #endif // CODEC_VER_H
@@ -0,0 +1,314 @@
1
+ #include "videotoolbox_encoder.h"
2
+
3
+ #include <CoreFoundation/CoreFoundation.h>
4
+ #include <CoreMedia/CoreMedia.h>
5
+ #include <CoreVideo/CoreVideo.h>
6
+ #include <VideoToolbox/VideoToolbox.h>
7
+
8
+ #include <algorithm>
9
+ #include <array>
10
+ #include <cstdint>
11
+ #include <cstring>
12
+ #include <mutex>
13
+ #include <string>
14
+ #include <type_traits>
15
+ #include <utility>
16
+
17
+ extern "C" {
18
+ #include <libavutil/frame.h>
19
+ #include <libavutil/log.h>
20
+ }
21
+
22
+ namespace media_proxy {
23
+
24
+ namespace {
25
+
26
+ constexpr int32_t kMicrosTimescale = 1000000;
27
+ constexpr size_t kMinNalLengthSize = 1;
28
+ constexpr size_t kMaxNalLengthSize = 4;
29
+ constexpr size_t kByteBits = 8;
30
+ constexpr Boolean kNo = 0;
31
+ constexpr size_t kKeyNameSize = 64;
32
+
33
+ struct CfDeleter {
34
+ void operator()(CFTypeRef ref) const { CFRelease(ref); }
35
+ };
36
+ template <typename T>
37
+ using Cf = std::unique_ptr<std::remove_pointer_t<T>, CfDeleter>;
38
+
39
+ PipelineError Failure(const char* what, OSStatus status) {
40
+ return {"MediaProxyError", std::string("VideoToolbox ") + what + " failed (OSStatus " + std::to_string(status) + ")"};
41
+ }
42
+
43
+ Cf<CFNumberRef> Number(int32_t value) { return Cf<CFNumberRef>(CFNumberCreate(nullptr, kCFNumberSInt32Type, &value)); }
44
+
45
+ Cf<CFNumberRef> Number(double value) { return Cf<CFNumberRef>(CFNumberCreate(nullptr, kCFNumberDoubleType, &value)); }
46
+
47
+ Cf<CFDictionaryRef> Dictionary(const void* key, const void* value) {
48
+ return Cf<CFDictionaryRef>(
49
+ CFDictionaryCreate(nullptr, &key, &value, 1, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks));
50
+ }
51
+
52
+ bool IsKeyFrame(CMSampleBufferRef sample) {
53
+ CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sample, kNo);
54
+ if (attachments == nullptr || CFArrayGetCount(attachments) == 0) {
55
+ return true;
56
+ }
57
+ const auto* first = static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0));
58
+ const void* notSync = CFDictionaryGetValue(first, kCMSampleAttachmentKey_NotSync);
59
+ return notSync == nullptr || CFBooleanGetValue(static_cast<CFBooleanRef>(notSync)) == 0U;
60
+ }
61
+
62
+ void AppendNal(std::vector<uint8_t>& out, const uint8_t* data, size_t size) {
63
+ out.insert(out.end(), kAnnexBStartCode.begin(), kAnnexBStartCode.end());
64
+ out.insert(out.end(), data, data + size);
65
+ }
66
+
67
+ class VideoToolboxEncoder final : public H264Encoder {
68
+ public:
69
+ VideoToolboxEncoder(const H264Params& params, H264Encoder::Sink sink) : params_(params), sink_(std::move(sink)) {
70
+ try {
71
+ Open(true);
72
+ } catch (const PipelineError&) {
73
+ Open(false); // older systems and some GPUs lack low-latency rate control
74
+ }
75
+ }
76
+
77
+ ~VideoToolboxEncoder() override { Invalidate(); }
78
+ VideoToolboxEncoder(const VideoToolboxEncoder&) = delete;
79
+ VideoToolboxEncoder& operator=(const VideoToolboxEncoder&) = delete;
80
+
81
+ void Encode(const AVFrame& nv12, bool forceKeyFrame) override {
82
+ CVPixelBufferRef raw = nullptr;
83
+ if (const CVReturn created =
84
+ CVPixelBufferCreate(nullptr, params_.width, params_.height, kPixelFormat, nullptr, &raw);
85
+ created != kCVReturnSuccess) {
86
+ throw Failure("pixel buffer allocation", created);
87
+ }
88
+ const Cf<CVPixelBufferRef> buffer(raw);
89
+ CopyPlanes(nv12, raw);
90
+
91
+ // The session needs increasing timestamps even if the source timeline jumped back.
92
+ lastPts_ = std::max(nv12.pts, lastPts_ + 1);
93
+ const CMTime pts = CMTimeMake(lastPts_, kMicrosTimescale);
94
+ Cf<CFDictionaryRef> properties;
95
+ if (forceKeyFrame) {
96
+ properties = Dictionary(kVTEncodeFrameOptionKey_ForceKeyFrame, kCFBooleanTrue);
97
+ }
98
+ if (const OSStatus status = VTCompressionSessionEncodeFrame(session_.get(), raw, pts, kCMTimeInvalid,
99
+ properties.get(), nullptr, nullptr);
100
+ status != noErr) {
101
+ throw Failure("encode", status);
102
+ }
103
+ // Waits for this frame (no reordering): output leaves in the same call instead of one frame later.
104
+ if (const OSStatus status = VTCompressionSessionCompleteFrames(session_.get(), pts); status != noErr) {
105
+ throw Failure("complete", status);
106
+ }
107
+ Deliver();
108
+ }
109
+
110
+ void Flush() override {
111
+ if (const OSStatus status = VTCompressionSessionCompleteFrames(session_.get(), kCMTimeInvalid); status != noErr) {
112
+ throw Failure("flush", status);
113
+ }
114
+ Deliver();
115
+ }
116
+
117
+ private:
118
+ static constexpr OSType kPixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
119
+
120
+ // Low-latency rate control takes the 99th-percentile encode time from ~50 ms to ~15 ms at 1170x2532 / 60 fps.
121
+ void Open(bool lowLatency) {
122
+ std::array<const void*, 2> keys = {kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder,
123
+ kVTVideoEncoderSpecification_EnableLowLatencyRateControl};
124
+ std::array<const void*, 2> values = {kCFBooleanTrue, kCFBooleanTrue};
125
+ const Cf<CFDictionaryRef> spec(CFDictionaryCreate(nullptr, keys.data(), values.data(), lowLatency ? 2 : 1,
126
+ &kCFTypeDictionaryKeyCallBacks,
127
+ &kCFTypeDictionaryValueCallBacks));
128
+ const Cf<CFDictionaryRef> pixelAttributes =
129
+ Dictionary(kCVPixelBufferPixelFormatTypeKey, Number(static_cast<int32_t>(kPixelFormat)).get());
130
+ VTCompressionSessionRef session = nullptr;
131
+ const OSStatus status =
132
+ VTCompressionSessionCreate(nullptr, params_.width, params_.height, kCMVideoCodecType_H264, spec.get(),
133
+ pixelAttributes.get(), nullptr, &OnOutput, this, &session);
134
+ if (status != noErr || session == nullptr) {
135
+ throw PipelineError{kEncoderUnavailableError, Failure("session creation", status).message};
136
+ }
137
+ session_.reset(session);
138
+ Configure();
139
+ if (const OSStatus prepared = VTCompressionSessionPrepareToEncodeFrames(session_.get()); prepared != noErr) {
140
+ Invalidate();
141
+ throw PipelineError{kEncoderUnavailableError, Failure("prepare", prepared).message};
142
+ }
143
+ }
144
+
145
+ struct SessionDeleter {
146
+ void operator()(VTCompressionSessionRef session) const { CFRelease(session); }
147
+ };
148
+
149
+ // Properties that shape the promised output must be accepted; the rest are tuning (rejections are logged).
150
+ void Configure() {
151
+ const auto set = [&](CFStringRef key, CFTypeRef value, bool required) {
152
+ const OSStatus status = VTSessionSetProperty(session_.get(), key, value);
153
+ if (status == noErr) {
154
+ return;
155
+ }
156
+ std::array<char, kKeyNameSize> name{};
157
+ CFStringGetCString(key, name.data(), name.size(), kCFStringEncodingUTF8);
158
+ if (required) {
159
+ Invalidate();
160
+ throw PipelineError{kEncoderUnavailableError,
161
+ Failure((std::string("setting ") + name.data()).c_str(), status).message};
162
+ }
163
+ av_log(nullptr, AV_LOG_WARNING, "VideoToolbox rejected %s (OSStatus %d)\n", name.data(),
164
+ static_cast<int>(status));
165
+ };
166
+ set(kVTCompressionPropertyKey_RealTime, kCFBooleanTrue, false);
167
+ set(kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_High_AutoLevel, false);
168
+ set(kVTCompressionPropertyKey_ExpectedFrameRate, Number(params_.fps).get(), false);
169
+ set(kVTCompressionPropertyKey_AllowFrameReordering, kCFBooleanFalse, true);
170
+ set(kVTCompressionPropertyKey_AverageBitRate,
171
+ Number(static_cast<int32_t>(std::min<int64_t>(params_.bitrate, INT32_MAX))).get(), true);
172
+ set(kVTCompressionPropertyKey_MaxKeyFrameInterval, Number(static_cast<int32_t>(params_.gopFrames)).get(), true);
173
+ set(kVTCompressionPropertyKey_ColorPrimaries, kCMFormatDescriptionColorPrimaries_ITU_R_709_2, true);
174
+ set(kVTCompressionPropertyKey_TransferFunction, kCMFormatDescriptionTransferFunction_ITU_R_709_2, true);
175
+ set(kVTCompressionPropertyKey_YCbCrMatrix, kCMFormatDescriptionYCbCrMatrix_ITU_R_709_2, true);
176
+ }
177
+
178
+ void Invalidate() {
179
+ if (session_) {
180
+ VTCompressionSessionInvalidate(session_.get());
181
+ session_.reset();
182
+ }
183
+ }
184
+
185
+ static void CopyPlanes(const AVFrame& nv12, CVPixelBufferRef buffer) {
186
+ if (const CVReturn locked = CVPixelBufferLockBaseAddress(buffer, 0); locked != kCVReturnSuccess) {
187
+ throw Failure("pixel buffer lock", locked);
188
+ }
189
+ for (size_t plane = 0; plane < 2; ++plane) {
190
+ auto* dst = static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(buffer, plane));
191
+ if (dst == nullptr) {
192
+ CVPixelBufferUnlockBaseAddress(buffer, 0);
193
+ throw Failure("pixel buffer mapping", kCVReturnError);
194
+ }
195
+ const size_t dstStride = CVPixelBufferGetBytesPerRowOfPlane(buffer, plane);
196
+ const size_t rows = CVPixelBufferGetHeightOfPlane(buffer, plane);
197
+ const auto srcStride = static_cast<size_t>(nv12.linesize[plane]);
198
+ const size_t rowBytes = std::min(srcStride, dstStride);
199
+ for (size_t row = 0; row < rows; ++row) {
200
+ std::memcpy(dst + (row * dstStride), nv12.data[plane] + (row * srcStride), rowBytes);
201
+ }
202
+ }
203
+ CVPixelBufferUnlockBaseAddress(buffer, 0);
204
+ }
205
+
206
+ // Runs on a VideoToolbox thread; the encoding thread is blocked in CompleteFrames until it returns.
207
+ static void OnOutput(void* self, void* /*frameRefCon*/, OSStatus status, VTEncodeInfoFlags flags,
208
+ CMSampleBufferRef sample) {
209
+ static_cast<VideoToolboxEncoder*>(self)->Collect(status, flags, sample);
210
+ }
211
+
212
+ void Collect(OSStatus status, VTEncodeInfoFlags flags, CMSampleBufferRef sample) {
213
+ const std::scoped_lock<std::mutex> lock(mutex_);
214
+ if (status != noErr) {
215
+ failure_ = status;
216
+ return;
217
+ }
218
+ if (sample == nullptr || (flags & kVTEncodeInfo_FrameDropped) != 0U || CMSampleBufferDataIsReady(sample) == 0U) {
219
+ return;
220
+ }
221
+ try {
222
+ pending_.push_back(ToAnnexB(sample));
223
+ } catch (...) { // this is a VideoToolbox thread: nothing may escape (bad_alloc included)
224
+ failure_ = kCMSampleBufferError_InvalidMediaFormat;
225
+ }
226
+ }
227
+
228
+ // AVCC (length-prefixed) to Annex-B with 4-byte start codes; SPS/PPS are prepended on key frames.
229
+ static EncodedUnit ToAnnexB(CMSampleBufferRef sample) {
230
+ EncodedUnit unit;
231
+ unit.isKeyFrame = IsKeyFrame(sample);
232
+ unit.timestampMicros = CMTimeConvertScale(CMSampleBufferGetPresentationTimeStamp(sample), kMicrosTimescale,
233
+ kCMTimeRoundingMethod_Default)
234
+ .value;
235
+ CMFormatDescriptionRef format = CMSampleBufferGetFormatDescription(sample);
236
+ size_t parameterSets = 0;
237
+ int lengthSize = 0;
238
+ const uint8_t* first = nullptr;
239
+ size_t firstSize = 0;
240
+ if (format == nullptr ||
241
+ CMVideoFormatDescriptionGetH264ParameterSetAtIndex(format, 0, &first, &firstSize, &parameterSets,
242
+ &lengthSize) != noErr ||
243
+ lengthSize < static_cast<int>(kMinNalLengthSize) || lengthSize > static_cast<int>(kMaxNalLengthSize)) {
244
+ throw PipelineError{"MediaProxyError", "VideoToolbox produced a sample without H.264 parameter sets"};
245
+ }
246
+ CMBlockBufferRef block = CMSampleBufferGetDataBuffer(sample);
247
+ size_t total = 0;
248
+ size_t contiguous = 0;
249
+ char* data = nullptr;
250
+ if (block == nullptr || CMBlockBufferGetDataPointer(block, 0, &contiguous, &total, &data) != noErr ||
251
+ contiguous != total) {
252
+ throw PipelineError{"MediaProxyError", "VideoToolbox produced a fragmented sample"};
253
+ }
254
+ unit.data.reserve(total + 128);
255
+ if (unit.isKeyFrame) {
256
+ for (size_t i = 0; i < parameterSets; ++i) {
257
+ const uint8_t* set = nullptr;
258
+ size_t size = 0;
259
+ if (CMVideoFormatDescriptionGetH264ParameterSetAtIndex(format, i, &set, &size, nullptr, nullptr) == noErr) {
260
+ AppendNal(unit.data, set, size);
261
+ }
262
+ }
263
+ }
264
+ const auto* bytes = reinterpret_cast<const uint8_t*>(data);
265
+ const auto prefix = static_cast<size_t>(lengthSize);
266
+ for (size_t offset = 0; offset + prefix <= total;) {
267
+ size_t length = 0;
268
+ for (size_t i = 0; i < prefix; ++i) {
269
+ length = (length << kByteBits) | bytes[offset + i];
270
+ }
271
+ offset += prefix;
272
+ if (length > total - offset) {
273
+ throw PipelineError{"MediaProxyError", "VideoToolbox produced a malformed sample"};
274
+ }
275
+ if (length > 0) {
276
+ AppendNal(unit.data, bytes + offset, length);
277
+ }
278
+ offset += length;
279
+ }
280
+ return unit;
281
+ }
282
+
283
+ void Deliver() {
284
+ std::vector<EncodedUnit> units;
285
+ OSStatus failure = noErr;
286
+ {
287
+ const std::scoped_lock<std::mutex> lock(mutex_);
288
+ units.swap(pending_);
289
+ failure = std::exchange(failure_, noErr);
290
+ }
291
+ for (EncodedUnit& unit : units) {
292
+ sink_(std::move(unit));
293
+ }
294
+ if (failure != noErr) {
295
+ throw Failure("output", failure);
296
+ }
297
+ }
298
+
299
+ H264Params params_;
300
+ H264Encoder::Sink sink_;
301
+ std::mutex mutex_;
302
+ std::vector<EncodedUnit> pending_;
303
+ OSStatus failure_ = noErr;
304
+ int64_t lastPts_ = -1;
305
+ std::unique_ptr<std::remove_pointer_t<VTCompressionSessionRef>, SessionDeleter> session_;
306
+ };
307
+
308
+ } // namespace
309
+
310
+ std::unique_ptr<H264Encoder> CreateVideoToolboxEncoder(const H264Params& params, const H264Encoder::Sink& sink) {
311
+ return std::make_unique<VideoToolboxEncoder>(params, sink);
312
+ }
313
+
314
+ } // namespace media_proxy
@@ -0,0 +1,13 @@
1
+ #pragma once
2
+
3
+ #include <memory>
4
+
5
+ #include "h264_encoder.h"
6
+
7
+ namespace media_proxy {
8
+
9
+ // macOS VideoToolbox, called directly: FFmpeg's wrapper only returns a packet on the next send, which costs
10
+ // several frames of latency. Throws PipelineError (MediaProxyEncoderUnavailableError) if the session can't open.
11
+ std::unique_ptr<H264Encoder> CreateVideoToolboxEncoder(const H264Params& params, const H264Encoder::Sink& sink);
12
+
13
+ } // namespace media_proxy
package/src/native.ts ADDED
@@ -0,0 +1,51 @@
1
+ import {createRequire} from 'node:module';
2
+
3
+ import {MediaProxyUnavailableError} from './errors.js';
4
+ import {PACKAGE_NAME, getPkgRoot} from './pkg-root.js';
5
+ import type {NativeModule} from './types.js';
6
+
7
+ const require = createRequire(import.meta.url);
8
+
9
+ export const SUPPORTED_PLATFORMS = ['darwin', 'linux', 'win32'] as const;
10
+ export const SUPPORTED_ARCHS = ['x64', 'arm64'] as const;
11
+
12
+ /** Throws {@link MediaProxyUnavailableError} unless this platform/arch has a build. */
13
+ export function assertSupportedPlatform(platform: string = process.platform, arch: string = process.arch): void {
14
+ if (
15
+ !(SUPPORTED_PLATFORMS as readonly string[]).includes(platform) ||
16
+ !(SUPPORTED_ARCHS as readonly string[]).includes(arch)
17
+ ) {
18
+ throw new MediaProxyUnavailableError(
19
+ `${PACKAGE_NAME} supports darwin/linux/win32 on x64/arm64, not ${platform}-${arch}`,
20
+ platform,
21
+ arch,
22
+ );
23
+ }
24
+ }
25
+
26
+ /**
27
+ * Wraps `load` so nothing runs until the first call. A failure is re-typed as
28
+ * {@link MediaProxyUnavailableError} and not cached (retried on the next call); success is cached.
29
+ */
30
+ export function createNativeLoader(load: () => unknown): () => NativeModule {
31
+ let native: NativeModule | undefined;
32
+ return () => {
33
+ if (native) {
34
+ return native;
35
+ }
36
+ assertSupportedPlatform();
37
+ try {
38
+ native = load() as NativeModule;
39
+ } catch (err) {
40
+ throw new MediaProxyUnavailableError(
41
+ `No usable ${PACKAGE_NAME} addon for ${process.platform}-${process.arch}: ${(err as Error).message}`,
42
+ process.platform,
43
+ process.arch,
44
+ );
45
+ }
46
+ return native;
47
+ };
48
+ }
49
+
50
+ // Never invoked at import or install time, only by the first API call that needs the addon.
51
+ export const loadNative = createNativeLoader(() => require('node-gyp-build')(getPkgRoot()));
@@ -0,0 +1,14 @@
1
+ import {fileURLToPath} from 'node:url';
2
+
3
+ import {node, util} from '@appium/support';
4
+
5
+ export const PACKAGE_NAME = '@appium/media-proxy';
6
+
7
+ /** Package root (contains binding.gyp, prebuilds/, or build/ after compile). */
8
+ export const getPkgRoot = util.memoize(function resolvePkgRoot(): string {
9
+ const root = node.getModuleRootSync(PACKAGE_NAME, fileURLToPath(import.meta.url));
10
+ if (!root) {
11
+ throw new Error(`Could not locate the root of the '${PACKAGE_NAME}' package`);
12
+ }
13
+ return root;
14
+ });