@appium/media-proxy 0.0.0-stage → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/LICENSE +201 -0
- package/README.md +184 -2
- package/THIRD_PARTY_NOTICES.md +29 -0
- package/binding.gyp +90 -0
- package/lib/src/background.d.ts +5 -0
- package/lib/src/background.d.ts.map +1 -0
- package/lib/src/background.js +16 -0
- package/lib/src/background.js.map +1 -0
- package/lib/src/capabilities.d.ts +9 -0
- package/lib/src/capabilities.d.ts.map +1 -0
- package/lib/src/capabilities.js +11 -0
- package/lib/src/capabilities.js.map +1 -0
- package/lib/src/errors.d.ts +26 -0
- package/lib/src/errors.d.ts.map +1 -0
- package/lib/src/errors.js +40 -0
- package/lib/src/errors.js.map +1 -0
- package/lib/src/framing/apst.d.ts +43 -0
- package/lib/src/framing/apst.d.ts.map +1 -0
- package/lib/src/framing/apst.js +126 -0
- package/lib/src/framing/apst.js.map +1 -0
- package/lib/src/framing/mjpeg-http.d.ts +30 -0
- package/lib/src/framing/mjpeg-http.d.ts.map +1 -0
- package/lib/src/framing/mjpeg-http.js +148 -0
- package/lib/src/framing/mjpeg-http.js.map +1 -0
- package/lib/src/framing/mjpeg-multipart.d.ts +11 -0
- package/lib/src/framing/mjpeg-multipart.d.ts.map +1 -0
- package/lib/src/framing/mjpeg-multipart.js +26 -0
- package/lib/src/framing/mjpeg-multipart.js.map +1 -0
- package/lib/src/index.d.ts +10 -0
- package/lib/src/index.d.ts.map +1 -0
- package/lib/src/index.js +7 -0
- package/lib/src/index.js.map +1 -0
- package/lib/src/native.d.ts +12 -0
- package/lib/src/native.d.ts.map +1 -0
- package/lib/src/native.js +36 -0
- package/lib/src/native.js.map +1 -0
- package/lib/src/pkg-root.d.ts +6 -0
- package/lib/src/pkg-root.d.ts.map +1 -0
- package/lib/src/pkg-root.js +12 -0
- package/lib/src/pkg-root.js.map +1 -0
- package/lib/src/queue.d.ts +34 -0
- package/lib/src/queue.d.ts.map +1 -0
- package/lib/src/queue.js +123 -0
- package/lib/src/queue.js.map +1 -0
- package/lib/src/transcoder.d.ts +81 -0
- package/lib/src/transcoder.d.ts.map +1 -0
- package/lib/src/transcoder.js +422 -0
- package/lib/src/transcoder.js.map +1 -0
- package/lib/src/types.d.ts +147 -0
- package/lib/src/types.d.ts.map +1 -0
- package/lib/src/types.js +2 -0
- package/lib/src/types.js.map +1 -0
- package/package.json +65 -4
- package/prebuilds/darwin-arm64/@appium+media-proxy.node +0 -0
- package/prebuilds/darwin-arm64/ffmpeg-LICENSE-LGPL-2.1.txt +502 -0
- package/prebuilds/darwin-arm64/ffmpeg-build-info.txt +3 -0
- package/prebuilds/darwin-arm64/libavcodec-appium.62.dylib +0 -0
- package/prebuilds/darwin-arm64/libavutil-appium.60.dylib +0 -0
- package/prebuilds/darwin-arm64/libswscale-appium.9.dylib +0 -0
- package/prebuilds/darwin-x64/@appium+media-proxy.node +0 -0
- package/prebuilds/darwin-x64/ffmpeg-LICENSE-LGPL-2.1.txt +502 -0
- package/prebuilds/darwin-x64/ffmpeg-build-info.txt +3 -0
- package/prebuilds/darwin-x64/libavcodec-appium.62.dylib +0 -0
- package/prebuilds/darwin-x64/libavutil-appium.60.dylib +0 -0
- package/prebuilds/darwin-x64/libswscale-appium.9.dylib +0 -0
- package/prebuilds/linux-arm64/@appium+media-proxy.node +0 -0
- package/prebuilds/linux-arm64/ffmpeg-LICENSE-LGPL-2.1.txt +502 -0
- package/prebuilds/linux-arm64/ffmpeg-build-info.txt +3 -0
- package/prebuilds/linux-arm64/libavcodec-appium.so.62 +0 -0
- package/prebuilds/linux-arm64/libavutil-appium.so.60 +0 -0
- package/prebuilds/linux-arm64/libswscale-appium.so.9 +0 -0
- package/prebuilds/linux-x64/@appium+media-proxy.node +0 -0
- package/prebuilds/linux-x64/ffmpeg-LICENSE-LGPL-2.1.txt +502 -0
- package/prebuilds/linux-x64/ffmpeg-build-info.txt +3 -0
- package/prebuilds/linux-x64/libavcodec-appium.so.62 +0 -0
- package/prebuilds/linux-x64/libavutil-appium.so.60 +0 -0
- package/prebuilds/linux-x64/libswscale-appium.so.9 +0 -0
- package/prebuilds/win32-arm64/@appium+media-proxy.node +0 -0
- package/prebuilds/win32-arm64/avcodec-appium-62.dll +0 -0
- package/prebuilds/win32-arm64/avutil-appium-60.dll +0 -0
- package/prebuilds/win32-arm64/ffmpeg-LICENSE-LGPL-2.1.txt +502 -0
- package/prebuilds/win32-arm64/ffmpeg-build-info.txt +3 -0
- package/prebuilds/win32-arm64/swscale-appium-9.dll +0 -0
- package/prebuilds/win32-x64/@appium+media-proxy.node +0 -0
- package/prebuilds/win32-x64/avcodec-appium-62.dll +0 -0
- package/prebuilds/win32-x64/avutil-appium-60.dll +0 -0
- package/prebuilds/win32-x64/ffmpeg-LICENSE-LGPL-2.1.txt +502 -0
- package/prebuilds/win32-x64/ffmpeg-build-info.txt +3 -0
- package/prebuilds/win32-x64/swscale-appium-9.dll +0 -0
- package/scripts/install.mjs +29 -0
- package/src/background.ts +15 -0
- package/src/capabilities.ts +12 -0
- package/src/errors.ts +45 -0
- package/src/framing/apst.ts +158 -0
- package/src/framing/mjpeg-http.ts +186 -0
- package/src/framing/mjpeg-multipart.ts +38 -0
- package/src/index.ts +43 -0
- package/src/native/addon.cc +402 -0
- package/src/native/audio_pipeline.cc +426 -0
- package/src/native/audio_pipeline.h +63 -0
- package/src/native/ffmpeg_util.h +37 -0
- package/src/native/h264_encoder.cc +133 -0
- package/src/native/h264_encoder.h +48 -0
- package/src/native/mediafoundation_encoder.cc +442 -0
- package/src/native/mediafoundation_encoder.h +14 -0
- package/src/native/napi_util.h +20 -0
- package/src/native/openh264_encoder.cc +285 -0
- package/src/native/openh264_encoder.h +14 -0
- package/src/native/pipeline.cc +555 -0
- package/src/native/pipeline.h +108 -0
- package/src/native/vendor/openh264/LICENSE +23 -0
- package/src/native/vendor/openh264/codec_api.h +592 -0
- package/src/native/vendor/openh264/codec_app_def.h +820 -0
- package/src/native/vendor/openh264/codec_def.h +216 -0
- package/src/native/vendor/openh264/codec_ver.h +15 -0
- package/src/native/videotoolbox_encoder.cc +314 -0
- package/src/native/videotoolbox_encoder.h +13 -0
- package/src/native.ts +51 -0
- package/src/pkg-root.ts +14 -0
- package/src/queue.ts +133 -0
- package/src/transcoder.ts +480 -0
- package/src/types.ts +158 -0
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* \copy
|
|
3
|
+
* Copyright (c) 2013, Cisco Systems
|
|
4
|
+
* All rights reserved.
|
|
5
|
+
*
|
|
6
|
+
* Redistribution and use in source and binary forms, with or without
|
|
7
|
+
* modification, are permitted provided that the following conditions
|
|
8
|
+
* are met:
|
|
9
|
+
*
|
|
10
|
+
* * Redistributions of source code must retain the above copyright
|
|
11
|
+
* notice, this list of conditions and the following disclaimer.
|
|
12
|
+
*
|
|
13
|
+
* * Redistributions in binary form must reproduce the above copyright
|
|
14
|
+
* notice, this list of conditions and the following disclaimer in
|
|
15
|
+
* the documentation and/or other materials provided with the
|
|
16
|
+
* distribution.
|
|
17
|
+
*
|
|
18
|
+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
19
|
+
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
20
|
+
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
|
|
21
|
+
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
|
|
22
|
+
* COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|
23
|
+
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
|
24
|
+
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
|
25
|
+
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
26
|
+
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
27
|
+
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
|
28
|
+
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
29
|
+
* POSSIBILITY OF SUCH DAMAGE.
|
|
30
|
+
*
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
#ifndef WELS_VIDEO_CODEC_DEFINITION_H__
|
|
34
|
+
#define WELS_VIDEO_CODEC_DEFINITION_H__
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* @file codec_def.h
|
|
38
|
+
*/
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* @brief Enumerate the type of video format
|
|
42
|
+
*/
|
|
43
|
+
typedef enum {
|
|
44
|
+
videoFormatRGB = 1, ///< rgb color formats
|
|
45
|
+
videoFormatRGBA = 2,
|
|
46
|
+
videoFormatRGB555 = 3,
|
|
47
|
+
videoFormatRGB565 = 4,
|
|
48
|
+
videoFormatBGR = 5,
|
|
49
|
+
videoFormatBGRA = 6,
|
|
50
|
+
videoFormatABGR = 7,
|
|
51
|
+
videoFormatARGB = 8,
|
|
52
|
+
|
|
53
|
+
videoFormatYUY2 = 20, ///< yuv color formats
|
|
54
|
+
videoFormatYVYU = 21,
|
|
55
|
+
videoFormatUYVY = 22,
|
|
56
|
+
videoFormatI420 = 23, ///< the same as IYUV
|
|
57
|
+
videoFormatYV12 = 24,
|
|
58
|
+
videoFormatInternal = 25, ///< only used in SVC decoder testbed
|
|
59
|
+
|
|
60
|
+
videoFormatNV12 = 26, ///< new format for output by DXVA decoding
|
|
61
|
+
|
|
62
|
+
videoFormatVFlip = 0x80000000
|
|
63
|
+
} EVideoFormatType;
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* @brief Enumerate video frame type
|
|
67
|
+
*/
|
|
68
|
+
typedef enum {
|
|
69
|
+
videoFrameTypeInvalid, ///< encoder not ready or parameters are invalidate
|
|
70
|
+
videoFrameTypeIDR, ///< IDR frame in H.264
|
|
71
|
+
videoFrameTypeI, ///< I frame type
|
|
72
|
+
videoFrameTypeP, ///< P frame type
|
|
73
|
+
videoFrameTypeSkip, ///< skip the frame based encoder kernel
|
|
74
|
+
videoFrameTypeIPMixed ///< a frame where I and P slices are mixing, not supported yet
|
|
75
|
+
} EVideoFrameType;
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* @brief Enumerate return type
|
|
79
|
+
*/
|
|
80
|
+
typedef enum {
|
|
81
|
+
cmResultSuccess, ///< successful
|
|
82
|
+
cmInitParaError, ///< parameters are invalid
|
|
83
|
+
cmUnknownReason,
|
|
84
|
+
cmMallocMemeError, ///< malloc a memory error
|
|
85
|
+
cmInitExpected, ///< initial action is expected
|
|
86
|
+
cmUnsupportedData
|
|
87
|
+
} CM_RETURN;
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* @brief Enumulate the nal unit type
|
|
91
|
+
*/
|
|
92
|
+
enum ENalUnitType {
|
|
93
|
+
NAL_UNKNOWN = 0,
|
|
94
|
+
NAL_SLICE = 1,
|
|
95
|
+
NAL_SLICE_DPA = 2,
|
|
96
|
+
NAL_SLICE_DPB = 3,
|
|
97
|
+
NAL_SLICE_DPC = 4,
|
|
98
|
+
NAL_SLICE_IDR = 5, ///< ref_idc != 0
|
|
99
|
+
NAL_SEI = 6, ///< ref_idc == 0
|
|
100
|
+
NAL_SPS = 7,
|
|
101
|
+
NAL_PPS = 8
|
|
102
|
+
///< ref_idc == 0 for 6,9,10,11,12
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* @brief NRI: eNalRefIdc
|
|
107
|
+
*/
|
|
108
|
+
enum ENalPriority {
|
|
109
|
+
NAL_PRIORITY_DISPOSABLE = 0,
|
|
110
|
+
NAL_PRIORITY_LOW = 1,
|
|
111
|
+
NAL_PRIORITY_HIGH = 2,
|
|
112
|
+
NAL_PRIORITY_HIGHEST = 3
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
#define IS_PARAMETER_SET_NAL(eNalRefIdc, eNalType) \
|
|
116
|
+
( (eNalRefIdc == NAL_PRIORITY_HIGHEST) && (eNalType == (NAL_SPS|NAL_PPS) || eNalType == NAL_SPS) )
|
|
117
|
+
|
|
118
|
+
#define IS_IDR_NAL(eNalRefIdc, eNalType) \
|
|
119
|
+
( (eNalRefIdc == NAL_PRIORITY_HIGHEST) && (eNalType == NAL_SLICE_IDR) )
|
|
120
|
+
|
|
121
|
+
#define FRAME_NUM_PARAM_SET (-1)
|
|
122
|
+
#define FRAME_NUM_IDR 0
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* @brief eDeblockingIdc
|
|
126
|
+
*/
|
|
127
|
+
enum {
|
|
128
|
+
DEBLOCKING_IDC_0 = 0,
|
|
129
|
+
DEBLOCKING_IDC_1 = 1,
|
|
130
|
+
DEBLOCKING_IDC_2 = 2
|
|
131
|
+
};
|
|
132
|
+
#define DEBLOCKING_OFFSET (6)
|
|
133
|
+
#define DEBLOCKING_OFFSET_MINUS (-6)
|
|
134
|
+
|
|
135
|
+
/* Error Tools definition */
|
|
136
|
+
typedef unsigned short ERR_TOOL;
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
@brief to do
|
|
140
|
+
*/
|
|
141
|
+
enum {
|
|
142
|
+
ET_NONE = 0x00, ///< NONE Error Tools
|
|
143
|
+
ET_IP_SCALE = 0x01, ///< IP Scalable
|
|
144
|
+
ET_FMO = 0x02, ///< Flexible Macroblock Ordering
|
|
145
|
+
ET_IR_R1 = 0x04, ///< Intra Refresh in predifined 2% MB
|
|
146
|
+
ET_IR_R2 = 0x08, ///< Intra Refresh in predifined 5% MB
|
|
147
|
+
ET_IR_R3 = 0x10, ///< Intra Refresh in predifined 10% MB
|
|
148
|
+
ET_FEC_HALF = 0x20, ///< Forward Error Correction in 50% redundency mode
|
|
149
|
+
ET_FEC_FULL = 0x40, ///< Forward Error Correction in 100% redundency mode
|
|
150
|
+
ET_RFS = 0x80 ///< Reference Frame Selection
|
|
151
|
+
};
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* @brief Information of coded Slice(=NAL)(s)
|
|
155
|
+
*/
|
|
156
|
+
typedef struct SliceInformation {
|
|
157
|
+
unsigned char* pBufferOfSlices; ///< base buffer of coded slice(s)
|
|
158
|
+
int iCodedSliceCount; ///< number of coded slices
|
|
159
|
+
unsigned int* pLengthOfSlices; ///< array of slices length accordingly by number of slice
|
|
160
|
+
int iFecType; ///< FEC type[0, 50%FEC, 100%FEC]
|
|
161
|
+
unsigned char uiSliceIdx; ///< index of slice in frame [FMO: 0,..,uiSliceCount-1; No FMO: 0]
|
|
162
|
+
unsigned char uiSliceCount; ///< count number of slice in frame [FMO: 2-8; No FMO: 1]
|
|
163
|
+
char iFrameIndex; ///< index of frame[-1, .., idr_interval-1]
|
|
164
|
+
unsigned char uiNalRefIdc; ///< NRI, priority level of slice(NAL)
|
|
165
|
+
unsigned char uiNalType; ///< NAL type
|
|
166
|
+
unsigned char
|
|
167
|
+
uiContainingFinalNal; ///< whether final NAL is involved in buffer of coded slices, flag used in Pause feature in T27
|
|
168
|
+
} SliceInfo, *PSliceInfo;
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* @brief thresholds of the initial, maximal and minimal rate
|
|
172
|
+
*/
|
|
173
|
+
typedef struct {
|
|
174
|
+
int iWidth; ///< frame width
|
|
175
|
+
int iHeight; ///< frame height
|
|
176
|
+
int iThresholdOfInitRate; ///< threshold of initial rate
|
|
177
|
+
int iThresholdOfMaxRate; ///< threshold of maximal rate
|
|
178
|
+
int iThresholdOfMinRate; ///< threshold of minimal rate
|
|
179
|
+
int iMinThresholdFrameRate; ///< min frame rate min
|
|
180
|
+
int iSkipFrameRate; ///< skip to frame rate min
|
|
181
|
+
int iSkipFrameStep; ///< how many frames to skip
|
|
182
|
+
} SRateThresholds, *PRateThresholds;
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* @brief Structure for decoder memery
|
|
186
|
+
*/
|
|
187
|
+
typedef struct TagSysMemBuffer {
|
|
188
|
+
int iWidth; ///< width of decoded pic for display
|
|
189
|
+
int iHeight; ///< height of decoded pic for display
|
|
190
|
+
int iFormat; ///< type is "EVideoFormatType"
|
|
191
|
+
int iStride[2]; ///< stride of 2 component
|
|
192
|
+
} SSysMEMBuffer;
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* @brief Buffer info
|
|
196
|
+
*/
|
|
197
|
+
typedef struct TagBufferInfo {
|
|
198
|
+
int iBufferStatus; ///< 0: one frame data is not ready; 1: one frame data is ready
|
|
199
|
+
unsigned long long uiInBsTimeStamp; ///< input BS timestamp
|
|
200
|
+
unsigned long long uiOutYuvTimeStamp; ///< output YUV timestamp, when bufferstatus is 1
|
|
201
|
+
union {
|
|
202
|
+
SSysMEMBuffer sSystemBuffer; ///< memory info for one picture
|
|
203
|
+
} UsrData; ///< output buffer info
|
|
204
|
+
unsigned char* pDst[3]; //point to picture YUV data
|
|
205
|
+
} SBufferInfo;
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* @brief In a GOP, multiple of the key frame number, derived from
|
|
210
|
+
* the number of layers(index or array below)
|
|
211
|
+
*/
|
|
212
|
+
static const char kiKeyNumMultiple[] = {
|
|
213
|
+
1, 1, 2, 4, 8, 16,
|
|
214
|
+
};
|
|
215
|
+
|
|
216
|
+
#endif//WELS_VIDEO_CODEC_DEFINITION_H__
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
//The current file is auto-generated by script: generate_codec_ver.sh
|
|
2
|
+
#ifndef CODEC_VER_H
|
|
3
|
+
#define CODEC_VER_H
|
|
4
|
+
|
|
5
|
+
#include "codec_app_def.h"
|
|
6
|
+
|
|
7
|
+
static const OpenH264Version g_stCodecVersion = {2, 6, 0, 2502};
|
|
8
|
+
static const char* const g_strCodecVer = "OpenH264 version:2.6.0.2502";
|
|
9
|
+
|
|
10
|
+
#define OPENH264_MAJOR (2)
|
|
11
|
+
#define OPENH264_MINOR (6)
|
|
12
|
+
#define OPENH264_REVISION (0)
|
|
13
|
+
#define OPENH264_RESERVED (2502)
|
|
14
|
+
|
|
15
|
+
#endif // CODEC_VER_H
|
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
#include "videotoolbox_encoder.h"
|
|
2
|
+
|
|
3
|
+
#include <CoreFoundation/CoreFoundation.h>
|
|
4
|
+
#include <CoreMedia/CoreMedia.h>
|
|
5
|
+
#include <CoreVideo/CoreVideo.h>
|
|
6
|
+
#include <VideoToolbox/VideoToolbox.h>
|
|
7
|
+
|
|
8
|
+
#include <algorithm>
|
|
9
|
+
#include <array>
|
|
10
|
+
#include <cstdint>
|
|
11
|
+
#include <cstring>
|
|
12
|
+
#include <mutex>
|
|
13
|
+
#include <string>
|
|
14
|
+
#include <type_traits>
|
|
15
|
+
#include <utility>
|
|
16
|
+
|
|
17
|
+
extern "C" {
|
|
18
|
+
#include <libavutil/frame.h>
|
|
19
|
+
#include <libavutil/log.h>
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
namespace media_proxy {
|
|
23
|
+
|
|
24
|
+
namespace {
|
|
25
|
+
|
|
26
|
+
constexpr int32_t kMicrosTimescale = 1000000;
|
|
27
|
+
constexpr size_t kMinNalLengthSize = 1;
|
|
28
|
+
constexpr size_t kMaxNalLengthSize = 4;
|
|
29
|
+
constexpr size_t kByteBits = 8;
|
|
30
|
+
constexpr Boolean kNo = 0;
|
|
31
|
+
constexpr size_t kKeyNameSize = 64;
|
|
32
|
+
|
|
33
|
+
struct CfDeleter {
|
|
34
|
+
void operator()(CFTypeRef ref) const { CFRelease(ref); }
|
|
35
|
+
};
|
|
36
|
+
template <typename T>
|
|
37
|
+
using Cf = std::unique_ptr<std::remove_pointer_t<T>, CfDeleter>;
|
|
38
|
+
|
|
39
|
+
PipelineError Failure(const char* what, OSStatus status) {
|
|
40
|
+
return {"MediaProxyError", std::string("VideoToolbox ") + what + " failed (OSStatus " + std::to_string(status) + ")"};
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
Cf<CFNumberRef> Number(int32_t value) { return Cf<CFNumberRef>(CFNumberCreate(nullptr, kCFNumberSInt32Type, &value)); }
|
|
44
|
+
|
|
45
|
+
Cf<CFNumberRef> Number(double value) { return Cf<CFNumberRef>(CFNumberCreate(nullptr, kCFNumberDoubleType, &value)); }
|
|
46
|
+
|
|
47
|
+
Cf<CFDictionaryRef> Dictionary(const void* key, const void* value) {
|
|
48
|
+
return Cf<CFDictionaryRef>(
|
|
49
|
+
CFDictionaryCreate(nullptr, &key, &value, 1, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks));
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
bool IsKeyFrame(CMSampleBufferRef sample) {
|
|
53
|
+
CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sample, kNo);
|
|
54
|
+
if (attachments == nullptr || CFArrayGetCount(attachments) == 0) {
|
|
55
|
+
return true;
|
|
56
|
+
}
|
|
57
|
+
const auto* first = static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0));
|
|
58
|
+
const void* notSync = CFDictionaryGetValue(first, kCMSampleAttachmentKey_NotSync);
|
|
59
|
+
return notSync == nullptr || CFBooleanGetValue(static_cast<CFBooleanRef>(notSync)) == 0U;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
void AppendNal(std::vector<uint8_t>& out, const uint8_t* data, size_t size) {
|
|
63
|
+
out.insert(out.end(), kAnnexBStartCode.begin(), kAnnexBStartCode.end());
|
|
64
|
+
out.insert(out.end(), data, data + size);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
class VideoToolboxEncoder final : public H264Encoder {
|
|
68
|
+
public:
|
|
69
|
+
VideoToolboxEncoder(const H264Params& params, H264Encoder::Sink sink) : params_(params), sink_(std::move(sink)) {
|
|
70
|
+
try {
|
|
71
|
+
Open(true);
|
|
72
|
+
} catch (const PipelineError&) {
|
|
73
|
+
Open(false); // older systems and some GPUs lack low-latency rate control
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
~VideoToolboxEncoder() override { Invalidate(); }
|
|
78
|
+
VideoToolboxEncoder(const VideoToolboxEncoder&) = delete;
|
|
79
|
+
VideoToolboxEncoder& operator=(const VideoToolboxEncoder&) = delete;
|
|
80
|
+
|
|
81
|
+
void Encode(const AVFrame& nv12, bool forceKeyFrame) override {
|
|
82
|
+
CVPixelBufferRef raw = nullptr;
|
|
83
|
+
if (const CVReturn created =
|
|
84
|
+
CVPixelBufferCreate(nullptr, params_.width, params_.height, kPixelFormat, nullptr, &raw);
|
|
85
|
+
created != kCVReturnSuccess) {
|
|
86
|
+
throw Failure("pixel buffer allocation", created);
|
|
87
|
+
}
|
|
88
|
+
const Cf<CVPixelBufferRef> buffer(raw);
|
|
89
|
+
CopyPlanes(nv12, raw);
|
|
90
|
+
|
|
91
|
+
// The session needs increasing timestamps even if the source timeline jumped back.
|
|
92
|
+
lastPts_ = std::max(nv12.pts, lastPts_ + 1);
|
|
93
|
+
const CMTime pts = CMTimeMake(lastPts_, kMicrosTimescale);
|
|
94
|
+
Cf<CFDictionaryRef> properties;
|
|
95
|
+
if (forceKeyFrame) {
|
|
96
|
+
properties = Dictionary(kVTEncodeFrameOptionKey_ForceKeyFrame, kCFBooleanTrue);
|
|
97
|
+
}
|
|
98
|
+
if (const OSStatus status = VTCompressionSessionEncodeFrame(session_.get(), raw, pts, kCMTimeInvalid,
|
|
99
|
+
properties.get(), nullptr, nullptr);
|
|
100
|
+
status != noErr) {
|
|
101
|
+
throw Failure("encode", status);
|
|
102
|
+
}
|
|
103
|
+
// Waits for this frame (no reordering): output leaves in the same call instead of one frame later.
|
|
104
|
+
if (const OSStatus status = VTCompressionSessionCompleteFrames(session_.get(), pts); status != noErr) {
|
|
105
|
+
throw Failure("complete", status);
|
|
106
|
+
}
|
|
107
|
+
Deliver();
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
void Flush() override {
|
|
111
|
+
if (const OSStatus status = VTCompressionSessionCompleteFrames(session_.get(), kCMTimeInvalid); status != noErr) {
|
|
112
|
+
throw Failure("flush", status);
|
|
113
|
+
}
|
|
114
|
+
Deliver();
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
private:
|
|
118
|
+
static constexpr OSType kPixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
|
|
119
|
+
|
|
120
|
+
// Low-latency rate control takes the 99th-percentile encode time from ~50 ms to ~15 ms at 1170x2532 / 60 fps.
|
|
121
|
+
void Open(bool lowLatency) {
|
|
122
|
+
std::array<const void*, 2> keys = {kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder,
|
|
123
|
+
kVTVideoEncoderSpecification_EnableLowLatencyRateControl};
|
|
124
|
+
std::array<const void*, 2> values = {kCFBooleanTrue, kCFBooleanTrue};
|
|
125
|
+
const Cf<CFDictionaryRef> spec(CFDictionaryCreate(nullptr, keys.data(), values.data(), lowLatency ? 2 : 1,
|
|
126
|
+
&kCFTypeDictionaryKeyCallBacks,
|
|
127
|
+
&kCFTypeDictionaryValueCallBacks));
|
|
128
|
+
const Cf<CFDictionaryRef> pixelAttributes =
|
|
129
|
+
Dictionary(kCVPixelBufferPixelFormatTypeKey, Number(static_cast<int32_t>(kPixelFormat)).get());
|
|
130
|
+
VTCompressionSessionRef session = nullptr;
|
|
131
|
+
const OSStatus status =
|
|
132
|
+
VTCompressionSessionCreate(nullptr, params_.width, params_.height, kCMVideoCodecType_H264, spec.get(),
|
|
133
|
+
pixelAttributes.get(), nullptr, &OnOutput, this, &session);
|
|
134
|
+
if (status != noErr || session == nullptr) {
|
|
135
|
+
throw PipelineError{kEncoderUnavailableError, Failure("session creation", status).message};
|
|
136
|
+
}
|
|
137
|
+
session_.reset(session);
|
|
138
|
+
Configure();
|
|
139
|
+
if (const OSStatus prepared = VTCompressionSessionPrepareToEncodeFrames(session_.get()); prepared != noErr) {
|
|
140
|
+
Invalidate();
|
|
141
|
+
throw PipelineError{kEncoderUnavailableError, Failure("prepare", prepared).message};
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
struct SessionDeleter {
|
|
146
|
+
void operator()(VTCompressionSessionRef session) const { CFRelease(session); }
|
|
147
|
+
};
|
|
148
|
+
|
|
149
|
+
// Properties that shape the promised output must be accepted; the rest are tuning (rejections are logged).
|
|
150
|
+
void Configure() {
|
|
151
|
+
const auto set = [&](CFStringRef key, CFTypeRef value, bool required) {
|
|
152
|
+
const OSStatus status = VTSessionSetProperty(session_.get(), key, value);
|
|
153
|
+
if (status == noErr) {
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
std::array<char, kKeyNameSize> name{};
|
|
157
|
+
CFStringGetCString(key, name.data(), name.size(), kCFStringEncodingUTF8);
|
|
158
|
+
if (required) {
|
|
159
|
+
Invalidate();
|
|
160
|
+
throw PipelineError{kEncoderUnavailableError,
|
|
161
|
+
Failure((std::string("setting ") + name.data()).c_str(), status).message};
|
|
162
|
+
}
|
|
163
|
+
av_log(nullptr, AV_LOG_WARNING, "VideoToolbox rejected %s (OSStatus %d)\n", name.data(),
|
|
164
|
+
static_cast<int>(status));
|
|
165
|
+
};
|
|
166
|
+
set(kVTCompressionPropertyKey_RealTime, kCFBooleanTrue, false);
|
|
167
|
+
set(kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_High_AutoLevel, false);
|
|
168
|
+
set(kVTCompressionPropertyKey_ExpectedFrameRate, Number(params_.fps).get(), false);
|
|
169
|
+
set(kVTCompressionPropertyKey_AllowFrameReordering, kCFBooleanFalse, true);
|
|
170
|
+
set(kVTCompressionPropertyKey_AverageBitRate,
|
|
171
|
+
Number(static_cast<int32_t>(std::min<int64_t>(params_.bitrate, INT32_MAX))).get(), true);
|
|
172
|
+
set(kVTCompressionPropertyKey_MaxKeyFrameInterval, Number(static_cast<int32_t>(params_.gopFrames)).get(), true);
|
|
173
|
+
set(kVTCompressionPropertyKey_ColorPrimaries, kCMFormatDescriptionColorPrimaries_ITU_R_709_2, true);
|
|
174
|
+
set(kVTCompressionPropertyKey_TransferFunction, kCMFormatDescriptionTransferFunction_ITU_R_709_2, true);
|
|
175
|
+
set(kVTCompressionPropertyKey_YCbCrMatrix, kCMFormatDescriptionYCbCrMatrix_ITU_R_709_2, true);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
void Invalidate() {
|
|
179
|
+
if (session_) {
|
|
180
|
+
VTCompressionSessionInvalidate(session_.get());
|
|
181
|
+
session_.reset();
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
static void CopyPlanes(const AVFrame& nv12, CVPixelBufferRef buffer) {
|
|
186
|
+
if (const CVReturn locked = CVPixelBufferLockBaseAddress(buffer, 0); locked != kCVReturnSuccess) {
|
|
187
|
+
throw Failure("pixel buffer lock", locked);
|
|
188
|
+
}
|
|
189
|
+
for (size_t plane = 0; plane < 2; ++plane) {
|
|
190
|
+
auto* dst = static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(buffer, plane));
|
|
191
|
+
if (dst == nullptr) {
|
|
192
|
+
CVPixelBufferUnlockBaseAddress(buffer, 0);
|
|
193
|
+
throw Failure("pixel buffer mapping", kCVReturnError);
|
|
194
|
+
}
|
|
195
|
+
const size_t dstStride = CVPixelBufferGetBytesPerRowOfPlane(buffer, plane);
|
|
196
|
+
const size_t rows = CVPixelBufferGetHeightOfPlane(buffer, plane);
|
|
197
|
+
const auto srcStride = static_cast<size_t>(nv12.linesize[plane]);
|
|
198
|
+
const size_t rowBytes = std::min(srcStride, dstStride);
|
|
199
|
+
for (size_t row = 0; row < rows; ++row) {
|
|
200
|
+
std::memcpy(dst + (row * dstStride), nv12.data[plane] + (row * srcStride), rowBytes);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
CVPixelBufferUnlockBaseAddress(buffer, 0);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// Runs on a VideoToolbox thread; the encoding thread is blocked in CompleteFrames until it returns.
|
|
207
|
+
static void OnOutput(void* self, void* /*frameRefCon*/, OSStatus status, VTEncodeInfoFlags flags,
|
|
208
|
+
CMSampleBufferRef sample) {
|
|
209
|
+
static_cast<VideoToolboxEncoder*>(self)->Collect(status, flags, sample);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
void Collect(OSStatus status, VTEncodeInfoFlags flags, CMSampleBufferRef sample) {
|
|
213
|
+
const std::scoped_lock<std::mutex> lock(mutex_);
|
|
214
|
+
if (status != noErr) {
|
|
215
|
+
failure_ = status;
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
if (sample == nullptr || (flags & kVTEncodeInfo_FrameDropped) != 0U || CMSampleBufferDataIsReady(sample) == 0U) {
|
|
219
|
+
return;
|
|
220
|
+
}
|
|
221
|
+
try {
|
|
222
|
+
pending_.push_back(ToAnnexB(sample));
|
|
223
|
+
} catch (...) { // this is a VideoToolbox thread: nothing may escape (bad_alloc included)
|
|
224
|
+
failure_ = kCMSampleBufferError_InvalidMediaFormat;
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// AVCC (length-prefixed) to Annex-B with 4-byte start codes; SPS/PPS are prepended on key frames.
|
|
229
|
+
static EncodedUnit ToAnnexB(CMSampleBufferRef sample) {
|
|
230
|
+
EncodedUnit unit;
|
|
231
|
+
unit.isKeyFrame = IsKeyFrame(sample);
|
|
232
|
+
unit.timestampMicros = CMTimeConvertScale(CMSampleBufferGetPresentationTimeStamp(sample), kMicrosTimescale,
|
|
233
|
+
kCMTimeRoundingMethod_Default)
|
|
234
|
+
.value;
|
|
235
|
+
CMFormatDescriptionRef format = CMSampleBufferGetFormatDescription(sample);
|
|
236
|
+
size_t parameterSets = 0;
|
|
237
|
+
int lengthSize = 0;
|
|
238
|
+
const uint8_t* first = nullptr;
|
|
239
|
+
size_t firstSize = 0;
|
|
240
|
+
if (format == nullptr ||
|
|
241
|
+
CMVideoFormatDescriptionGetH264ParameterSetAtIndex(format, 0, &first, &firstSize, ¶meterSets,
|
|
242
|
+
&lengthSize) != noErr ||
|
|
243
|
+
lengthSize < static_cast<int>(kMinNalLengthSize) || lengthSize > static_cast<int>(kMaxNalLengthSize)) {
|
|
244
|
+
throw PipelineError{"MediaProxyError", "VideoToolbox produced a sample without H.264 parameter sets"};
|
|
245
|
+
}
|
|
246
|
+
CMBlockBufferRef block = CMSampleBufferGetDataBuffer(sample);
|
|
247
|
+
size_t total = 0;
|
|
248
|
+
size_t contiguous = 0;
|
|
249
|
+
char* data = nullptr;
|
|
250
|
+
if (block == nullptr || CMBlockBufferGetDataPointer(block, 0, &contiguous, &total, &data) != noErr ||
|
|
251
|
+
contiguous != total) {
|
|
252
|
+
throw PipelineError{"MediaProxyError", "VideoToolbox produced a fragmented sample"};
|
|
253
|
+
}
|
|
254
|
+
unit.data.reserve(total + 128);
|
|
255
|
+
if (unit.isKeyFrame) {
|
|
256
|
+
for (size_t i = 0; i < parameterSets; ++i) {
|
|
257
|
+
const uint8_t* set = nullptr;
|
|
258
|
+
size_t size = 0;
|
|
259
|
+
if (CMVideoFormatDescriptionGetH264ParameterSetAtIndex(format, i, &set, &size, nullptr, nullptr) == noErr) {
|
|
260
|
+
AppendNal(unit.data, set, size);
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
const auto* bytes = reinterpret_cast<const uint8_t*>(data);
|
|
265
|
+
const auto prefix = static_cast<size_t>(lengthSize);
|
|
266
|
+
for (size_t offset = 0; offset + prefix <= total;) {
|
|
267
|
+
size_t length = 0;
|
|
268
|
+
for (size_t i = 0; i < prefix; ++i) {
|
|
269
|
+
length = (length << kByteBits) | bytes[offset + i];
|
|
270
|
+
}
|
|
271
|
+
offset += prefix;
|
|
272
|
+
if (length > total - offset) {
|
|
273
|
+
throw PipelineError{"MediaProxyError", "VideoToolbox produced a malformed sample"};
|
|
274
|
+
}
|
|
275
|
+
if (length > 0) {
|
|
276
|
+
AppendNal(unit.data, bytes + offset, length);
|
|
277
|
+
}
|
|
278
|
+
offset += length;
|
|
279
|
+
}
|
|
280
|
+
return unit;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
void Deliver() {
|
|
284
|
+
std::vector<EncodedUnit> units;
|
|
285
|
+
OSStatus failure = noErr;
|
|
286
|
+
{
|
|
287
|
+
const std::scoped_lock<std::mutex> lock(mutex_);
|
|
288
|
+
units.swap(pending_);
|
|
289
|
+
failure = std::exchange(failure_, noErr);
|
|
290
|
+
}
|
|
291
|
+
for (EncodedUnit& unit : units) {
|
|
292
|
+
sink_(std::move(unit));
|
|
293
|
+
}
|
|
294
|
+
if (failure != noErr) {
|
|
295
|
+
throw Failure("output", failure);
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
H264Params params_;
|
|
300
|
+
H264Encoder::Sink sink_;
|
|
301
|
+
std::mutex mutex_;
|
|
302
|
+
std::vector<EncodedUnit> pending_;
|
|
303
|
+
OSStatus failure_ = noErr;
|
|
304
|
+
int64_t lastPts_ = -1;
|
|
305
|
+
std::unique_ptr<std::remove_pointer_t<VTCompressionSessionRef>, SessionDeleter> session_;
|
|
306
|
+
};
|
|
307
|
+
|
|
308
|
+
} // namespace
|
|
309
|
+
|
|
310
|
+
std::unique_ptr<H264Encoder> CreateVideoToolboxEncoder(const H264Params& params, const H264Encoder::Sink& sink) {
|
|
311
|
+
return std::make_unique<VideoToolboxEncoder>(params, sink);
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
} // namespace media_proxy
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
#pragma once
|
|
2
|
+
|
|
3
|
+
#include <memory>
|
|
4
|
+
|
|
5
|
+
#include "h264_encoder.h"
|
|
6
|
+
|
|
7
|
+
namespace media_proxy {
|
|
8
|
+
|
|
9
|
+
// macOS VideoToolbox, called directly: FFmpeg's wrapper only returns a packet on the next send, which costs
|
|
10
|
+
// several frames of latency. Throws PipelineError (MediaProxyEncoderUnavailableError) if the session can't open.
|
|
11
|
+
std::unique_ptr<H264Encoder> CreateVideoToolboxEncoder(const H264Params& params, const H264Encoder::Sink& sink);
|
|
12
|
+
|
|
13
|
+
} // namespace media_proxy
|
package/src/native.ts
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import {createRequire} from 'node:module';
|
|
2
|
+
|
|
3
|
+
import {MediaProxyUnavailableError} from './errors.js';
|
|
4
|
+
import {PACKAGE_NAME, getPkgRoot} from './pkg-root.js';
|
|
5
|
+
import type {NativeModule} from './types.js';
|
|
6
|
+
|
|
7
|
+
const require = createRequire(import.meta.url);
|
|
8
|
+
|
|
9
|
+
export const SUPPORTED_PLATFORMS = ['darwin', 'linux', 'win32'] as const;
|
|
10
|
+
export const SUPPORTED_ARCHS = ['x64', 'arm64'] as const;
|
|
11
|
+
|
|
12
|
+
/** Throws {@link MediaProxyUnavailableError} unless this platform/arch has a build. */
|
|
13
|
+
export function assertSupportedPlatform(platform: string = process.platform, arch: string = process.arch): void {
|
|
14
|
+
if (
|
|
15
|
+
!(SUPPORTED_PLATFORMS as readonly string[]).includes(platform) ||
|
|
16
|
+
!(SUPPORTED_ARCHS as readonly string[]).includes(arch)
|
|
17
|
+
) {
|
|
18
|
+
throw new MediaProxyUnavailableError(
|
|
19
|
+
`${PACKAGE_NAME} supports darwin/linux/win32 on x64/arm64, not ${platform}-${arch}`,
|
|
20
|
+
platform,
|
|
21
|
+
arch,
|
|
22
|
+
);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Wraps `load` so nothing runs until the first call. A failure is re-typed as
|
|
28
|
+
* {@link MediaProxyUnavailableError} and not cached (retried on the next call); success is cached.
|
|
29
|
+
*/
|
|
30
|
+
export function createNativeLoader(load: () => unknown): () => NativeModule {
|
|
31
|
+
let native: NativeModule | undefined;
|
|
32
|
+
return () => {
|
|
33
|
+
if (native) {
|
|
34
|
+
return native;
|
|
35
|
+
}
|
|
36
|
+
assertSupportedPlatform();
|
|
37
|
+
try {
|
|
38
|
+
native = load() as NativeModule;
|
|
39
|
+
} catch (err) {
|
|
40
|
+
throw new MediaProxyUnavailableError(
|
|
41
|
+
`No usable ${PACKAGE_NAME} addon for ${process.platform}-${process.arch}: ${(err as Error).message}`,
|
|
42
|
+
process.platform,
|
|
43
|
+
process.arch,
|
|
44
|
+
);
|
|
45
|
+
}
|
|
46
|
+
return native;
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// Never invoked at import or install time, only by the first API call that needs the addon.
|
|
51
|
+
export const loadNative = createNativeLoader(() => require('node-gyp-build')(getPkgRoot()));
|
package/src/pkg-root.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import {fileURLToPath} from 'node:url';
|
|
2
|
+
|
|
3
|
+
import {node, util} from '@appium/support';
|
|
4
|
+
|
|
5
|
+
export const PACKAGE_NAME = '@appium/media-proxy';
|
|
6
|
+
|
|
7
|
+
/** Package root (contains binding.gyp, prebuilds/, or build/ after compile). */
|
|
8
|
+
export const getPkgRoot = util.memoize(function resolvePkgRoot(): string {
|
|
9
|
+
const root = node.getModuleRootSync(PACKAGE_NAME, fileURLToPath(import.meta.url));
|
|
10
|
+
if (!root) {
|
|
11
|
+
throw new Error(`Could not locate the root of the '${PACKAGE_NAME}' package`);
|
|
12
|
+
}
|
|
13
|
+
return root;
|
|
14
|
+
});
|