@stream-io/video-react-native-sdk 1.0.0-rc2.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -83
- package/dist/commonjs/components/Call/CallContent/CallContent.js +10 -5
- package/dist/commonjs/components/Call/CallContent/CallContent.js.map +1 -1
- package/dist/commonjs/components/Call/CallContent/RTCViewPipIOS.js +109 -0
- package/dist/commonjs/components/Call/CallContent/RTCViewPipIOS.js.map +1 -0
- package/dist/commonjs/components/Call/CallContent/index.js +11 -0
- package/dist/commonjs/components/Call/CallContent/index.js.map +1 -1
- package/dist/commonjs/components/Call/CallLayout/CallParticipantsGrid.js +4 -3
- package/dist/commonjs/components/Call/CallLayout/CallParticipantsGrid.js.map +1 -1
- package/dist/commonjs/components/Call/CallLayout/CallParticipantsSpotlight.js +4 -3
- package/dist/commonjs/components/Call/CallLayout/CallParticipantsSpotlight.js.map +1 -1
- package/dist/commonjs/hooks/useAutoEnterPiPEffect.js +3 -3
- package/dist/commonjs/hooks/useAutoEnterPiPEffect.js.map +1 -1
- package/dist/commonjs/hooks/useIsInPiPMode.js +4 -4
- package/dist/commonjs/hooks/useIsInPiPMode.js.map +1 -1
- package/dist/commonjs/providers/StreamCall.js +5 -2
- package/dist/commonjs/providers/StreamCall.js.map +1 -1
- package/dist/commonjs/utils/internal/shouldDisableIOSLocalVideoOnBackground.js +10 -0
- package/dist/commonjs/utils/internal/shouldDisableIOSLocalVideoOnBackground.js.map +1 -0
- package/dist/commonjs/version.js +1 -1
- package/dist/commonjs/version.js.map +1 -1
- package/dist/module/components/Call/CallContent/CallContent.js +10 -5
- package/dist/module/components/Call/CallContent/CallContent.js.map +1 -1
- package/dist/module/components/Call/CallContent/RTCViewPipIOS.js +101 -0
- package/dist/module/components/Call/CallContent/RTCViewPipIOS.js.map +1 -0
- package/dist/module/components/Call/CallContent/index.js +1 -0
- package/dist/module/components/Call/CallContent/index.js.map +1 -1
- package/dist/module/components/Call/CallLayout/CallParticipantsGrid.js +4 -3
- package/dist/module/components/Call/CallLayout/CallParticipantsGrid.js.map +1 -1
- package/dist/module/components/Call/CallLayout/CallParticipantsSpotlight.js +4 -3
- package/dist/module/components/Call/CallLayout/CallParticipantsSpotlight.js.map +1 -1
- package/dist/module/hooks/useAutoEnterPiPEffect.js +3 -3
- package/dist/module/hooks/useAutoEnterPiPEffect.js.map +1 -1
- package/dist/module/hooks/useIsInPiPMode.js +4 -4
- package/dist/module/hooks/useIsInPiPMode.js.map +1 -1
- package/dist/module/providers/StreamCall.js +5 -2
- package/dist/module/providers/StreamCall.js.map +1 -1
- package/dist/module/utils/internal/shouldDisableIOSLocalVideoOnBackground.js +4 -0
- package/dist/module/utils/internal/shouldDisableIOSLocalVideoOnBackground.js.map +1 -0
- package/dist/module/version.js +1 -1
- package/dist/module/version.js.map +1 -1
- package/dist/typescript/components/Call/CallContent/CallContent.d.ts +6 -1
- package/dist/typescript/components/Call/CallContent/CallContent.d.ts.map +1 -1
- package/dist/typescript/components/Call/CallContent/RTCViewPipIOS.d.ts +7 -0
- package/dist/typescript/components/Call/CallContent/RTCViewPipIOS.d.ts.map +1 -0
- package/dist/typescript/components/Call/CallContent/index.d.ts +1 -0
- package/dist/typescript/components/Call/CallContent/index.d.ts.map +1 -1
- package/dist/typescript/components/Call/CallLayout/CallParticipantsGrid.d.ts +2 -2
- package/dist/typescript/components/Call/CallLayout/CallParticipantsGrid.d.ts.map +1 -1
- package/dist/typescript/components/Call/CallLayout/CallParticipantsSpotlight.d.ts +2 -2
- package/dist/typescript/components/Call/CallLayout/CallParticipantsSpotlight.d.ts.map +1 -1
- package/dist/typescript/hooks/useAutoEnterPiPEffect.d.ts +1 -1
- package/dist/typescript/hooks/useAutoEnterPiPEffect.d.ts.map +1 -1
- package/dist/typescript/hooks/useIsInPiPMode.d.ts +1 -1
- package/dist/typescript/hooks/useIsInPiPMode.d.ts.map +1 -1
- package/dist/typescript/providers/StreamCall.d.ts.map +1 -1
- package/dist/typescript/utils/internal/shouldDisableIOSLocalVideoOnBackground.d.ts +4 -0
- package/dist/typescript/utils/internal/shouldDisableIOSLocalVideoOnBackground.d.ts.map +1 -0
- package/dist/typescript/version.d.ts +1 -1
- package/dist/typescript/version.d.ts.map +1 -1
- package/ios/PictureInPicture/SampleBufferVideoCallView.swift +52 -0
- package/ios/PictureInPicture/StreamAVPictureInPictureVideoCallViewController.swift +83 -0
- package/ios/PictureInPicture/StreamBufferTransformer.swift +96 -0
- package/ios/PictureInPicture/StreamPictureInPictureController.swift +185 -0
- package/ios/PictureInPicture/StreamPictureInPictureTrackStateAdapter.swift +68 -0
- package/ios/PictureInPicture/StreamPictureInPictureVideoRenderer.swift +250 -0
- package/ios/PictureInPicture/StreamPixelBufferPool.swift +118 -0
- package/ios/PictureInPicture/StreamPixelBufferRepository.swift +98 -0
- package/ios/PictureInPicture/StreamRTCYUVBuffer.swift +249 -0
- package/ios/PictureInPicture/StreamYUVToARGBConversion.swift +128 -0
- package/ios/PictureInPicture/WindowSizePolicy/StreamPictureInPictureAdaptiveWindowSizePolicy.swift +25 -0
- package/ios/PictureInPicture/WindowSizePolicy/StreamPictureInPictureFixedWindowSizePolicy.swift +29 -0
- package/ios/PictureInPicture/WindowSizePolicy/StreamPictureInPictureWindowSizePolicy.swift +14 -0
- package/ios/PictureInPicture/YpCbCrPixelRange+Default.swift +32 -0
- package/ios/RTCViewPip.swift +69 -0
- package/ios/RTCViewPipManager.mm +16 -0
- package/ios/RTCViewPipManager.swift +34 -0
- package/ios/StreamVideoReactNative-Bridging-Header.h +11 -0
- package/package.json +4 -4
- package/src/components/Call/CallContent/CallContent.tsx +58 -40
- package/src/components/Call/CallContent/RTCViewPipIOS.tsx +138 -0
- package/src/components/Call/CallContent/index.ts +1 -0
- package/src/components/Call/CallLayout/CallParticipantsGrid.tsx +7 -3
- package/src/components/Call/CallLayout/CallParticipantsSpotlight.tsx +7 -3
- package/src/hooks/useAutoEnterPiPEffect.tsx +7 -3
- package/src/hooks/useIsInPiPMode.tsx +6 -4
- package/src/providers/StreamCall.tsx +5 -2
- package/src/utils/internal/shouldDisableIOSLocalVideoOnBackground.ts +3 -0
- package/src/version.ts +1 -1
- package/stream-video-react-native.podspec +27 -4
- package/ios/StreamVideoReactNative.xcodeproj/project.pbxproj +0 -274
- package/ios/StreamVideoReactNative.xcodeproj/project.xcworkspace/contents.xcworkspacedata +0 -7
- package/ios/StreamVideoReactNative.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +0 -8
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright © 2024 Stream.io Inc. All rights reserved.
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import Accelerate
|
|
6
|
+
import CoreVideo
|
|
7
|
+
import Foundation
|
|
8
|
+
|
|
9
|
+
/// A class that encapsulates the conversion of RTC video frame buffers from YUV to ARGB format.
|
|
10
|
+
final class StreamRTCYUVBuffer: NSObject, RTCVideoFrameBuffer {
|
|
11
|
+
|
|
12
|
+
private let pixelBufferRepository = StreamPixelBufferRepository.init()
|
|
13
|
+
|
|
14
|
+
/// The original source of the video frame, conforming to `RTCVideoFrameBuffer`.
|
|
15
|
+
private let source: RTCVideoFrameBuffer
|
|
16
|
+
|
|
17
|
+
/// The conversion mechanism from YUV to ARGB.
|
|
18
|
+
private let conversion: StreamYUVToARGBConversion
|
|
19
|
+
|
|
20
|
+
/// The width of the video frame.
|
|
21
|
+
var width: Int32 { source.width }
|
|
22
|
+
|
|
23
|
+
/// The height of the video frame.
|
|
24
|
+
var height: Int32 { source.height }
|
|
25
|
+
|
|
26
|
+
/// Lazily initialized pixel buffer that stores the converted YUV to ARGB data.
|
|
27
|
+
private lazy var i420ToYUVPixelBuffer = buildI420ToYUVPixelBuffer()
|
|
28
|
+
|
|
29
|
+
/// Initializes a new buffer with the given source and conversion setup.
|
|
30
|
+
///
|
|
31
|
+
/// - Parameters:
|
|
32
|
+
/// - source: The video frame source.
|
|
33
|
+
/// - conversion: The conversion configuration, default initialized if not provided.
|
|
34
|
+
init(
|
|
35
|
+
source: RTCVideoFrameBuffer,
|
|
36
|
+
conversion: StreamYUVToARGBConversion = .init()
|
|
37
|
+
) {
|
|
38
|
+
self.source = source
|
|
39
|
+
self.conversion = conversion
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/// Converts the frame to the I420 format.
|
|
43
|
+
///
|
|
44
|
+
/// - Returns: An object conforming to `RTCI420BufferProtocol`.
|
|
45
|
+
func toI420() -> any RTCI420BufferProtocol {
|
|
46
|
+
if let i420 = source as? RTCI420Buffer {
|
|
47
|
+
return i420
|
|
48
|
+
} else {
|
|
49
|
+
return source.toI420()
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/// Resizes the current buffer resized to the target size.
|
|
54
|
+
///
|
|
55
|
+
/// - Parameter targetSize: The target size for the buffer.
|
|
56
|
+
/// - Returns: A new `StreamRTCYUVBuffer` with the resized content or nil if resizing fails.
|
|
57
|
+
func resize(to targetSize: CGSize) -> StreamRTCYUVBuffer? {
|
|
58
|
+
if let i420 = source as? RTCI420Buffer {
|
|
59
|
+
let resizedSource = i420.cropAndScale(
|
|
60
|
+
with: 0,
|
|
61
|
+
offsetY: 0,
|
|
62
|
+
cropWidth: Int32(source.width),
|
|
63
|
+
cropHeight: Int32(source.height),
|
|
64
|
+
scaleWidth: Int32(targetSize.width),
|
|
65
|
+
scaleHeight: Int32(targetSize.height)
|
|
66
|
+
)
|
|
67
|
+
return .init(source: resizedSource, conversion: conversion)
|
|
68
|
+
} else if
|
|
69
|
+
let pixelBuffer = source as? RTCCVPixelBuffer,
|
|
70
|
+
let dequeuedPixelBuffer = try? pixelBufferRepository.dequeuePixelBuffer(
|
|
71
|
+
of: targetSize,
|
|
72
|
+
pixelFormat: CVPixelBufferGetPixelFormatType(pixelBuffer.pixelBuffer)
|
|
73
|
+
) {
|
|
74
|
+
let count = pixelBuffer.bufferSizeForCroppingAndScaling(toWidth: Int32(targetSize.width), height: Int32(targetSize.height))
|
|
75
|
+
let tempBuffer: UnsafeMutableRawPointer? = malloc(Int(count))
|
|
76
|
+
pixelBuffer.cropAndScale(to: dequeuedPixelBuffer, withTempBuffer: tempBuffer)
|
|
77
|
+
tempBuffer?.deallocate()
|
|
78
|
+
return .init(source: RTCCVPixelBuffer(pixelBuffer: dequeuedPixelBuffer))
|
|
79
|
+
} else {
|
|
80
|
+
return nil
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/// Retrieves the underlying pixel buffer if available.
|
|
85
|
+
var pixelBuffer: CVPixelBuffer? {
|
|
86
|
+
if source is RTCI420Buffer {
|
|
87
|
+
return i420ToYUVPixelBuffer
|
|
88
|
+
} else if let pixelBuffer = source as? RTCCVPixelBuffer {
|
|
89
|
+
return pixelBuffer.pixelBuffer
|
|
90
|
+
} else {
|
|
91
|
+
return nil
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/// Creates a CMSampleBuffer from the current pixel buffer, if available.
|
|
96
|
+
var sampleBuffer: CMSampleBuffer? {
|
|
97
|
+
guard let pixelBuffer else {
|
|
98
|
+
return nil
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
var sampleBuffer: CMSampleBuffer?
|
|
102
|
+
|
|
103
|
+
var timingInfo = CMSampleTimingInfo()
|
|
104
|
+
var formatDescription: CMFormatDescription?
|
|
105
|
+
CMVideoFormatDescriptionCreateForImageBuffer(
|
|
106
|
+
allocator: kCFAllocatorDefault,
|
|
107
|
+
imageBuffer: pixelBuffer,
|
|
108
|
+
formatDescriptionOut: &formatDescription
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
guard let formatDescription = formatDescription else {
|
|
112
|
+
// log.error("Cannot create sample buffer formatDescription.")
|
|
113
|
+
return nil
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
_ = CMSampleBufferCreateReadyWithImageBuffer(
|
|
117
|
+
allocator: kCFAllocatorDefault,
|
|
118
|
+
imageBuffer: pixelBuffer,
|
|
119
|
+
formatDescription: formatDescription,
|
|
120
|
+
sampleTiming: &timingInfo,
|
|
121
|
+
sampleBufferOut: &sampleBuffer
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
guard let buffer = sampleBuffer else {
|
|
125
|
+
// log.error("Cannot create sample buffer")
|
|
126
|
+
return nil
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
let attachments: CFArray! = CMSampleBufferGetSampleAttachmentsArray(
|
|
130
|
+
buffer,
|
|
131
|
+
createIfNecessary: true
|
|
132
|
+
)
|
|
133
|
+
let dictionary = unsafeBitCast(
|
|
134
|
+
CFArrayGetValueAtIndex(attachments, 0),
|
|
135
|
+
to: CFMutableDictionary.self
|
|
136
|
+
)
|
|
137
|
+
let key = Unmanaged.passUnretained(kCMSampleAttachmentKey_DisplayImmediately).toOpaque()
|
|
138
|
+
let value = Unmanaged.passUnretained(kCFBooleanTrue).toOpaque()
|
|
139
|
+
CFDictionarySetValue(dictionary, key, value)
|
|
140
|
+
|
|
141
|
+
return buffer
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// MARK: - Private Helpers
|
|
145
|
+
|
|
146
|
+
/// Creates a pixel buffer converted from I420 to YUV format.
|
|
147
|
+
///
|
|
148
|
+
/// - Returns: A `CVPixelBuffer` containing the converted data or nil if the conversion fails.
|
|
149
|
+
private func buildI420ToYUVPixelBuffer() -> CVPixelBuffer? {
|
|
150
|
+
guard let source = source as? RTCI420Buffer else {
|
|
151
|
+
return nil
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
do {
|
|
155
|
+
let pixelBuffer = try pixelBufferRepository.dequeuePixelBuffer(
|
|
156
|
+
of: .init(width: Int(width), height: Int(height))
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
var YpImageBuffer = buildYpImageBuffer(source)
|
|
160
|
+
var CbImageBuffer = buildCbImageBuffer(source)
|
|
161
|
+
var CrImageBuffer = buildCrImageBuffer(source)
|
|
162
|
+
|
|
163
|
+
CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly)
|
|
164
|
+
var output = buildImageBuffer(from: pixelBuffer)
|
|
165
|
+
/// The `vImageConvert_420Yp8_Cb8_Cr8ToARGB8888` will convert our buffer
|
|
166
|
+
/// to ARGB pixel format. However, for rendering we require BGRA pixel format. The
|
|
167
|
+
/// permuteMaps adds an instruction to move around the generated ARGB buffers to the
|
|
168
|
+
/// positions described by the array:
|
|
169
|
+
/// Example:
|
|
170
|
+
/// The resulted array will be: [0: Alpha, 1: Red, 2: Green, 3: Blue]. The array that we want
|
|
171
|
+
/// to get though will have the format [0: Blue, 1: Green, 2: Red, 3: Alpha].
|
|
172
|
+
let error = vImageConvert_420Yp8_Cb8_Cr8ToARGB8888(
|
|
173
|
+
&YpImageBuffer,
|
|
174
|
+
&CbImageBuffer,
|
|
175
|
+
&CrImageBuffer,
|
|
176
|
+
&output,
|
|
177
|
+
&conversion.output,
|
|
178
|
+
[3, 2, 1, 0],
|
|
179
|
+
255,
|
|
180
|
+
vImage_Flags(kvImageNoFlags)
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly)
|
|
184
|
+
|
|
185
|
+
if error == kvImageNoError {
|
|
186
|
+
return pixelBuffer
|
|
187
|
+
} else {
|
|
188
|
+
// log.error(error)
|
|
189
|
+
return nil
|
|
190
|
+
}
|
|
191
|
+
} catch {
|
|
192
|
+
// log.error(error)
|
|
193
|
+
return nil
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/// Constructs a `vImage_Buffer` for the Y plane from the source I420 buffer.
|
|
198
|
+
///
|
|
199
|
+
/// - Parameter source: The source I420 buffer.
|
|
200
|
+
/// - Returns: A `vImage_Buffer` representing the Y plane.
|
|
201
|
+
private func buildYpImageBuffer(_ source: RTCI420Buffer) -> vImage_Buffer {
|
|
202
|
+
vImage_Buffer(
|
|
203
|
+
data: UnsafeMutablePointer(mutating: source.dataY),
|
|
204
|
+
height: vImagePixelCount(height),
|
|
205
|
+
width: vImagePixelCount(width),
|
|
206
|
+
rowBytes: Int(source.strideY)
|
|
207
|
+
)
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/// Constructs a `vImage_Buffer` for the Cb plane from the source I420 buffer.
|
|
211
|
+
///
|
|
212
|
+
/// - Parameter source: The source I420 buffer.
|
|
213
|
+
/// - Returns: A `vImage_Buffer` representing the Cb plane.
|
|
214
|
+
private func buildCbImageBuffer(_ source: RTCI420Buffer) -> vImage_Buffer {
|
|
215
|
+
vImage_Buffer(
|
|
216
|
+
data: UnsafeMutablePointer(mutating: source.dataU),
|
|
217
|
+
height: vImagePixelCount(source.chromaHeight),
|
|
218
|
+
width: vImagePixelCount(source.chromaWidth),
|
|
219
|
+
rowBytes: Int(source.strideU)
|
|
220
|
+
)
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/// Constructs a `vImage_Buffer` for the Cr plane from the source I420 buffer.
|
|
224
|
+
///
|
|
225
|
+
/// - Parameter source: The source I420 buffer.
|
|
226
|
+
/// - Returns: A `vImage_Buffer` representing the Cr plane.
|
|
227
|
+
private func buildCrImageBuffer(_ source: RTCI420Buffer) -> vImage_Buffer {
|
|
228
|
+
vImage_Buffer(
|
|
229
|
+
data: UnsafeMutablePointer(mutating: source.dataV),
|
|
230
|
+
height: vImagePixelCount(source.chromaHeight),
|
|
231
|
+
width: vImagePixelCount(source.chromaWidth),
|
|
232
|
+
rowBytes: Int(source.strideV)
|
|
233
|
+
)
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/// Creates a `vImage_Buffer` from a CVPixelBuffer.
|
|
237
|
+
///
|
|
238
|
+
/// - Parameter pixelBuffer: The pixel buffer to convert.
|
|
239
|
+
/// - Returns: A `vImage_Buffer` representing the given pixel buffer.
|
|
240
|
+
private func buildImageBuffer(from pixelBuffer: CVPixelBuffer) -> vImage_Buffer {
|
|
241
|
+
let baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer)!
|
|
242
|
+
return vImage_Buffer(
|
|
243
|
+
data: baseAddress,
|
|
244
|
+
height: vImagePixelCount(height),
|
|
245
|
+
width: vImagePixelCount(width),
|
|
246
|
+
rowBytes: CVPixelBufferGetBytesPerRow(pixelBuffer)
|
|
247
|
+
)
|
|
248
|
+
}
|
|
249
|
+
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright © 2024 Stream.io Inc. All rights reserved.
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import Accelerate
|
|
6
|
+
import Foundation
|
|
7
|
+
|
|
8
|
+
/// A class dedicated to converting YUV (YpCbCr) image data to ARGB format.
|
|
9
|
+
final class StreamYUVToARGBConversion {
|
|
10
|
+
|
|
11
|
+
/// Enum defining the color conversion coefficients for YUV to ARGB conversion.
|
|
12
|
+
///
|
|
13
|
+
/// The coefficient in the context of `vImageConvert_YpCbCrToARGB_GenerateConversion`
|
|
14
|
+
/// refers to a set of values used to define the color conversion matrix when transforming YpCbCr pixel
|
|
15
|
+
/// format images to ARGB format. This conversion is crucial in image and video processing where color
|
|
16
|
+
/// space compatibility is necessary for various rendering, compositing, or display purposes.
|
|
17
|
+
///
|
|
18
|
+
/// Understanding YpCbCr to ARGB Conversion YpCbCr is a color space used in video compression
|
|
19
|
+
/// and broadcasting, where Yp represents the luma component (the brightness), and Cb and Cr
|
|
20
|
+
/// represent the chroma components (the color details). Converting YpCbCr to ARGB (Alpha, Red, Green, Blue)
|
|
21
|
+
/// involves transforming these components into a format commonly used in digital images, which
|
|
22
|
+
/// includes separate channels for red, green, blue, and an alpha transparency channel.
|
|
23
|
+
///
|
|
24
|
+
/// Role of the Coefficient
|
|
25
|
+
/// - Color Space Transformation: The coefficients are used to create a matrix that mathematically
|
|
26
|
+
/// transforms the YpCbCr values into RGB values. This matrix accounts for the differences in color
|
|
27
|
+
/// representation between the two formats, ensuring accurate color rendition.
|
|
28
|
+
///
|
|
29
|
+
/// - Handling Various Standards: Different video standards (like ITU-R BT.601, ITU-R BT.709, etc.)
|
|
30
|
+
/// define different coefficients because they assume different color primaries (red, green, and blue points)
|
|
31
|
+
/// and different luma/chroma formulations. The coefficient matrix you choose should match the standard
|
|
32
|
+
/// used when the YpCbCr data was originally created to ensure color accuracy.
|
|
33
|
+
///
|
|
34
|
+
/// - Performance Optimization: Using a precalculated conversion matrix (which the coefficients help define)
|
|
35
|
+
/// allows for highly optimized, performant image processing. This is critical in real-time applications,
|
|
36
|
+
/// like video playback or editing, where processing speed is crucial.
|
|
37
|
+
///
|
|
38
|
+
/// - Adjusting Luminance and Chrominance: The coefficients can also adjust the scale and bias of the
|
|
39
|
+
/// luminance (Yp) and chrominance (Cb and Cr) to match the expected range of the ARGB format.
|
|
40
|
+
/// This is essential for maintaining the correct brightness, contrast, and color saturation in the
|
|
41
|
+
/// converted image.
|
|
42
|
+
///
|
|
43
|
+
/// Usage in `vImageConvert_YpCbCrToARGB_GenerateConversion`
|
|
44
|
+
/// When you use `vImageConvert_YpCbCrToARGB_GenerateConversion`, you typically provide a
|
|
45
|
+
/// `vImage_YpCbCrToARGB` structure that includes the coefficients. The function then uses these
|
|
46
|
+
/// coefficients to populate the structure with the necessary data to perform the conversion efficiently.
|
|
47
|
+
/// The populated structure can subsequently be used with other vImage functions to convert image
|
|
48
|
+
/// buffers from YpCbCr to ARGB.
|
|
49
|
+
///
|
|
50
|
+
/// The correct selection and use of the coefficient matrix are vital for achieving accurate color conversion,
|
|
51
|
+
/// maintaining image quality, and ensuring consistency across various processing stages or devices.
|
|
52
|
+
/// The ability to specify different coefficients makes the vImage framework flexible and capable of
|
|
53
|
+
/// handling various video standards and custom conversion needs.
|
|
54
|
+
///
|
|
55
|
+
/// - Parameters:
|
|
56
|
+
/// - YpCbCrToARGBMatrix_ITU_R_601_4: ITU-R Recommendation BT.601, often used for
|
|
57
|
+
/// standard-definition television.
|
|
58
|
+
/// - YpCbCrToARGBMatrix_ITU_R_709_2: ITU-R Recommendation BT.709, often used for
|
|
59
|
+
/// high-definition television.
|
|
60
|
+
enum Coefficient {
|
|
61
|
+
/// ITU-R Recommendation BT.601, often used for standard-definition video.
|
|
62
|
+
case YpCbCrToARGBMatrix_ITU_R_601_4
|
|
63
|
+
|
|
64
|
+
/// ITU-R Recommendation BT.709, often used for high-definition video.
|
|
65
|
+
case YpCbCrToARGBMatrix_ITU_R_709_2
|
|
66
|
+
|
|
67
|
+
/// Computed property to provide a pointer to the relevant conversion matrix.
|
|
68
|
+
///
|
|
69
|
+
/// - Returns: A pointer to the selected color conversion matrix.
|
|
70
|
+
var value: UnsafePointer<vImage_YpCbCrToARGBMatrix> {
|
|
71
|
+
switch self {
|
|
72
|
+
case .YpCbCrToARGBMatrix_ITU_R_601_4: return kvImage_YpCbCrToARGBMatrix_ITU_R_601_4
|
|
73
|
+
case .YpCbCrToARGBMatrix_ITU_R_709_2: return kvImage_YpCbCrToARGBMatrix_ITU_R_709_2
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/// The pixel range configuration for YUV to ARGB conversion, defaulting to standard range.
|
|
79
|
+
private var pixelRange: vImage_YpCbCrPixelRange
|
|
80
|
+
|
|
81
|
+
/// The coefficient matrix to use, defaulting to ITU-R BT.601.
|
|
82
|
+
private var coefficient: Coefficient
|
|
83
|
+
|
|
84
|
+
/// The type of YpCbCr pixel data, default set to a common format.
|
|
85
|
+
private var inYpCbCrType: vImageYpCbCrType
|
|
86
|
+
|
|
87
|
+
/// The output ARGB pixel format, default set to 8 bits per channel.
|
|
88
|
+
private var outARGBType: vImageARGBType
|
|
89
|
+
|
|
90
|
+
/// Flags for the conversion process, with no flags set by default.
|
|
91
|
+
private var flags: UInt32
|
|
92
|
+
|
|
93
|
+
/// The resulting conversion object to be used for converting YUV to ARGB.
|
|
94
|
+
var output: vImage_YpCbCrToARGB
|
|
95
|
+
|
|
96
|
+
/// Initializes the conversion setup with optional custom parameters.
|
|
97
|
+
///
|
|
98
|
+
/// - Parameters:
|
|
99
|
+
/// - pixelRange: The pixel range configuration, defaulting to `.default`.
|
|
100
|
+
/// - coefficient: The coefficient matrix to use, defaulting to `.YpCbCrToARGBMatrix_ITU_R_601_4`.
|
|
101
|
+
/// - inYpCbCrType: The type of YpCbCr pixel data, default set to `kvImage420Yp8_Cb8_Cr8`.
|
|
102
|
+
/// - outARGBType: The output ARGB pixel format, default set to `kvImageARGB8888`.
|
|
103
|
+
/// - flags: Flags for the conversion process, with no flags set by default.
|
|
104
|
+
init(
|
|
105
|
+
pixelRange: vImage_YpCbCrPixelRange = .default,
|
|
106
|
+
coefficient: Coefficient = .YpCbCrToARGBMatrix_ITU_R_601_4,
|
|
107
|
+
inYpCbCrType: vImageYpCbCrType = kvImage420Yp8_Cb8_Cr8,
|
|
108
|
+
outARGBType: vImageARGBType = kvImageARGB8888,
|
|
109
|
+
flags: UInt32 = UInt32(kvImageNoFlags)
|
|
110
|
+
) {
|
|
111
|
+
self.pixelRange = pixelRange
|
|
112
|
+
self.coefficient = coefficient
|
|
113
|
+
self.inYpCbCrType = inYpCbCrType
|
|
114
|
+
self.outARGBType = outARGBType
|
|
115
|
+
self.flags = flags
|
|
116
|
+
output = vImage_YpCbCrToARGB()
|
|
117
|
+
|
|
118
|
+
/// Generates a conversion setup for converting YpCbCr to ARGB using specified parameters.
|
|
119
|
+
vImageConvert_YpCbCrToARGB_GenerateConversion(
|
|
120
|
+
self.coefficient.value,
|
|
121
|
+
&self.pixelRange,
|
|
122
|
+
&output,
|
|
123
|
+
inYpCbCrType,
|
|
124
|
+
outARGBType,
|
|
125
|
+
flags
|
|
126
|
+
)
|
|
127
|
+
}
|
|
128
|
+
}
|
package/ios/PictureInPicture/WindowSizePolicy/StreamPictureInPictureAdaptiveWindowSizePolicy.swift
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
// Copyright © 2024 Stream.io Inc. All rights reserved.
|
|
2
|
+
//
|
|
3
|
+
|
|
4
|
+
import Foundation
|
|
5
|
+
|
|
6
|
+
/// An adaptive window size policy for Picture-in-Picture (PiP) views.
|
|
7
|
+
final class StreamPictureInPictureAdaptiveWindowSizePolicy: PictureInPictureWindowSizePolicy {
|
|
8
|
+
|
|
9
|
+
/// The current size of the track to be displayed in the PiP window.
|
|
10
|
+
var trackSize: CGSize = .zero {
|
|
11
|
+
didSet {
|
|
12
|
+
// Only update the controller's preferred content size if the track size has changed and is not zero.
|
|
13
|
+
guard trackSize != oldValue, trackSize != .zero else {
|
|
14
|
+
return
|
|
15
|
+
}
|
|
16
|
+
controller?.preferredContentSize = trackSize
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/// The controller that manages the PiP view.
|
|
21
|
+
weak var controller: StreamAVPictureInPictureViewControlling?
|
|
22
|
+
|
|
23
|
+
/// Initializes a new instance of the adaptive window size policy.
|
|
24
|
+
init() {}
|
|
25
|
+
}
|
package/ios/PictureInPicture/WindowSizePolicy/StreamPictureInPictureFixedWindowSizePolicy.swift
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright © 2024 Stream.io Inc. All rights reserved.
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import Foundation
|
|
6
|
+
|
|
7
|
+
/// A fixed window size policy for Picture-in-Picture (PiP) views.
|
|
8
|
+
final class StreamPictureInPictureFixedWindowSizePolicy: PictureInPictureWindowSizePolicy {
|
|
9
|
+
|
|
10
|
+
/// The current size of the track to be displayed in the PiP window. This is not used in this policy.
|
|
11
|
+
var trackSize: CGSize = .zero
|
|
12
|
+
|
|
13
|
+
/// The controller that manages the PiP view.
|
|
14
|
+
weak var controller: (any StreamAVPictureInPictureViewControlling)? {
|
|
15
|
+
didSet {
|
|
16
|
+
// Set the preferred content size of the controller to the fixed size.
|
|
17
|
+
controller?.preferredContentSize = fixedSize
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/// The fixed size for the PiP window.
|
|
22
|
+
private let fixedSize: CGSize
|
|
23
|
+
|
|
24
|
+
/// Initializes a new instance of the fixed window size policy with a specified fixed size.
|
|
25
|
+
/// - Parameter fixedSize: The fixed size for the PiP window. Default is 640x480.
|
|
26
|
+
init(_ fixedSize: CGSize = .init(width: 640, height: 480)) {
|
|
27
|
+
self.fixedSize = fixedSize
|
|
28
|
+
}
|
|
29
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright © 2024 Stream.io Inc. All rights reserved.
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import Foundation
|
|
6
|
+
|
|
7
|
+
/// Protocol defining the policy for determining the window size of a Picture-in-Picture (PiP) view.
|
|
8
|
+
protocol PictureInPictureWindowSizePolicy {
|
|
9
|
+
/// The current size of the track to be displayed in the PiP window.
|
|
10
|
+
var trackSize: CGSize { get set }
|
|
11
|
+
|
|
12
|
+
/// The controller that manages the PiP view.
|
|
13
|
+
var controller: StreamAVPictureInPictureViewControlling? { get set }
|
|
14
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright © 2024 Stream.io Inc. All rights reserved.
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import Accelerate
|
|
6
|
+
import Foundation
|
|
7
|
+
|
|
8
|
+
/// Extension of vImage_YpCbCrPixelRange to include a default configuration.
|
|
9
|
+
extension vImage_YpCbCrPixelRange {
|
|
10
|
+
|
|
11
|
+
/// Initializes a default pixel range for YpCbCr pixel format.
|
|
12
|
+
/// This default configuration is often used when converting YpCbCr to RGB.
|
|
13
|
+
/// - Yp_bias: The bias for the Y' (luma) component, typically 0.
|
|
14
|
+
/// - CbCr_bias: The bias for the Cb and Cr (chroma) components, usually set to 128 to center the chroma values.
|
|
15
|
+
/// - YpRangeMax: The maximum value for the Y' range, typically 255.
|
|
16
|
+
/// - CbCrRangeMax: The maximum value for the Cb and Cr range, also usually 255.
|
|
17
|
+
/// - YpMax: The maximum possible value for Y', generally 255.
|
|
18
|
+
/// - YpMin: The minimum possible value for Y', usually set to 1 for video ranges.
|
|
19
|
+
/// - CbCrMax: The maximum possible value for Cb and Cr, typically 255.
|
|
20
|
+
/// - CbCrMin: The minimum possible value for Cb and Cr, often 0.
|
|
21
|
+
/// Reference: [Apple's documentation on vImageConvert_YpCbCrToARGB](https://developer.apple.com/documentation/accelerate/1533189-vimageconvert_ypcbcrtoargb_gener)
|
|
22
|
+
static let `default` = vImage_YpCbCrPixelRange(
|
|
23
|
+
Yp_bias: 0, /// The bias applied to the Y' component.
|
|
24
|
+
CbCr_bias: 128, /// The bias applied to the Cb and Cr components.
|
|
25
|
+
YpRangeMax: 255, /// The maximum value of the Y' range.
|
|
26
|
+
CbCrRangeMax: 255, /// The maximum value of the Cb and Cr range.
|
|
27
|
+
YpMax: 255, /// The maximum value of Y'.
|
|
28
|
+
YpMin: 1, /// The minimum value of Y' (often used for setting video range).
|
|
29
|
+
CbCrMax: 255, /// The maximum value of Cb and Cr.
|
|
30
|
+
CbCrMin: 0 /// The minimum value of Cb and Cr.
|
|
31
|
+
)
|
|
32
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
//
|
|
2
|
+
// RTCViewPip.swift
|
|
3
|
+
// stream-video-react-native
|
|
4
|
+
//
|
|
5
|
+
// Created by santhosh vaiyapuri on 22/08/2024.
|
|
6
|
+
//
|
|
7
|
+
|
|
8
|
+
import Foundation
|
|
9
|
+
|
|
10
|
+
@objc(RTCViewPip)
|
|
11
|
+
class RTCViewPip: UIView {
|
|
12
|
+
|
|
13
|
+
private lazy var pictureInPictureController = StreamPictureInPictureController()
|
|
14
|
+
private var webRtcModule: WebRTCModule?
|
|
15
|
+
|
|
16
|
+
override init(frame: CGRect) {
|
|
17
|
+
super.init(frame: frame)
|
|
18
|
+
setupView()
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
required init?(coder aDecoder: NSCoder) {
|
|
22
|
+
super.init(coder: aDecoder)
|
|
23
|
+
setupView()
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
private func setupView() {
|
|
27
|
+
let videoView = UIView()
|
|
28
|
+
self.addSubview(videoView)
|
|
29
|
+
pictureInPictureController?.sourceView = videoView
|
|
30
|
+
videoView.backgroundColor = UIColor.clear // make it transparent
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
func setWebRtcModule(_ module: WebRTCModule) {
|
|
34
|
+
webRtcModule = module
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
@objc func setStreamURL(_ streamReactTag: NSString) {
|
|
38
|
+
webRtcModule?.workerQueue.async {
|
|
39
|
+
let stream = self.webRtcModule?.stream(forReactTag: String(streamReactTag))
|
|
40
|
+
let videoTracks = stream?.videoTracks ?? []
|
|
41
|
+
let videoTrack = videoTracks.first
|
|
42
|
+
if videoTrack == nil {
|
|
43
|
+
NSLog("PiP - No video stream for react tag: -\(streamReactTag)")
|
|
44
|
+
} else {
|
|
45
|
+
DispatchQueue.main.async {
|
|
46
|
+
self.pictureInPictureController?.track = videoTrack
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
@objc
|
|
53
|
+
func onCallClosed() {
|
|
54
|
+
DispatchQueue.main.async {
|
|
55
|
+
self.pictureInPictureController?.cleanup()
|
|
56
|
+
self.pictureInPictureController = nil
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
override func didMoveToWindow() {
|
|
61
|
+
super.didMoveToWindow()
|
|
62
|
+
let isVisible = self.superview != nil && self.window != nil;
|
|
63
|
+
if (!isVisible) {
|
|
64
|
+
// view is detached so we cleanup the pip controller
|
|
65
|
+
// taken from: https://github.com/software-mansion/react-native-screens/blob/main/Example/ios/ScreensExample/RNSSampleLifecycleAwareView.m
|
|
66
|
+
onCallClosed()
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
//
|
|
2
|
+
// RTCViewPipManager.m
|
|
3
|
+
// stream-video-react-native
|
|
4
|
+
//
|
|
5
|
+
// Created by santhosh vaiyapuri on 22/08/2024.
|
|
6
|
+
//
|
|
7
|
+
// tutorial used: https://teabreak.e-spres-oh.com/swift-in-react-native-the-ultimate-guide-part-2-ui-components-907767123d9e
|
|
8
|
+
|
|
9
|
+
#import <React/RCTViewManager.h>
|
|
10
|
+
|
|
11
|
+
@interface RCT_EXTERN_MODULE(RTCViewPipManager, RCTViewManager)
|
|
12
|
+
|
|
13
|
+
RCT_EXPORT_VIEW_PROPERTY(streamURL, NSString)
|
|
14
|
+
RCT_EXTERN_METHOD(onCallClosed:(nonnull NSNumber*) reactTag)
|
|
15
|
+
|
|
16
|
+
@end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
//
|
|
2
|
+
// RTCViewPipManager.swift
|
|
3
|
+
// stream-video-react-native
|
|
4
|
+
//
|
|
5
|
+
// Created by santhosh vaiyapuri on 22/08/2024.
|
|
6
|
+
//
|
|
7
|
+
|
|
8
|
+
import Foundation
|
|
9
|
+
|
|
10
|
+
@objc (RTCViewPipManager)
|
|
11
|
+
class RTCViewPipManager: RCTViewManager {
|
|
12
|
+
|
|
13
|
+
override func view() -> UIView! {
|
|
14
|
+
let view = RTCViewPip()
|
|
15
|
+
view.setWebRtcModule(self.bridge.module(forName: "WebRTCModule") as! WebRTCModule)
|
|
16
|
+
return view
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
override static func requiresMainQueueSetup() -> Bool {
|
|
20
|
+
return true
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
@objc func onCallClosed(_ reactTag: NSNumber) {
|
|
24
|
+
self.bridge!.uiManager.addUIBlock { (_: RCTUIManager?, viewRegistry: [NSNumber: UIView]?) in
|
|
25
|
+
guard let view = viewRegistry?[reactTag] as? RTCViewPip else {
|
|
26
|
+
if RCT_DEBUG == 1 {
|
|
27
|
+
print("Invalid view returned from registry, expecting RTCViewPip")
|
|
28
|
+
}
|
|
29
|
+
return
|
|
30
|
+
}
|
|
31
|
+
view.onCallClosed()
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
}
|
|
@@ -1,3 +1,14 @@
|
|
|
1
1
|
#import <React/RCTBridgeModule.h>
|
|
2
2
|
#import <React/RCTViewManager.h>
|
|
3
3
|
#import <React/RCTEventEmitter.h>
|
|
4
|
+
#import <React/RCTLog.h>
|
|
5
|
+
#import <React/RCTUIManager.h>
|
|
6
|
+
#import <React/RCTView.h>
|
|
7
|
+
#import <React/RCTBridge.h>
|
|
8
|
+
|
|
9
|
+
#import <WebRTC/RTCCVPixelBuffer.h>
|
|
10
|
+
#import <WebRTC/RTCVideoFrame.h>
|
|
11
|
+
#import <WebRTC/RTCVideoTrack.h>
|
|
12
|
+
#import <WebRTC/RTCVideoRenderer.h>
|
|
13
|
+
#import <WebRTC/RTCVideoFrameBuffer.h>
|
|
14
|
+
#import "WebRTCModule.h"
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@stream-io/video-react-native-sdk",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.1",
|
|
4
4
|
"packageManager": "yarn@3.2.4",
|
|
5
5
|
"main": "dist/commonjs/index.js",
|
|
6
6
|
"module": "dist/module/index.js",
|
|
@@ -46,8 +46,8 @@
|
|
|
46
46
|
"!**/.*"
|
|
47
47
|
],
|
|
48
48
|
"dependencies": {
|
|
49
|
-
"@stream-io/video-client": "1.6.0
|
|
50
|
-
"@stream-io/video-react-bindings": "
|
|
49
|
+
"@stream-io/video-client": "1.6.0",
|
|
50
|
+
"@stream-io/video-react-bindings": "1.0.0",
|
|
51
51
|
"intl-pluralrules": "2.0.1",
|
|
52
52
|
"lodash.merge": "^4.6.2",
|
|
53
53
|
"react-native-url-polyfill": "1.3.0",
|
|
@@ -125,7 +125,7 @@
|
|
|
125
125
|
"@react-native-firebase/messaging": "17.5.0",
|
|
126
126
|
"@react-native/eslint-config": "^0.74.84",
|
|
127
127
|
"@stream-io/react-native-webrtc": "118.1.0",
|
|
128
|
-
"@stream-io/video-filters-react-native": "^0.2.
|
|
128
|
+
"@stream-io/video-filters-react-native": "^0.2.2",
|
|
129
129
|
"@testing-library/jest-native": "^5.4.2",
|
|
130
130
|
"@testing-library/react-native": "^12.1.2",
|
|
131
131
|
"@tsconfig/node14": "14.1.0",
|