@stream-io/video-react-native-sdk 0.10.3 → 0.10.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/dist/commonjs/components/Call/CallContent/CallContent.js +10 -5
- package/dist/commonjs/components/Call/CallContent/CallContent.js.map +1 -1
- package/dist/commonjs/components/Call/CallContent/RTCViewPipIOS.js +109 -0
- package/dist/commonjs/components/Call/CallContent/RTCViewPipIOS.js.map +1 -0
- package/dist/commonjs/components/Call/CallContent/index.js +11 -0
- package/dist/commonjs/components/Call/CallContent/index.js.map +1 -1
- package/dist/commonjs/components/Call/CallLayout/CallParticipantsGrid.js +4 -3
- package/dist/commonjs/components/Call/CallLayout/CallParticipantsGrid.js.map +1 -1
- package/dist/commonjs/components/Call/CallLayout/CallParticipantsSpotlight.js +4 -3
- package/dist/commonjs/components/Call/CallLayout/CallParticipantsSpotlight.js.map +1 -1
- package/dist/commonjs/hooks/useAutoEnterPiPEffect.js +3 -3
- package/dist/commonjs/hooks/useAutoEnterPiPEffect.js.map +1 -1
- package/dist/commonjs/hooks/useIsInPiPMode.js +4 -4
- package/dist/commonjs/hooks/useIsInPiPMode.js.map +1 -1
- package/dist/commonjs/providers/StreamCall.js +5 -2
- package/dist/commonjs/providers/StreamCall.js.map +1 -1
- package/dist/commonjs/utils/internal/shouldDisableIOSLocalVideoOnBackground.js +10 -0
- package/dist/commonjs/utils/internal/shouldDisableIOSLocalVideoOnBackground.js.map +1 -0
- package/dist/commonjs/version.js +1 -1
- package/dist/module/components/Call/CallContent/CallContent.js +10 -5
- package/dist/module/components/Call/CallContent/CallContent.js.map +1 -1
- package/dist/module/components/Call/CallContent/RTCViewPipIOS.js +101 -0
- package/dist/module/components/Call/CallContent/RTCViewPipIOS.js.map +1 -0
- package/dist/module/components/Call/CallContent/index.js +1 -0
- package/dist/module/components/Call/CallContent/index.js.map +1 -1
- package/dist/module/components/Call/CallLayout/CallParticipantsGrid.js +4 -3
- package/dist/module/components/Call/CallLayout/CallParticipantsGrid.js.map +1 -1
- package/dist/module/components/Call/CallLayout/CallParticipantsSpotlight.js +4 -3
- package/dist/module/components/Call/CallLayout/CallParticipantsSpotlight.js.map +1 -1
- package/dist/module/hooks/useAutoEnterPiPEffect.js +3 -3
- package/dist/module/hooks/useAutoEnterPiPEffect.js.map +1 -1
- package/dist/module/hooks/useIsInPiPMode.js +4 -4
- package/dist/module/hooks/useIsInPiPMode.js.map +1 -1
- package/dist/module/providers/StreamCall.js +5 -2
- package/dist/module/providers/StreamCall.js.map +1 -1
- package/dist/module/utils/internal/shouldDisableIOSLocalVideoOnBackground.js +4 -0
- package/dist/module/utils/internal/shouldDisableIOSLocalVideoOnBackground.js.map +1 -0
- package/dist/module/version.js +1 -1
- package/dist/typescript/components/Call/CallContent/CallContent.d.ts +6 -1
- package/dist/typescript/components/Call/CallContent/CallContent.d.ts.map +1 -1
- package/dist/typescript/components/Call/CallContent/RTCViewPipIOS.d.ts +7 -0
- package/dist/typescript/components/Call/CallContent/RTCViewPipIOS.d.ts.map +1 -0
- package/dist/typescript/components/Call/CallContent/index.d.ts +1 -0
- package/dist/typescript/components/Call/CallContent/index.d.ts.map +1 -1
- package/dist/typescript/components/Call/CallLayout/CallParticipantsGrid.d.ts +2 -2
- package/dist/typescript/components/Call/CallLayout/CallParticipantsGrid.d.ts.map +1 -1
- package/dist/typescript/components/Call/CallLayout/CallParticipantsSpotlight.d.ts +2 -2
- package/dist/typescript/components/Call/CallLayout/CallParticipantsSpotlight.d.ts.map +1 -1
- package/dist/typescript/hooks/useAutoEnterPiPEffect.d.ts +1 -1
- package/dist/typescript/hooks/useAutoEnterPiPEffect.d.ts.map +1 -1
- package/dist/typescript/hooks/useIsInPiPMode.d.ts +1 -1
- package/dist/typescript/hooks/useIsInPiPMode.d.ts.map +1 -1
- package/dist/typescript/providers/StreamCall.d.ts.map +1 -1
- package/dist/typescript/utils/internal/shouldDisableIOSLocalVideoOnBackground.d.ts +4 -0
- package/dist/typescript/utils/internal/shouldDisableIOSLocalVideoOnBackground.d.ts.map +1 -0
- package/dist/typescript/version.d.ts +1 -1
- package/ios/PictureInPicture/SampleBufferVideoCallView.swift +52 -0
- package/ios/PictureInPicture/StreamAVPictureInPictureVideoCallViewController.swift +83 -0
- package/ios/PictureInPicture/StreamBufferTransformer.swift +96 -0
- package/ios/PictureInPicture/StreamPictureInPictureController.swift +185 -0
- package/ios/PictureInPicture/StreamPictureInPictureTrackStateAdapter.swift +68 -0
- package/ios/PictureInPicture/StreamPictureInPictureVideoRenderer.swift +250 -0
- package/ios/PictureInPicture/StreamPixelBufferPool.swift +118 -0
- package/ios/PictureInPicture/StreamPixelBufferRepository.swift +98 -0
- package/ios/PictureInPicture/StreamRTCYUVBuffer.swift +249 -0
- package/ios/PictureInPicture/StreamYUVToARGBConversion.swift +128 -0
- package/ios/PictureInPicture/WindowSizePolicy/StreamPictureInPictureAdaptiveWindowSizePolicy.swift +25 -0
- package/ios/PictureInPicture/WindowSizePolicy/StreamPictureInPictureFixedWindowSizePolicy.swift +29 -0
- package/ios/PictureInPicture/WindowSizePolicy/StreamPictureInPictureWindowSizePolicy.swift +14 -0
- package/ios/PictureInPicture/YpCbCrPixelRange+Default.swift +32 -0
- package/ios/RTCViewPip.swift +69 -0
- package/ios/RTCViewPipManager.mm +16 -0
- package/ios/RTCViewPipManager.swift +31 -0
- package/ios/StreamVideoReactNative-Bridging-Header.h +8 -0
- package/ios/StreamVideoReactNative.xcodeproj/project.pbxproj +87 -0
- package/package.json +3 -3
- package/src/components/Call/CallContent/CallContent.tsx +58 -40
- package/src/components/Call/CallContent/RTCViewPipIOS.tsx +138 -0
- package/src/components/Call/CallContent/index.ts +1 -0
- package/src/components/Call/CallLayout/CallParticipantsGrid.tsx +7 -3
- package/src/components/Call/CallLayout/CallParticipantsSpotlight.tsx +7 -3
- package/src/hooks/useAutoEnterPiPEffect.tsx +7 -3
- package/src/hooks/useIsInPiPMode.tsx +6 -4
- package/src/providers/StreamCall.tsx +5 -2
- package/src/utils/internal/shouldDisableIOSLocalVideoOnBackground.ts +3 -0
- package/src/version.ts +1 -1
- package/stream-video-react-native.podspec +29 -3
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright © 2024 Stream.io Inc. All rights reserved.
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import AVKit
|
|
6
|
+
import Foundation
|
|
7
|
+
|
|
8
|
+
/// Describes an object that can be used to present picture-in-picture content.
|
|
9
|
+
protocol StreamAVPictureInPictureViewControlling: AnyObject {
|
|
10
|
+
|
|
11
|
+
/// The closure to call whenever the picture-in-picture window size changes.
|
|
12
|
+
var onSizeUpdate: ((CGSize) -> Void)? { get set }
|
|
13
|
+
|
|
14
|
+
/// The track that will be rendered on picture-in-picture window.
|
|
15
|
+
var track: RTCVideoTrack? { get set }
|
|
16
|
+
|
|
17
|
+
/// The preferred size for the picture-in-picture window.
|
|
18
|
+
/// - Important: This should **always** be greater to ``CGSize.zero``. If not, iOS throws
|
|
19
|
+
/// a cryptic error with content `PGPegasus code:-1003`
|
|
20
|
+
var preferredContentSize: CGSize { get set }
|
|
21
|
+
|
|
22
|
+
/// The layer that renders the incoming frames from WebRTC.
|
|
23
|
+
var displayLayer: CALayer { get }
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
@available(iOS 15.0, *)
|
|
27
|
+
final class StreamAVPictureInPictureVideoCallViewController: AVPictureInPictureVideoCallViewController,
|
|
28
|
+
StreamAVPictureInPictureViewControlling {
|
|
29
|
+
|
|
30
|
+
private let contentView: StreamPictureInPictureVideoRenderer =
|
|
31
|
+
.init(windowSizePolicy: StreamPictureInPictureAdaptiveWindowSizePolicy())
|
|
32
|
+
|
|
33
|
+
var onSizeUpdate: ((CGSize) -> Void)?
|
|
34
|
+
|
|
35
|
+
var track: RTCVideoTrack? {
|
|
36
|
+
get { contentView.track }
|
|
37
|
+
set { contentView.track = newValue }
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
var displayLayer: CALayer { contentView.displayLayer }
|
|
41
|
+
|
|
42
|
+
// MARK: - Lifecycle
|
|
43
|
+
|
|
44
|
+
@available(*, unavailable)
|
|
45
|
+
required init?(coder: NSCoder) { fatalError("init(coder:) has not been implemented") }
|
|
46
|
+
|
|
47
|
+
/// Initializes a new instance and sets the `preferredContentSize` to `Self.defaultPreferredContentSize`
|
|
48
|
+
/// value.
|
|
49
|
+
required init() {
|
|
50
|
+
super.init(nibName: nil, bundle: nil)
|
|
51
|
+
contentView.pictureInPictureWindowSizePolicy.controller = self
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
override func viewDidLoad() {
|
|
56
|
+
super.viewDidLoad()
|
|
57
|
+
setUp()
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
override func viewDidLayoutSubviews() {
|
|
61
|
+
super.viewDidLayoutSubviews()
|
|
62
|
+
contentView.bounds = view.bounds
|
|
63
|
+
onSizeUpdate?(contentView.bounds.size)
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// MARK: - Private helpers
|
|
67
|
+
|
|
68
|
+
private func setUp() {
|
|
69
|
+
view.subviews.forEach { $0.removeFromSuperview() }
|
|
70
|
+
|
|
71
|
+
contentView.translatesAutoresizingMaskIntoConstraints = false
|
|
72
|
+
|
|
73
|
+
view.addSubview(contentView)
|
|
74
|
+
NSLayoutConstraint.activate([
|
|
75
|
+
contentView.leadingAnchor.constraint(equalTo: view.leadingAnchor),
|
|
76
|
+
contentView.trailingAnchor.constraint(equalTo: view.trailingAnchor),
|
|
77
|
+
contentView.topAnchor.constraint(equalTo: view.topAnchor),
|
|
78
|
+
contentView.bottomAnchor.constraint(equalTo: view.bottomAnchor)
|
|
79
|
+
])
|
|
80
|
+
|
|
81
|
+
contentView.bounds = view.bounds
|
|
82
|
+
}
|
|
83
|
+
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright © 2024 Stream.io Inc. All rights reserved.
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import Foundation
|
|
6
|
+
|
|
7
|
+
/// `StreamBufferTransformer` is a struct that provides methods for transforming RTCI420Buffer to
|
|
8
|
+
/// CVPixelBuffer, while performing downsampling when necessary.
|
|
9
|
+
struct StreamBufferTransformer {
|
|
10
|
+
|
|
11
|
+
var requiresResize = false
|
|
12
|
+
|
|
13
|
+
/// Transforms an RTCVideoFrame's buffer to a CMSampleBuffer with optional resizing.
|
|
14
|
+
/// - Note: The current implementation always handles an i420 buffer as RTCCVPixelBuffer have been
|
|
15
|
+
/// proven problematic.
|
|
16
|
+
/// - Parameters:
|
|
17
|
+
/// - source: The source RTCVideoFrameBuffer to be transformed.
|
|
18
|
+
/// - targetSize: The target size for the resulting CMSampleBuffer.
|
|
19
|
+
/// - Returns: A transformed RTCVideoFrame or nil if transformation fails.
|
|
20
|
+
func transformAndResizeIfRequired(
|
|
21
|
+
_ frame: RTCVideoFrame,
|
|
22
|
+
targetSize: CGSize
|
|
23
|
+
) -> RTCVideoFrame? {
|
|
24
|
+
let pixelBuffer: RTCVideoFrameBuffer? = {
|
|
25
|
+
if let i420buffer = frame.buffer as? RTCI420Buffer {
|
|
26
|
+
return i420buffer
|
|
27
|
+
} else {
|
|
28
|
+
return frame.buffer
|
|
29
|
+
}
|
|
30
|
+
}()
|
|
31
|
+
|
|
32
|
+
if
|
|
33
|
+
let pixelBuffer = pixelBuffer,
|
|
34
|
+
let buffer = transformAndResizeIfRequired(pixelBuffer, targetSize: targetSize) {
|
|
35
|
+
return .init(
|
|
36
|
+
buffer: buffer,
|
|
37
|
+
rotation: frame.rotation,
|
|
38
|
+
timeStampNs: frame.timeStampNs
|
|
39
|
+
)
|
|
40
|
+
} else {
|
|
41
|
+
return nil
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// MARK: - Private API
|
|
46
|
+
|
|
47
|
+
/// Transforms an RTCVideoFrameBuffer to a CMSampleBuffer with optional resizing.
|
|
48
|
+
/// - Note: The current implementation always handles an i420 buffer as RTCCVPixelBuffer have been
|
|
49
|
+
/// proven problematic.
|
|
50
|
+
/// - Parameters:
|
|
51
|
+
/// - source: The source RTCVideoFrameBuffer to be transformed.
|
|
52
|
+
/// - targetSize: The target size for the resulting CMSampleBuffer.
|
|
53
|
+
/// - Returns: A transformed CMSampleBuffer or nil if transformation fails.
|
|
54
|
+
func transformAndResizeIfRequired(
|
|
55
|
+
_ source: RTCVideoFrameBuffer,
|
|
56
|
+
targetSize: CGSize
|
|
57
|
+
) -> StreamRTCYUVBuffer? {
|
|
58
|
+
let sourceSize = CGSize(width: Int(source.width), height: Int(source.height))
|
|
59
|
+
let resultBuffer: StreamRTCYUVBuffer? = {
|
|
60
|
+
if requiresResize {
|
|
61
|
+
return .init(source: source)
|
|
62
|
+
.resize(to: resizeSize(sourceSize, toFitWithin: targetSize))
|
|
63
|
+
} else {
|
|
64
|
+
return .init(source: source)
|
|
65
|
+
}
|
|
66
|
+
}()
|
|
67
|
+
|
|
68
|
+
return resultBuffer
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/// Calculates the new size to fit within a container size while maintaining the aspect ratio.
|
|
72
|
+
///
|
|
73
|
+
/// - Parameters:
|
|
74
|
+
/// - size: The original size.
|
|
75
|
+
/// - containerSize: The container size to fit within.
|
|
76
|
+
/// - Returns: The new size that fits within the container while preserving the aspect ratio.
|
|
77
|
+
private func resizeSize(
|
|
78
|
+
_ size: CGSize,
|
|
79
|
+
toFitWithin containerSize: CGSize
|
|
80
|
+
) -> CGSize {
|
|
81
|
+
let widthRatio = containerSize.width / size.width
|
|
82
|
+
let heightRatio = containerSize.height / size.height
|
|
83
|
+
|
|
84
|
+
// Choose the smaller ratio to ensure that the entire original size fits
|
|
85
|
+
// within the container.
|
|
86
|
+
let ratioToUse = min(widthRatio, heightRatio)
|
|
87
|
+
|
|
88
|
+
// Calculate the new size while maintaining the aspect ratio.
|
|
89
|
+
let newSize = CGSize(
|
|
90
|
+
width: size.width * ratioToUse,
|
|
91
|
+
height: size.height * ratioToUse
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
return newSize
|
|
95
|
+
}
|
|
96
|
+
}
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright © 2024 Stream.io Inc. All rights reserved.
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import AVKit
|
|
6
|
+
import Combine
|
|
7
|
+
import Foundation
|
|
8
|
+
|
|
9
|
+
/// A controller class for picture-in-picture whenever that is possible.
|
|
10
|
+
@objc final class StreamPictureInPictureController: NSObject, AVPictureInPictureControllerDelegate {
|
|
11
|
+
|
|
12
|
+
// MARK: - Properties
|
|
13
|
+
|
|
14
|
+
/// The RTCVideoTrack for which the picture-in-picture session is created.
|
|
15
|
+
@objc public var track: RTCVideoTrack? {
|
|
16
|
+
didSet {
|
|
17
|
+
didUpdate(track) // Called when the `track` property changes
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/// The UIView that contains the video content.
|
|
22
|
+
@objc public var sourceView: UIView? {
|
|
23
|
+
didSet {
|
|
24
|
+
didUpdate(sourceView) // Called when the `sourceView` property changes
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/// A closure called when the picture-in-picture view's size changes.
|
|
29
|
+
public var onSizeUpdate: ((CGSize) -> Void)? {
|
|
30
|
+
didSet {
|
|
31
|
+
contentViewController?.onSizeUpdate = onSizeUpdate // Updates the onSizeUpdate closure of the content view controller
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/// A boolean value indicating whether the picture-in-picture session should start automatically when the app enters background.
|
|
36
|
+
public var canStartPictureInPictureAutomaticallyFromInline: Bool
|
|
37
|
+
|
|
38
|
+
// MARK: - Private Properties
|
|
39
|
+
|
|
40
|
+
/// The AVPictureInPictureController object.
|
|
41
|
+
private var pictureInPictureController: AVPictureInPictureController?
|
|
42
|
+
|
|
43
|
+
/// The StreamAVPictureInPictureViewControlling object that manages the picture-in-picture view.
|
|
44
|
+
private var contentViewController: StreamAVPictureInPictureViewControlling?
|
|
45
|
+
|
|
46
|
+
/// A set of `AnyCancellable` objects used to manage subscriptions.
|
|
47
|
+
private var cancellableBag: Set<AnyCancellable> = []
|
|
48
|
+
|
|
49
|
+
/// A `AnyCancellable` object used to ensure that the active track is enabled while in picture-in-picture
|
|
50
|
+
/// mode.
|
|
51
|
+
private var ensureActiveTrackIsEnabledCancellable: AnyCancellable?
|
|
52
|
+
|
|
53
|
+
/// A `StreamPictureInPictureTrackStateAdapter` object that manages the state of the
|
|
54
|
+
/// active track.
|
|
55
|
+
private let trackStateAdapter: StreamPictureInPictureTrackStateAdapter = .init()
|
|
56
|
+
|
|
57
|
+
// MARK: - Lifecycle
|
|
58
|
+
|
|
59
|
+
/// Initializes the controller and creates the content view
|
|
60
|
+
///
|
|
61
|
+
/// - Parameter canStartPictureInPictureAutomaticallyFromInline A boolean value
|
|
62
|
+
/// indicating whether the picture-in-picture session should start automatically when the app enters
|
|
63
|
+
/// background.
|
|
64
|
+
///
|
|
65
|
+
/// - Returns `nil` if AVPictureInPictureController is not supported, or the controller otherwise.
|
|
66
|
+
init?(canStartPictureInPictureAutomaticallyFromInline: Bool = true) {
|
|
67
|
+
guard AVPictureInPictureController.isPictureInPictureSupported() else {
|
|
68
|
+
return nil
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
let contentViewController: StreamAVPictureInPictureViewControlling? = {
|
|
72
|
+
if #available(iOS 15.0, *) {
|
|
73
|
+
return StreamAVPictureInPictureVideoCallViewController()
|
|
74
|
+
} else {
|
|
75
|
+
return nil
|
|
76
|
+
}
|
|
77
|
+
}()
|
|
78
|
+
// contentViewController?.preferredContentSize = .init(width: 400, height: 320)
|
|
79
|
+
self.contentViewController = contentViewController
|
|
80
|
+
self.canStartPictureInPictureAutomaticallyFromInline = canStartPictureInPictureAutomaticallyFromInline
|
|
81
|
+
super.init()
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// MARK: - AVPictureInPictureControllerDelegate
|
|
85
|
+
|
|
86
|
+
func pictureInPictureController(
|
|
87
|
+
_ pictureInPictureController: AVPictureInPictureController,
|
|
88
|
+
restoreUserInterfaceForPictureInPictureStopWithCompletionHandler completionHandler: @escaping (Bool) -> Void
|
|
89
|
+
) {
|
|
90
|
+
completionHandler(true)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
public func pictureInPictureControllerWillStartPictureInPicture(
|
|
94
|
+
_ pictureInPictureController: AVPictureInPictureController
|
|
95
|
+
) {
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
public func pictureInPictureControllerDidStartPictureInPicture(
|
|
99
|
+
_ pictureInPictureController: AVPictureInPictureController
|
|
100
|
+
) {
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
public func pictureInPictureController(
|
|
104
|
+
_ pictureInPictureController: AVPictureInPictureController,
|
|
105
|
+
failedToStartPictureInPictureWithError error: Error
|
|
106
|
+
) {
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
public func pictureInPictureControllerWillStopPictureInPicture(
|
|
110
|
+
_ pictureInPictureController: AVPictureInPictureController
|
|
111
|
+
) {
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
public func pictureInPictureControllerDidStopPictureInPicture(
|
|
115
|
+
_ pictureInPictureController: AVPictureInPictureController
|
|
116
|
+
) {
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// MARK: - Private helpers
|
|
120
|
+
|
|
121
|
+
private func didUpdate(_ track: RTCVideoTrack?) {
|
|
122
|
+
contentViewController?.track = track
|
|
123
|
+
trackStateAdapter.activeTrack = track
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
@objc private func didUpdate(_ sourceView: UIView?) {
|
|
127
|
+
if let sourceView {
|
|
128
|
+
// If picture-in-picture isn't active, just create a new controller.
|
|
129
|
+
if pictureInPictureController?.isPictureInPictureActive != true {
|
|
130
|
+
makePictureInPictureController(with: sourceView)
|
|
131
|
+
|
|
132
|
+
pictureInPictureController?
|
|
133
|
+
.publisher(for: \.isPictureInPicturePossible)
|
|
134
|
+
.sink { NSLog("PiP - isPictureInPicturePossible:\($0)") }
|
|
135
|
+
.store(in: &cancellableBag)
|
|
136
|
+
|
|
137
|
+
pictureInPictureController?
|
|
138
|
+
.publisher(for: \.isPictureInPictureActive)
|
|
139
|
+
.removeDuplicates()
|
|
140
|
+
.sink { [weak self] in self?.didUpdatePictureInPictureActiveState($0) }
|
|
141
|
+
.store(in: &cancellableBag)
|
|
142
|
+
} else {
|
|
143
|
+
// If picture-in-picture is active, simply update the sourceView.
|
|
144
|
+
makePictureInPictureController(with: sourceView)
|
|
145
|
+
}
|
|
146
|
+
} else {
|
|
147
|
+
if #available(iOS 15.0, *) {
|
|
148
|
+
pictureInPictureController?.contentSource = nil
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
@objc func cleanup() {
|
|
154
|
+
contentViewController?.track = nil
|
|
155
|
+
if #available(iOS 15.0, *) {
|
|
156
|
+
pictureInPictureController?.contentSource = nil
|
|
157
|
+
}
|
|
158
|
+
pictureInPictureController?.delegate = nil
|
|
159
|
+
pictureInPictureController = nil
|
|
160
|
+
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
private func makePictureInPictureController(with sourceView: UIView) {
|
|
164
|
+
if #available(iOS 15.0, *),
|
|
165
|
+
let contentViewController = contentViewController as? StreamAVPictureInPictureVideoCallViewController {
|
|
166
|
+
pictureInPictureController = .init(
|
|
167
|
+
contentSource: .init(
|
|
168
|
+
activeVideoCallSourceView: sourceView,
|
|
169
|
+
contentViewController: contentViewController
|
|
170
|
+
)
|
|
171
|
+
)
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
if #available(iOS 14.2, *) {
|
|
175
|
+
pictureInPictureController?
|
|
176
|
+
.canStartPictureInPictureAutomaticallyFromInline = canStartPictureInPictureAutomaticallyFromInline
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
pictureInPictureController?.delegate = self
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
private func didUpdatePictureInPictureActiveState(_ isActive: Bool) {
|
|
183
|
+
trackStateAdapter.isEnabled = isActive
|
|
184
|
+
}
|
|
185
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright © 2024 Stream.io Inc. All rights reserved.
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import Combine
|
|
6
|
+
import Foundation
|
|
7
|
+
|
|
8
|
+
/// StreamPictureInPictureTrackStateAdapter serves as an adapter for managing the state of a video track
|
|
9
|
+
/// used for picture-in-picture functionality. It can enable or disable observers based on its isEnabled property
|
|
10
|
+
/// and ensures that the active track is always enabled when necessary.
|
|
11
|
+
final class StreamPictureInPictureTrackStateAdapter {
|
|
12
|
+
|
|
13
|
+
/// This property represents whether the adapter is enabled or not.
|
|
14
|
+
var isEnabled: Bool = false {
|
|
15
|
+
didSet {
|
|
16
|
+
/// When the 'isEnabled' property changes, this didSet observer is called.
|
|
17
|
+
/// It checks if the new value is different from the old value, and if so,
|
|
18
|
+
/// it calls the 'enableObserver' function.
|
|
19
|
+
guard isEnabled != oldValue else { return }
|
|
20
|
+
enableObserver(isEnabled)
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/// This property represents the active RTCVideoTrack.
|
|
25
|
+
var activeTrack: RTCVideoTrack? {
|
|
26
|
+
didSet {
|
|
27
|
+
/// When the 'activeTrack' property changes, this didSet observer is called.
|
|
28
|
+
/// If the adapter is enabled and the new 'activeTrack' is different from the old one,
|
|
29
|
+
/// it disables the old track (if it exists).
|
|
30
|
+
if isEnabled, oldValue?.trackId != activeTrack?.trackId {
|
|
31
|
+
oldValue?.isEnabled = false
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
// MARK: - Private helpers
|
|
37
|
+
|
|
38
|
+
/// This property holds a reference to the observer cancellable.
|
|
39
|
+
private var observerCancellable: AnyCancellable?
|
|
40
|
+
|
|
41
|
+
/// This private function enables or disables an observer based on the 'isActive' parameter.
|
|
42
|
+
///
|
|
43
|
+
/// - Parameter isActive: A Boolean value indicating whether the observer should be active.
|
|
44
|
+
private func enableObserver(_ isActive: Bool) {
|
|
45
|
+
if isActive {
|
|
46
|
+
/// If 'isActive' is true, it sets up an observer that checks tracks state periodically.
|
|
47
|
+
observerCancellable = Timer
|
|
48
|
+
.publish(every: 0.1, on: .main, in: .default)
|
|
49
|
+
.autoconnect()
|
|
50
|
+
.receive(on: DispatchQueue.main)
|
|
51
|
+
.sink { [weak self] _ in
|
|
52
|
+
self?.checkTracksState()
|
|
53
|
+
}
|
|
54
|
+
} else {
|
|
55
|
+
/// If 'isActive' is false, it cancels the observer.
|
|
56
|
+
observerCancellable?.cancel()
|
|
57
|
+
observerCancellable = nil
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/// This private function checks the state of the active track and enables it if it's not already enabled.
|
|
62
|
+
private func checkTracksState() {
|
|
63
|
+
if let activeTrack, !activeTrack.isEnabled {
|
|
64
|
+
// log.info("⚙️Active track:\(activeTrack.trackId) for picture-in-picture will be enabled now.")
|
|
65
|
+
self.activeTrack?.isEnabled = true
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright © 2024 Stream.io Inc. All rights reserved.
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import Combine
|
|
6
|
+
import Foundation
|
|
7
|
+
|
|
8
|
+
/// A view that can be used to render an instance of `RTCVideoTrack`
|
|
9
|
+
final class StreamPictureInPictureVideoRenderer: UIView, RTCVideoRenderer {
|
|
10
|
+
|
|
11
|
+
/// The rendering track.
|
|
12
|
+
var track: RTCVideoTrack? {
|
|
13
|
+
didSet {
|
|
14
|
+
// Whenever the track changes we perform the following operations if possible:
|
|
15
|
+
// - stopFrameStreaming for the old track
|
|
16
|
+
// - startFrameStreaming for the new track and only if we are already
|
|
17
|
+
// in picture-in-picture.
|
|
18
|
+
guard oldValue != track else { return }
|
|
19
|
+
prepareForTrackRendering(oldValue)
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/// The layer that renders the track's frames.
|
|
24
|
+
var displayLayer: CALayer { contentView.layer }
|
|
25
|
+
|
|
26
|
+
/// A policy defining how the Picture in Picture window should be resized in order to better fit
|
|
27
|
+
/// the rendering frame size.
|
|
28
|
+
var pictureInPictureWindowSizePolicy: PictureInPictureWindowSizePolicy
|
|
29
|
+
|
|
30
|
+
/// The publisher which is used to streamline the frames received from the track.
|
|
31
|
+
private let bufferPublisher: PassthroughSubject<CMSampleBuffer, Never> = .init()
|
|
32
|
+
|
|
33
|
+
/// The view that contains the rendering layer.
|
|
34
|
+
private lazy var contentView: SampleBufferVideoCallView = {
|
|
35
|
+
let contentView = SampleBufferVideoCallView()
|
|
36
|
+
contentView.translatesAutoresizingMaskIntoConstraints = false
|
|
37
|
+
contentView.contentMode = .scaleAspectFill
|
|
38
|
+
contentView.videoGravity = .resizeAspectFill
|
|
39
|
+
contentView.preventsDisplaySleepDuringVideoPlayback = true
|
|
40
|
+
return contentView
|
|
41
|
+
}()
|
|
42
|
+
|
|
43
|
+
/// The transformer used to transform and downsample a RTCVideoFrame's buffer.
|
|
44
|
+
private var bufferTransformer = StreamBufferTransformer()
|
|
45
|
+
|
|
46
|
+
/// The cancellable used to control the bufferPublisher stream.
|
|
47
|
+
private var bufferUpdatesCancellable: AnyCancellable?
|
|
48
|
+
|
|
49
|
+
/// The view's size.
|
|
50
|
+
/// - Note: We are using this property instead for `frame.size` or `bounds.size` so we can
|
|
51
|
+
/// access it from any thread.
|
|
52
|
+
private var contentSize: CGSize = .zero
|
|
53
|
+
|
|
54
|
+
/// The track's size.
|
|
55
|
+
private var trackSize: CGSize = .zero {
|
|
56
|
+
didSet {
|
|
57
|
+
guard trackSize != oldValue else { return }
|
|
58
|
+
didUpdateTrackSize()
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/// A property that defines if the RTCVideoFrame instances that will be rendered need to be resized
|
|
63
|
+
/// to fid the view's contentSize.
|
|
64
|
+
private var requiresResize = false {
|
|
65
|
+
didSet { bufferTransformer.requiresResize = requiresResize }
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/// As we are operate in smaller rendering bounds we skip frames depending on this property's value
|
|
69
|
+
/// to improve performance.
|
|
70
|
+
/// - Note: The number of frames to skip is being calculated based on the ``trackSize`` and
|
|
71
|
+
/// ``contentSize``. It takes into account also the ``sizeRatioThreshold``
|
|
72
|
+
private var noOfFramesToSkipAfterRendering = 1
|
|
73
|
+
|
|
74
|
+
/// The number of frames that we have skipped so far. This is used as a step counter in the
|
|
75
|
+
/// ``renderFrame(_:)``.
|
|
76
|
+
private var skippedFrames = 0
|
|
77
|
+
|
|
78
|
+
/// We render frames every time the stepper/counter value is 0 and have a valid trackSize.
|
|
79
|
+
private var shouldRenderFrame: Bool { skippedFrames == 0 && trackSize != .zero }
|
|
80
|
+
|
|
81
|
+
/// A size ratio threshold used to determine if resizing is required.
|
|
82
|
+
/// - Note: It seems that Picture-in-Picture doesn't like rendering frames that are bigger than its
|
|
83
|
+
/// window size. For this reason, we are setting the resizeThreshold to `1`.
|
|
84
|
+
private let resizeRequiredSizeRatioThreshold: CGFloat = 1
|
|
85
|
+
|
|
86
|
+
/// A size ratio threshold used to determine if skipping frames is required.
|
|
87
|
+
private let sizeRatioThreshold: CGFloat = 15
|
|
88
|
+
|
|
89
|
+
// MARK: - Lifecycle
|
|
90
|
+
|
|
91
|
+
@available(*, unavailable)
|
|
92
|
+
required init?(coder: NSCoder) { fatalError("init(coder:) has not been implemented") }
|
|
93
|
+
|
|
94
|
+
init(windowSizePolicy: PictureInPictureWindowSizePolicy) {
|
|
95
|
+
pictureInPictureWindowSizePolicy = windowSizePolicy
|
|
96
|
+
super.init(frame: .zero)
|
|
97
|
+
setUp()
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
override func willMove(toWindow newWindow: UIWindow?) {
|
|
101
|
+
super.willMove(toWindow: newWindow)
|
|
102
|
+
// Depending on the window we are moving we either start or stop
|
|
103
|
+
// streaming frames from the track.
|
|
104
|
+
if newWindow != nil {
|
|
105
|
+
startFrameStreaming(for: track, on: newWindow)
|
|
106
|
+
} else {
|
|
107
|
+
stopFrameStreaming(for: track)
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
override func layoutSubviews() {
|
|
112
|
+
super.layoutSubviews()
|
|
113
|
+
contentSize = frame.size
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// MARK: - Rendering lifecycle
|
|
117
|
+
|
|
118
|
+
/// This method is being called from WebRTC and asks the container to set its size to the track's size.
|
|
119
|
+
func setSize(_ size: CGSize) {
|
|
120
|
+
trackSize = size
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
func renderFrame(_ frame: RTCVideoFrame?) {
|
|
124
|
+
guard let frame = frame else {
|
|
125
|
+
return
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// Update the trackSize and re-calculate rendering properties if the size
|
|
129
|
+
// has changed.
|
|
130
|
+
trackSize = .init(width: Int(frame.width), height: Int(frame.height))
|
|
131
|
+
|
|
132
|
+
defer {
|
|
133
|
+
handleFrameSkippingIfRequired()
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
guard shouldRenderFrame else {
|
|
137
|
+
return
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
if
|
|
141
|
+
let yuvBuffer = bufferTransformer.transformAndResizeIfRequired(frame, targetSize: contentSize)?
|
|
142
|
+
.buffer as? StreamRTCYUVBuffer,
|
|
143
|
+
let sampleBuffer = yuvBuffer.sampleBuffer {
|
|
144
|
+
bufferPublisher.send(sampleBuffer)
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// MARK: - Private helpers
|
|
149
|
+
|
|
150
|
+
/// Set up the view's hierarchy.
|
|
151
|
+
private func setUp() {
|
|
152
|
+
addSubview(contentView)
|
|
153
|
+
NSLayoutConstraint.activate([
|
|
154
|
+
contentView.leadingAnchor.constraint(equalTo: leadingAnchor),
|
|
155
|
+
contentView.trailingAnchor.constraint(equalTo: trailingAnchor),
|
|
156
|
+
contentView.topAnchor.constraint(equalTo: topAnchor),
|
|
157
|
+
contentView.bottomAnchor.constraint(equalTo: bottomAnchor)
|
|
158
|
+
])
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/// A method used to process the frame's buffer and enqueue on the rendering view.
|
|
162
|
+
private func process(_ buffer: CMSampleBuffer) {
|
|
163
|
+
guard
|
|
164
|
+
bufferUpdatesCancellable != nil,
|
|
165
|
+
let trackId = track?.trackId,
|
|
166
|
+
buffer.isValid
|
|
167
|
+
else {
|
|
168
|
+
contentView.renderingComponent.flush()
|
|
169
|
+
return
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
if #available(iOS 14.0, *) {
|
|
173
|
+
if contentView.renderingComponent.requiresFlushToResumeDecoding == true {
|
|
174
|
+
contentView.renderingComponent.flush()
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
if contentView.renderingComponent.isReadyForMoreMediaData {
|
|
179
|
+
contentView.renderingComponent.enqueue(buffer)
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/// A method used to start consuming frames from the track.
|
|
184
|
+
/// - Note: In order to avoid unnecessary processing, we only start consuming track's frames when
|
|
185
|
+
/// the view has been added on a window (which means that picture-in-picture view is visible).
|
|
186
|
+
private func startFrameStreaming(
|
|
187
|
+
for track: RTCVideoTrack?,
|
|
188
|
+
on window: UIWindow?
|
|
189
|
+
) {
|
|
190
|
+
guard window != nil, let track else { return }
|
|
191
|
+
|
|
192
|
+
bufferUpdatesCancellable = bufferPublisher
|
|
193
|
+
.receive(on: DispatchQueue.main)
|
|
194
|
+
.sink { [weak self] in self?.process($0) }
|
|
195
|
+
|
|
196
|
+
track.add(self)
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/// A method that stops the frame consumption from the track. Used automatically when the rendering
|
|
200
|
+
/// view move's away from the window or when the track changes.
|
|
201
|
+
private func stopFrameStreaming(for track: RTCVideoTrack?) {
|
|
202
|
+
guard bufferUpdatesCancellable != nil else { return }
|
|
203
|
+
bufferUpdatesCancellable?.cancel()
|
|
204
|
+
bufferUpdatesCancellable = nil
|
|
205
|
+
track?.remove(self)
|
|
206
|
+
contentView.renderingComponent.flush()
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/// A method used to calculate rendering required properties, every time the trackSize changes.
|
|
210
|
+
private func didUpdateTrackSize() {
|
|
211
|
+
guard contentSize != .zero, trackSize != .zero else { return }
|
|
212
|
+
|
|
213
|
+
let widthDiffRatio = trackSize.width / contentSize.width
|
|
214
|
+
let heightDiffRatio = trackSize.height / contentSize.height
|
|
215
|
+
requiresResize = widthDiffRatio >= resizeRequiredSizeRatioThreshold || heightDiffRatio >= resizeRequiredSizeRatioThreshold
|
|
216
|
+
let requiresFramesSkipping = widthDiffRatio >= sizeRatioThreshold || heightDiffRatio >= sizeRatioThreshold
|
|
217
|
+
|
|
218
|
+
/// Skipping frames is decided based on how much bigger is the incoming frame's size compared
|
|
219
|
+
/// to PiP window's size.
|
|
220
|
+
noOfFramesToSkipAfterRendering = requiresFramesSkipping ? max(Int(max(Int(widthDiffRatio), Int(heightDiffRatio)) / 2), 1) :
|
|
221
|
+
0
|
|
222
|
+
skippedFrames = 0
|
|
223
|
+
|
|
224
|
+
/// We update the provided windowSizePolicy with the size of the track we received, transformed
|
|
225
|
+
/// to the value that fits.
|
|
226
|
+
pictureInPictureWindowSizePolicy.trackSize = trackSize
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/// A method used to handle the frameSkipping(step) during frame consumption.
|
|
230
|
+
private func handleFrameSkippingIfRequired() {
|
|
231
|
+
if noOfFramesToSkipAfterRendering > 0 {
|
|
232
|
+
if skippedFrames == noOfFramesToSkipAfterRendering {
|
|
233
|
+
skippedFrames = 0
|
|
234
|
+
} else {
|
|
235
|
+
skippedFrames += 1
|
|
236
|
+
}
|
|
237
|
+
} else if skippedFrames > 0 {
|
|
238
|
+
skippedFrames = 0
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/// A method used to prepare the view for a new track rendering.
|
|
243
|
+
private func prepareForTrackRendering(_ oldValue: RTCVideoTrack?) {
|
|
244
|
+
stopFrameStreaming(for: oldValue)
|
|
245
|
+
noOfFramesToSkipAfterRendering = 0
|
|
246
|
+
skippedFrames = 0
|
|
247
|
+
requiresResize = false
|
|
248
|
+
startFrameStreaming(for: track, on: window)
|
|
249
|
+
}
|
|
250
|
+
}
|