@sbhjt-gr/react-native-webrtc 124.0.0 → 124.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.clang-format +11 -11
- package/.eslintignore +6 -6
- package/.nvmrc +1 -1
- package/ISSUE_TEMPLATE.md +40 -40
- package/LICENSE +22 -22
- package/README.md +103 -103
- package/android/build.gradle +37 -37
- package/android/consumer-rules.pro +3 -3
- package/android/src/main/AndroidManifest.xml +11 -11
- package/android/src/main/java/com/oney/WebRTCModule/AbstractVideoCaptureController.java +113 -113
- package/android/src/main/java/com/oney/WebRTCModule/CameraCaptureController.java +338 -338
- package/android/src/main/java/com/oney/WebRTCModule/CameraEventsHandler.java +49 -49
- package/android/src/main/java/com/oney/WebRTCModule/DataChannelWrapper.java +99 -99
- package/android/src/main/java/com/oney/WebRTCModule/DataPacketCryptorManager.java +62 -62
- package/android/src/main/java/com/oney/WebRTCModule/DisplayUtils.java +16 -16
- package/android/src/main/java/com/oney/WebRTCModule/EglUtils.java +66 -66
- package/android/src/main/java/com/oney/WebRTCModule/GetUserMediaImpl.java +539 -539
- package/android/src/main/java/com/oney/WebRTCModule/LibraryLoader.java +21 -21
- package/android/src/main/java/com/oney/WebRTCModule/MediaProjectionNotification.java +70 -70
- package/android/src/main/java/com/oney/WebRTCModule/MediaProjectionService.java +82 -82
- package/android/src/main/java/com/oney/WebRTCModule/PeerConnectionObserver.java +588 -588
- package/android/src/main/java/com/oney/WebRTCModule/RTCCryptoManager.java +493 -493
- package/android/src/main/java/com/oney/WebRTCModule/RTCVideoViewManager.java +98 -98
- package/android/src/main/java/com/oney/WebRTCModule/ReactBridgeUtil.java +35 -35
- package/android/src/main/java/com/oney/WebRTCModule/ScreenCaptureController.java +94 -94
- package/android/src/main/java/com/oney/WebRTCModule/SerializeUtils.java +342 -342
- package/android/src/main/java/com/oney/WebRTCModule/StringUtils.java +100 -100
- package/android/src/main/java/com/oney/WebRTCModule/ThreadUtils.java +41 -41
- package/android/src/main/java/com/oney/WebRTCModule/TrackCapturerEventsEmitter.java +34 -34
- package/android/src/main/java/com/oney/WebRTCModule/VideoTrackAdapter.java +137 -137
- package/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java +1643 -1643
- package/android/src/main/java/com/oney/WebRTCModule/WebRTCModuleOptions.java +33 -33
- package/android/src/main/java/com/oney/WebRTCModule/WebRTCModulePackage.java +21 -21
- package/android/src/main/java/com/oney/WebRTCModule/WebRTCView.java +583 -583
- package/android/src/main/java/com/oney/WebRTCModule/palabra/PalabraClient.java +464 -464
- package/android/src/main/java/com/oney/WebRTCModule/palabra/PalabraConfig.java +17 -17
- package/android/src/main/java/com/oney/WebRTCModule/palabra/PalabraListener.java +7 -7
- package/android/src/main/java/com/oney/WebRTCModule/videoEffects/ProcessorProvider.java +38 -38
- package/android/src/main/java/com/oney/WebRTCModule/videoEffects/VideoEffectProcessor.java +59 -59
- package/android/src/main/java/com/oney/WebRTCModule/videoEffects/VideoFrameProcessor.java +19 -19
- package/android/src/main/java/com/oney/WebRTCModule/videoEffects/VideoFrameProcessorFactoryInterface.java +12 -12
- package/android/src/main/java/com/oney/WebRTCModule/webrtcutils/H264AndSoftwareVideoDecoderFactory.java +73 -73
- package/android/src/main/java/com/oney/WebRTCModule/webrtcutils/H264AndSoftwareVideoEncoderFactory.java +73 -73
- package/android/src/main/java/com/oney/WebRTCModule/webrtcutils/SoftwareVideoDecoderFactoryProxy.java +36 -36
- package/android/src/main/java/com/oney/WebRTCModule/webrtcutils/SoftwareVideoEncoderFactoryProxy.java +36 -36
- package/android/src/main/java/org/webrtc/Camera1Helper.java +54 -54
- package/android/src/main/java/org/webrtc/Camera2Helper.java +52 -52
- package/android/src/main/res/values/strings.xml +5 -5
- package/android/src/main/res/values/styles.xml +8 -8
- package/ios/RCTWebRTC/CaptureController.h +18 -18
- package/ios/RCTWebRTC/CaptureController.m +28 -28
- package/ios/RCTWebRTC/CapturerEventsDelegate.h +12 -12
- package/ios/RCTWebRTC/DataChannelWrapper.h +27 -27
- package/ios/RCTWebRTC/DataChannelWrapper.m +42 -42
- package/ios/RCTWebRTC/I420Converter.h +22 -22
- package/ios/RCTWebRTC/I420Converter.m +164 -164
- package/ios/RCTWebRTC/PIPController.h +24 -24
- package/ios/RCTWebRTC/PIPController.m +234 -234
- package/ios/RCTWebRTC/PalabraAudioSink.h +13 -13
- package/ios/RCTWebRTC/PalabraAudioSink.m +18 -18
- package/ios/RCTWebRTC/PalabraClient.h +36 -36
- package/ios/RCTWebRTC/PalabraClient.m +584 -584
- package/ios/RCTWebRTC/RCTConvert+WebRTC.h +16 -16
- package/ios/RCTWebRTC/RCTConvert+WebRTC.m +206 -206
- package/ios/RCTWebRTC/RTCMediaStreamTrack+React.h +10 -10
- package/ios/RCTWebRTC/RTCMediaStreamTrack+React.m +16 -16
- package/ios/RCTWebRTC/RTCVideoViewManager.h +29 -29
- package/ios/RCTWebRTC/RTCVideoViewManager.m +411 -411
- package/ios/RCTWebRTC/SampleBufferVideoCallView.h +12 -12
- package/ios/RCTWebRTC/SampleBufferVideoCallView.m +178 -178
- package/ios/RCTWebRTC/ScreenCaptureController.h +20 -20
- package/ios/RCTWebRTC/ScreenCaptureController.m +82 -82
- package/ios/RCTWebRTC/ScreenCapturePickerViewManager.h +7 -7
- package/ios/RCTWebRTC/ScreenCapturePickerViewManager.m +59 -59
- package/ios/RCTWebRTC/ScreenCapturer.h +19 -19
- package/ios/RCTWebRTC/ScreenCapturer.m +263 -263
- package/ios/RCTWebRTC/SerializeUtils.h +28 -28
- package/ios/RCTWebRTC/SerializeUtils.m +314 -314
- package/ios/RCTWebRTC/SocketConnection.h +13 -13
- package/ios/RCTWebRTC/SocketConnection.m +137 -137
- package/ios/RCTWebRTC/TrackCapturerEventsEmitter.h +14 -14
- package/ios/RCTWebRTC/TrackCapturerEventsEmitter.m +36 -36
- package/ios/RCTWebRTC/VideoCaptureController.h +21 -21
- package/ios/RCTWebRTC/VideoCaptureController.m +328 -328
- package/ios/RCTWebRTC/WebRTCModule+Palabra.h +4 -4
- package/ios/RCTWebRTC/WebRTCModule+Palabra.m +83 -83
- package/ios/RCTWebRTC/WebRTCModule+Permissions.m +75 -75
- package/ios/RCTWebRTC/WebRTCModule+RTCAudioSession.m +20 -20
- package/ios/RCTWebRTC/WebRTCModule+RTCDataChannel.h +14 -14
- package/ios/RCTWebRTC/WebRTCModule+RTCDataChannel.m +165 -165
- package/ios/RCTWebRTC/WebRTCModule+RTCFrameCryptor.m +611 -611
- package/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.h +13 -13
- package/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m +728 -533
- package/ios/RCTWebRTC/WebRTCModule+RTCPeerConnection.h +24 -24
- package/ios/RCTWebRTC/WebRTCModule+RTCPeerConnection.m +1004 -1004
- package/ios/RCTWebRTC/WebRTCModule+Transceivers.m +267 -267
- package/ios/RCTWebRTC/WebRTCModule+VideoTrackAdapter.h +12 -12
- package/ios/RCTWebRTC/WebRTCModule+VideoTrackAdapter.m +166 -166
- package/ios/RCTWebRTC/WebRTCModule.h +58 -51
- package/ios/RCTWebRTC/WebRTCModule.m +169 -151
- package/ios/RCTWebRTC/WebRTCModuleOptions.h +24 -24
- package/ios/RCTWebRTC/WebRTCModuleOptions.m +31 -31
- package/ios/RCTWebRTC/videoEffects/ProcessorProvider.h +9 -9
- package/ios/RCTWebRTC/videoEffects/ProcessorProvider.m +23 -23
- package/ios/RCTWebRTC/videoEffects/VideoEffectProcessor.h +13 -13
- package/ios/RCTWebRTC/videoEffects/VideoEffectProcessor.m +23 -23
- package/ios/RCTWebRTC/videoEffects/VideoFrameProcessor.h +8 -8
- package/ios/RCTWebRTC.xcodeproj/project.pbxproj +404 -404
- package/ios/RCTWebRTC.xcworkspace/contents.xcworkspacedata +10 -10
- package/lib/commonjs/Constraints.js.map +1 -1
- package/lib/commonjs/EventEmitter.js.map +1 -1
- package/lib/commonjs/Logger.js.map +1 -1
- package/lib/commonjs/MediaDevices.js +17 -17
- package/lib/commonjs/MediaDevices.js.map +1 -1
- package/lib/commonjs/MediaStream.js +19 -19
- package/lib/commonjs/MediaStream.js.map +1 -1
- package/lib/commonjs/MediaStreamError.js.map +1 -1
- package/lib/commonjs/MediaStreamErrorEvent.js.map +1 -1
- package/lib/commonjs/MediaStreamTrack.js +28 -28
- package/lib/commonjs/MediaStreamTrack.js.map +1 -1
- package/lib/commonjs/MediaStreamTrackEvent.js +6 -6
- package/lib/commonjs/MediaStreamTrackEvent.js.map +1 -1
- package/lib/commonjs/MessageEvent.js +7 -7
- package/lib/commonjs/MessageEvent.js.map +1 -1
- package/lib/commonjs/Permissions.js +28 -28
- package/lib/commonjs/Permissions.js.map +1 -1
- package/lib/commonjs/RTCAudioSession.js +4 -4
- package/lib/commonjs/RTCAudioSession.js.map +1 -1
- package/lib/commonjs/RTCDataChannel.js +2 -2
- package/lib/commonjs/RTCDataChannel.js.map +1 -1
- package/lib/commonjs/RTCDataChannelEvent.js +6 -6
- package/lib/commonjs/RTCDataChannelEvent.js.map +1 -1
- package/lib/commonjs/RTCDataPacketCryptor.js.map +1 -1
- package/lib/commonjs/RTCDataPacketCryptorFactory.js.map +1 -1
- package/lib/commonjs/RTCErrorEvent.js +3 -3
- package/lib/commonjs/RTCErrorEvent.js.map +1 -1
- package/lib/commonjs/RTCFrameCryptor.js +8 -8
- package/lib/commonjs/RTCFrameCryptor.js.map +1 -1
- package/lib/commonjs/RTCFrameCryptorFactory.js.map +1 -1
- package/lib/commonjs/RTCIceCandidate.js.map +1 -1
- package/lib/commonjs/RTCIceCandidateEvent.js +7 -7
- package/lib/commonjs/RTCIceCandidateEvent.js.map +1 -1
- package/lib/commonjs/RTCKeyProvider.js.map +1 -1
- package/lib/commonjs/RTCPIPView.js +2 -2
- package/lib/commonjs/RTCPIPView.js.map +1 -1
- package/lib/commonjs/RTCPIPView.web.js.map +1 -1
- package/lib/commonjs/RTCPeerConnection.js +36 -36
- package/lib/commonjs/RTCPeerConnection.js.map +1 -1
- package/lib/commonjs/RTCRtcpParameters.js.map +1 -1
- package/lib/commonjs/RTCRtpCapabilities.js +2 -2
- package/lib/commonjs/RTCRtpCapabilities.js.map +1 -1
- package/lib/commonjs/RTCRtpCodecCapability.js.map +1 -1
- package/lib/commonjs/RTCRtpCodecParameters.js.map +1 -1
- package/lib/commonjs/RTCRtpEncodingParameters.js.map +1 -1
- package/lib/commonjs/RTCRtpHeaderExtension.js.map +1 -1
- package/lib/commonjs/RTCRtpParameters.js.map +1 -1
- package/lib/commonjs/RTCRtpReceiveParameters.js.map +1 -1
- package/lib/commonjs/RTCRtpReceiver.js +7 -7
- package/lib/commonjs/RTCRtpReceiver.js.map +1 -1
- package/lib/commonjs/RTCRtpSendParameters.js +3 -3
- package/lib/commonjs/RTCRtpSendParameters.js.map +1 -1
- package/lib/commonjs/RTCRtpSender.js +7 -7
- package/lib/commonjs/RTCRtpSender.js.map +1 -1
- package/lib/commonjs/RTCRtpTransceiver.js.map +1 -1
- package/lib/commonjs/RTCSessionDescription.js.map +1 -1
- package/lib/commonjs/RTCTrackEvent.js +6 -6
- package/lib/commonjs/RTCTrackEvent.js.map +1 -1
- package/lib/commonjs/RTCUtil.js +28 -28
- package/lib/commonjs/RTCUtil.js.map +1 -1
- package/lib/commonjs/RTCView.js +5 -5
- package/lib/commonjs/RTCView.js.map +1 -1
- package/lib/commonjs/RTCView.web.js.map +1 -1
- package/lib/commonjs/ScreenCapturePickerView.js.map +1 -1
- package/lib/commonjs/ScreenCapturePickerView.web.js.map +1 -1
- package/lib/commonjs/getDisplayMedia.js.map +1 -1
- package/lib/commonjs/getUserMedia.js.map +1 -1
- package/lib/commonjs/index.js.map +1 -1
- package/lib/commonjs/index.web.js.map +1 -1
- package/lib/commonjs/webStream.js.map +1 -1
- package/lib/module/Constraints.js.map +1 -1
- package/lib/module/EventEmitter.js.map +1 -1
- package/lib/module/Logger.js.map +1 -1
- package/lib/module/MediaDevices.js +17 -17
- package/lib/module/MediaDevices.js.map +1 -1
- package/lib/module/MediaStream.js +19 -19
- package/lib/module/MediaStream.js.map +1 -1
- package/lib/module/MediaStreamError.js.map +1 -1
- package/lib/module/MediaStreamErrorEvent.js.map +1 -1
- package/lib/module/MediaStreamTrack.js +28 -28
- package/lib/module/MediaStreamTrack.js.map +1 -1
- package/lib/module/MediaStreamTrackEvent.js +6 -6
- package/lib/module/MediaStreamTrackEvent.js.map +1 -1
- package/lib/module/MessageEvent.js +7 -7
- package/lib/module/MessageEvent.js.map +1 -1
- package/lib/module/Permissions.js +28 -28
- package/lib/module/Permissions.js.map +1 -1
- package/lib/module/RTCAudioSession.js +4 -4
- package/lib/module/RTCAudioSession.js.map +1 -1
- package/lib/module/RTCDataChannel.js +2 -2
- package/lib/module/RTCDataChannel.js.map +1 -1
- package/lib/module/RTCDataChannelEvent.js +6 -6
- package/lib/module/RTCDataChannelEvent.js.map +1 -1
- package/lib/module/RTCDataPacketCryptor.js.map +1 -1
- package/lib/module/RTCDataPacketCryptorFactory.js.map +1 -1
- package/lib/module/RTCErrorEvent.js +3 -3
- package/lib/module/RTCErrorEvent.js.map +1 -1
- package/lib/module/RTCFrameCryptor.js +8 -8
- package/lib/module/RTCFrameCryptor.js.map +1 -1
- package/lib/module/RTCFrameCryptorFactory.js.map +1 -1
- package/lib/module/RTCIceCandidate.js.map +1 -1
- package/lib/module/RTCIceCandidateEvent.js +7 -7
- package/lib/module/RTCIceCandidateEvent.js.map +1 -1
- package/lib/module/RTCKeyProvider.js.map +1 -1
- package/lib/module/RTCPIPView.js +2 -2
- package/lib/module/RTCPIPView.js.map +1 -1
- package/lib/module/RTCPIPView.web.js.map +1 -1
- package/lib/module/RTCPeerConnection.js +36 -36
- package/lib/module/RTCPeerConnection.js.map +1 -1
- package/lib/module/RTCRtcpParameters.js.map +1 -1
- package/lib/module/RTCRtpCapabilities.js +2 -2
- package/lib/module/RTCRtpCapabilities.js.map +1 -1
- package/lib/module/RTCRtpCodecCapability.js.map +1 -1
- package/lib/module/RTCRtpCodecParameters.js.map +1 -1
- package/lib/module/RTCRtpEncodingParameters.js.map +1 -1
- package/lib/module/RTCRtpHeaderExtension.js.map +1 -1
- package/lib/module/RTCRtpParameters.js.map +1 -1
- package/lib/module/RTCRtpReceiveParameters.js.map +1 -1
- package/lib/module/RTCRtpReceiver.js +7 -7
- package/lib/module/RTCRtpReceiver.js.map +1 -1
- package/lib/module/RTCRtpSendParameters.js +3 -3
- package/lib/module/RTCRtpSendParameters.js.map +1 -1
- package/lib/module/RTCRtpSender.js +7 -7
- package/lib/module/RTCRtpSender.js.map +1 -1
- package/lib/module/RTCRtpTransceiver.js.map +1 -1
- package/lib/module/RTCSessionDescription.js.map +1 -1
- package/lib/module/RTCTrackEvent.js +6 -6
- package/lib/module/RTCTrackEvent.js.map +1 -1
- package/lib/module/RTCUtil.js +28 -28
- package/lib/module/RTCUtil.js.map +1 -1
- package/lib/module/RTCView.js +5 -5
- package/lib/module/RTCView.js.map +1 -1
- package/lib/module/RTCView.web.js.map +1 -1
- package/lib/module/ScreenCapturePickerView.js.map +1 -1
- package/lib/module/ScreenCapturePickerView.web.js.map +1 -1
- package/lib/module/getDisplayMedia.js.map +1 -1
- package/lib/module/getUserMedia.js.map +1 -1
- package/lib/module/index.js.map +1 -1
- package/lib/module/index.web.js.map +1 -1
- package/lib/module/webStream.js.map +1 -1
- package/lib/typescript/Constraints.d.ts +19 -19
- package/lib/typescript/EventEmitter.d.ts +6 -6
- package/lib/typescript/Logger.d.ts +13 -13
- package/lib/typescript/MediaDevices.d.ts +30 -30
- package/lib/typescript/MediaStream.d.ts +48 -48
- package/lib/typescript/MediaStreamError.d.ts +6 -6
- package/lib/typescript/MediaStreamErrorEvent.d.ts +6 -6
- package/lib/typescript/MediaStreamTrack.d.ts +101 -101
- package/lib/typescript/MediaStreamTrackEvent.d.ts +19 -19
- package/lib/typescript/MessageEvent.d.ts +20 -20
- package/lib/typescript/Permissions.d.ts +55 -55
- package/lib/typescript/RTCAudioSession.d.ts +10 -10
- package/lib/typescript/RTCDataChannel.d.ts +43 -43
- package/lib/typescript/RTCDataChannelEvent.d.ts +19 -19
- package/lib/typescript/RTCDataPacketCryptor.d.ts +12 -12
- package/lib/typescript/RTCDataPacketCryptorFactory.d.ts +6 -6
- package/lib/typescript/RTCErrorEvent.d.ts +12 -12
- package/lib/typescript/RTCFrameCryptor.d.ts +47 -47
- package/lib/typescript/RTCFrameCryptorFactory.d.ts +21 -21
- package/lib/typescript/RTCIceCandidate.d.ts +17 -17
- package/lib/typescript/RTCIceCandidateEvent.d.ts +20 -20
- package/lib/typescript/RTCKeyProvider.d.ts +21 -21
- package/lib/typescript/RTCPIPView.d.ts +15 -15
- package/lib/typescript/RTCPIPView.web.d.ts +13 -13
- package/lib/typescript/RTCPeerConnection.d.ts +117 -117
- package/lib/typescript/RTCRtcpParameters.d.ts +10 -10
- package/lib/typescript/RTCRtpCapabilities.d.ts +9 -9
- package/lib/typescript/RTCRtpCodecCapability.d.ts +7 -7
- package/lib/typescript/RTCRtpCodecParameters.d.ts +16 -16
- package/lib/typescript/RTCRtpEncodingParameters.d.ts +23 -23
- package/lib/typescript/RTCRtpHeaderExtension.d.ts +12 -12
- package/lib/typescript/RTCRtpParameters.d.ts +19 -19
- package/lib/typescript/RTCRtpReceiveParameters.d.ts +4 -4
- package/lib/typescript/RTCRtpReceiver.d.ts +21 -21
- package/lib/typescript/RTCRtpSendParameters.d.ts +20 -20
- package/lib/typescript/RTCRtpSender.d.ts +22 -22
- package/lib/typescript/RTCRtpTransceiver.d.ts +31 -31
- package/lib/typescript/RTCSessionDescription.d.ts +12 -12
- package/lib/typescript/RTCTrackEvent.d.ts +29 -29
- package/lib/typescript/RTCUtil.d.ts +37 -37
- package/lib/typescript/RTCView.d.ts +117 -117
- package/lib/typescript/RTCView.web.d.ts +25 -25
- package/lib/typescript/ScreenCapturePickerView.d.ts +2 -2
- package/lib/typescript/ScreenCapturePickerView.web.d.ts +1 -1
- package/lib/typescript/getDisplayMedia.d.ts +2 -2
- package/lib/typescript/getUserMedia.d.ts +7 -7
- package/lib/typescript/index.d.ts +22 -22
- package/lib/typescript/index.web.d.ts +101 -101
- package/lib/typescript/webStream.d.ts +3 -3
- package/livekit-react-native-webrtc.podspec +29 -29
- package/macos/RCTWebRTC.xcodeproj/project.pbxproj +324 -324
- package/macos/RCTWebRTC.xcodeproj/project.xcworkspace/contents.xcworkspacedata +7 -7
- package/macos/RCTWebRTC.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +8 -8
- package/metro.config.js +7 -7
- package/metro.config.macos.js +14 -14
- package/package.json +66 -66
- package/react-native.config.js +11 -11
- package/src/.eslintrc.cjs +67 -67
- package/src/Constraints.ts +20 -20
- package/src/EventEmitter.ts +65 -65
- package/src/Logger.ts +49 -49
- package/src/MediaDevices.ts +53 -53
- package/src/MediaStream.ts +161 -161
- package/src/MediaStreamError.ts +12 -12
- package/src/MediaStreamErrorEvent.ts +11 -11
- package/src/MediaStreamTrack.ts +282 -282
- package/src/MediaStreamTrackEvent.ts +25 -25
- package/src/MessageEvent.ts +26 -26
- package/src/Permissions.ts +133 -133
- package/src/RTCAudioSession.ts +25 -25
- package/src/RTCDataChannel.ts +190 -190
- package/src/RTCDataChannelEvent.ts +28 -28
- package/src/RTCDataPacketCryptor.ts +90 -90
- package/src/RTCDataPacketCryptorFactory.ts +24 -24
- package/src/RTCErrorEvent.ts +20 -20
- package/src/RTCFrameCryptor.ts +162 -162
- package/src/RTCFrameCryptorFactory.ts +101 -101
- package/src/RTCIceCandidate.ts +29 -29
- package/src/RTCIceCandidateEvent.ts +26 -26
- package/src/RTCKeyProvider.ts +117 -117
- package/src/RTCPIPView.tsx +46 -46
- package/src/RTCPIPView.web.tsx +18 -18
- package/src/RTCPeerConnection.ts +832 -832
- package/src/RTCRtcpParameters.ts +23 -23
- package/src/RTCRtpCapabilities.ts +16 -16
- package/src/RTCRtpCodecCapability.ts +12 -12
- package/src/RTCRtpCodecParameters.ts +44 -44
- package/src/RTCRtpEncodingParameters.ts +90 -90
- package/src/RTCRtpHeaderExtension.ts +27 -27
- package/src/RTCRtpParameters.ts +37 -37
- package/src/RTCRtpReceiveParameters.ts +7 -7
- package/src/RTCRtpReceiver.ts +60 -60
- package/src/RTCRtpSendParameters.ts +63 -63
- package/src/RTCRtpSender.ts +78 -78
- package/src/RTCRtpTransceiver.ts +107 -107
- package/src/RTCSessionDescription.ts +30 -30
- package/src/RTCTrackEvent.ts +42 -42
- package/src/RTCUtil.ts +211 -211
- package/src/RTCView.ts +122 -122
- package/src/RTCView.web.tsx +80 -80
- package/src/ScreenCapturePickerView.ts +4 -4
- package/src/ScreenCapturePickerView.web.tsx +3 -3
- package/src/getDisplayMedia.ts +30 -30
- package/src/getUserMedia.ts +111 -111
- package/src/index.ts +107 -107
- package/src/index.web.ts +191 -191
- package/src/webStream.ts +31 -31
- package/tools/format.sh +6 -6
- package/tools/release.sh +45 -45
- package/tsconfig.json +17 -17
- package/.claude/settings.local.json +0 -9
|
@@ -1,584 +1,584 @@
|
|
|
1
|
-
#import "PalabraClient.h"
|
|
2
|
-
#import "WebRTCModule.h"
|
|
3
|
-
#import <AVFoundation/AVFoundation.h>
|
|
4
|
-
|
|
5
|
-
static const int kSampleRateIn = 16000;
|
|
6
|
-
static const int kSampleRateOut = 24000;
|
|
7
|
-
static const int kChannels = 1;
|
|
8
|
-
static const int kChunkMs = 320;
|
|
9
|
-
static const int kChunkSamples = kSampleRateIn * kChunkMs / 1000;
|
|
10
|
-
static const int kChunkBytes = kChunkSamples * 2;
|
|
11
|
-
|
|
12
|
-
@interface PalabraClient () <NSURLSessionWebSocketDelegate, RTCAudioRenderer>
|
|
13
|
-
|
|
14
|
-
@property (nonatomic, strong) NSString *clientId;
|
|
15
|
-
@property (nonatomic, strong) NSString *clientSecret;
|
|
16
|
-
@property (nonatomic, strong) NSString *apiUrl;
|
|
17
|
-
@property (nonatomic, strong) NSString *sourceLang;
|
|
18
|
-
@property (nonatomic, strong) NSString *targetLang;
|
|
19
|
-
|
|
20
|
-
@property (nonatomic, strong) RTCAudioTrack *remoteTrack;
|
|
21
|
-
|
|
22
|
-
@property (nonatomic, strong) NSString *sessionId;
|
|
23
|
-
@property (nonatomic, strong) NSString *wsUrl;
|
|
24
|
-
@property (nonatomic, strong) NSString *publisherToken;
|
|
25
|
-
|
|
26
|
-
@property (nonatomic, strong) NSURLSession *urlSession;
|
|
27
|
-
@property (nonatomic, strong) NSURLSessionWebSocketTask *webSocket;
|
|
28
|
-
|
|
29
|
-
@property (nonatomic, strong) AVAudioEngine *audioEngine;
|
|
30
|
-
@property (nonatomic, strong) AVAudioPlayerNode *playerNode;
|
|
31
|
-
@property (nonatomic, strong) AVAudioFormat *audioFormat;
|
|
32
|
-
|
|
33
|
-
@property (nonatomic, assign) BOOL connected;
|
|
34
|
-
@property (nonatomic, assign) BOOL translating;
|
|
35
|
-
@property (nonatomic, assign) double originalVolume;
|
|
36
|
-
|
|
37
|
-
@property (nonatomic, strong) NSMutableData *audioBuffer;
|
|
38
|
-
@property (nonatomic, strong) NSLock *bufferLock;
|
|
39
|
-
|
|
40
|
-
@end
|
|
41
|
-
|
|
42
|
-
@implementation PalabraClient
|
|
43
|
-
|
|
44
|
-
- (instancetype)initWithClientId:(NSString *)clientId
|
|
45
|
-
clientSecret:(NSString *)clientSecret
|
|
46
|
-
apiUrl:(NSString *)apiUrl
|
|
47
|
-
module:(WebRTCModule *)module {
|
|
48
|
-
self = [super init];
|
|
49
|
-
if (self) {
|
|
50
|
-
_clientId = clientId;
|
|
51
|
-
_clientSecret = clientSecret;
|
|
52
|
-
_apiUrl = apiUrl;
|
|
53
|
-
_module = module;
|
|
54
|
-
_connected = NO;
|
|
55
|
-
_translating = NO;
|
|
56
|
-
_originalVolume = 1.0;
|
|
57
|
-
_audioBuffer = [NSMutableData new];
|
|
58
|
-
_bufferLock = [NSLock new];
|
|
59
|
-
|
|
60
|
-
NSURLSessionConfiguration *config = [NSURLSessionConfiguration defaultSessionConfiguration];
|
|
61
|
-
_urlSession = [NSURLSession sessionWithConfiguration:config delegate:self delegateQueue:nil];
|
|
62
|
-
|
|
63
|
-
[self setupAudioEngine];
|
|
64
|
-
}
|
|
65
|
-
return self;
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
- (BOOL)isConnected {
|
|
69
|
-
return _connected;
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
- (BOOL)isTranslating {
|
|
73
|
-
return _translating;
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
- (void)setupAudioEngine {
|
|
77
|
-
self.audioEngine = [[AVAudioEngine alloc] init];
|
|
78
|
-
self.playerNode = [[AVAudioPlayerNode alloc] init];
|
|
79
|
-
[self.audioEngine attachNode:self.playerNode];
|
|
80
|
-
|
|
81
|
-
self.audioFormat = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16
|
|
82
|
-
sampleRate:kSampleRateOut
|
|
83
|
-
channels:kChannels
|
|
84
|
-
interleaved:YES];
|
|
85
|
-
|
|
86
|
-
[self.audioEngine connect:self.playerNode
|
|
87
|
-
to:self.audioEngine.mainMixerNode
|
|
88
|
-
format:self.audioFormat];
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
- (void)startWithTrack:(RTCAudioTrack *)remoteAudioTrack
|
|
92
|
-
sourceLang:(NSString *)sourceLang
|
|
93
|
-
targetLang:(NSString *)targetLang {
|
|
94
|
-
|
|
95
|
-
if (self.translating) {
|
|
96
|
-
return;
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
self.remoteTrack = remoteAudioTrack;
|
|
100
|
-
self.sourceLang = sourceLang;
|
|
101
|
-
self.targetLang = targetLang;
|
|
102
|
-
|
|
103
|
-
self.originalVolume = remoteAudioTrack.source.volume;
|
|
104
|
-
remoteAudioTrack.source.volume = 0;
|
|
105
|
-
|
|
106
|
-
[self notifyConnectionState:@"connecting"];
|
|
107
|
-
|
|
108
|
-
__weak typeof(self) weakSelf = self;
|
|
109
|
-
[self createSessionWithCompletion:^(NSDictionary *session, NSError *error) {
|
|
110
|
-
__strong typeof(weakSelf) strongSelf = weakSelf;
|
|
111
|
-
if (!strongSelf) return;
|
|
112
|
-
|
|
113
|
-
if (error) {
|
|
114
|
-
[strongSelf notifyError:error];
|
|
115
|
-
strongSelf.remoteTrack.source.volume = strongSelf.originalVolume;
|
|
116
|
-
return;
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
NSDictionary *data = session[@"data"];
|
|
120
|
-
strongSelf.sessionId = data[@"id"];
|
|
121
|
-
strongSelf.wsUrl = data[@"ws_url"];
|
|
122
|
-
strongSelf.publisherToken = data[@"publisher"];
|
|
123
|
-
|
|
124
|
-
NSLog(@"palabra_ws_url: %@", strongSelf.wsUrl);
|
|
125
|
-
|
|
126
|
-
dispatch_async(dispatch_get_main_queue(), ^{
|
|
127
|
-
[strongSelf connectWebSocket];
|
|
128
|
-
});
|
|
129
|
-
}];
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
- (void)createSessionWithCompletion:(void (^)(NSDictionary *, NSError *))completion {
|
|
133
|
-
NSString *urlStr = [NSString stringWithFormat:@"%@/session-storage/session", self.apiUrl];
|
|
134
|
-
NSURL *url = [NSURL URLWithString:urlStr];
|
|
135
|
-
|
|
136
|
-
NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:url];
|
|
137
|
-
request.HTTPMethod = @"POST";
|
|
138
|
-
[request setValue:self.clientId forHTTPHeaderField:@"ClientId"];
|
|
139
|
-
[request setValue:self.clientSecret forHTTPHeaderField:@"ClientSecret"];
|
|
140
|
-
[request setValue:@"application/json" forHTTPHeaderField:@"Content-Type"];
|
|
141
|
-
|
|
142
|
-
NSDictionary *body = @{
|
|
143
|
-
@"data": @{
|
|
144
|
-
@"subscriber_count": @0,
|
|
145
|
-
@"publisher_can_subscribe": @YES
|
|
146
|
-
}
|
|
147
|
-
};
|
|
148
|
-
|
|
149
|
-
NSError *jsonError;
|
|
150
|
-
request.HTTPBody = [NSJSONSerialization dataWithJSONObject:body options:0 error:&jsonError];
|
|
151
|
-
|
|
152
|
-
if (jsonError) {
|
|
153
|
-
completion(nil, jsonError);
|
|
154
|
-
return;
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
NSURLSessionDataTask *task = [[NSURLSession sharedSession] dataTaskWithRequest:request
|
|
158
|
-
completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) {
|
|
159
|
-
if (error) {
|
|
160
|
-
dispatch_async(dispatch_get_main_queue(), ^{
|
|
161
|
-
completion(nil, error);
|
|
162
|
-
});
|
|
163
|
-
return;
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
NSHTTPURLResponse *httpResponse = (NSHTTPURLResponse *)response;
|
|
167
|
-
if (httpResponse.statusCode < 200 || httpResponse.statusCode >= 300) {
|
|
168
|
-
NSError *httpError = [NSError errorWithDomain:@"PalabraClient"
|
|
169
|
-
code:httpResponse.statusCode
|
|
170
|
-
userInfo:@{NSLocalizedDescriptionKey: [NSString stringWithFormat:@"session_http_error_%ld", (long)httpResponse.statusCode]}];
|
|
171
|
-
dispatch_async(dispatch_get_main_queue(), ^{
|
|
172
|
-
completion(nil, httpError);
|
|
173
|
-
});
|
|
174
|
-
return;
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
NSError *parseError;
|
|
178
|
-
NSDictionary *json = [NSJSONSerialization JSONObjectWithData:data options:0 error:&parseError];
|
|
179
|
-
|
|
180
|
-
dispatch_async(dispatch_get_main_queue(), ^{
|
|
181
|
-
completion(json, parseError);
|
|
182
|
-
});
|
|
183
|
-
}];
|
|
184
|
-
|
|
185
|
-
[task resume];
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
- (void)connectWebSocket {
|
|
189
|
-
NSString *endpoint = [NSString stringWithFormat:@"%@?token=%@", self.wsUrl, self.publisherToken];
|
|
190
|
-
NSLog(@"palabra_connecting_ws: %@", endpoint);
|
|
191
|
-
|
|
192
|
-
NSURL *url = [NSURL URLWithString:endpoint];
|
|
193
|
-
self.webSocket = [self.urlSession webSocketTaskWithURL:url];
|
|
194
|
-
[self.webSocket resume];
|
|
195
|
-
|
|
196
|
-
[self receiveMessage];
|
|
197
|
-
|
|
198
|
-
self.connected = YES;
|
|
199
|
-
self.translating = YES;
|
|
200
|
-
|
|
201
|
-
[self.remoteTrack addRenderer:self];
|
|
202
|
-
|
|
203
|
-
NSError *audioError;
|
|
204
|
-
[self.audioEngine startAndReturnError:&audioError];
|
|
205
|
-
if (audioError) {
|
|
206
|
-
NSLog(@"palabra_audio_engine_error: %@", audioError);
|
|
207
|
-
} else {
|
|
208
|
-
[self.playerNode play];
|
|
209
|
-
NSLog(@"palabra_audio_engine_started");
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
[self notifyConnectionState:@"connected"];
|
|
213
|
-
|
|
214
|
-
dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.5 * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{
|
|
215
|
-
[self sendSetTask];
|
|
216
|
-
});
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
- (void)receiveMessage {
|
|
220
|
-
if (!self.webSocket) return;
|
|
221
|
-
|
|
222
|
-
__weak typeof(self) weakSelf = self;
|
|
223
|
-
[self.webSocket receiveMessageWithCompletionHandler:^(NSURLSessionWebSocketMessage *message, NSError *error) {
|
|
224
|
-
__strong typeof(weakSelf) strongSelf = weakSelf;
|
|
225
|
-
if (!strongSelf) return;
|
|
226
|
-
|
|
227
|
-
if (error) {
|
|
228
|
-
NSLog(@"palabra_ws_error: %@", error);
|
|
229
|
-
dispatch_async(dispatch_get_main_queue(), ^{
|
|
230
|
-
[strongSelf stop];
|
|
231
|
-
[strongSelf notifyError:error];
|
|
232
|
-
});
|
|
233
|
-
return;
|
|
234
|
-
}
|
|
235
|
-
|
|
236
|
-
if (message.type == NSURLSessionWebSocketMessageTypeString) {
|
|
237
|
-
[strongSelf handleMessage:message.string];
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
[strongSelf receiveMessage];
|
|
241
|
-
}];
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
- (void)sendSetTask {
|
|
245
|
-
if (!self.webSocket || !self.connected) return;
|
|
246
|
-
|
|
247
|
-
NSDictionary *msg = @{
|
|
248
|
-
@"message_type": @"set_task",
|
|
249
|
-
@"data": @{
|
|
250
|
-
@"input_stream": @{
|
|
251
|
-
@"content_type": @"audio",
|
|
252
|
-
@"source": @{
|
|
253
|
-
@"type": @"ws",
|
|
254
|
-
@"format": @"pcm_s16le",
|
|
255
|
-
@"sample_rate": @(kSampleRateIn),
|
|
256
|
-
@"channels": @(kChannels)
|
|
257
|
-
}
|
|
258
|
-
},
|
|
259
|
-
@"output_stream": @{
|
|
260
|
-
@"content_type": @"audio",
|
|
261
|
-
@"target": @{
|
|
262
|
-
@"type": @"ws",
|
|
263
|
-
@"format": @"pcm_s16le"
|
|
264
|
-
}
|
|
265
|
-
},
|
|
266
|
-
@"pipeline": @{
|
|
267
|
-
@"transcription": @{
|
|
268
|
-
@"source_language": self.sourceLang
|
|
269
|
-
},
|
|
270
|
-
@"translations": @[@{
|
|
271
|
-
@"target_language": self.targetLang,
|
|
272
|
-
@"speech_generation": @{
|
|
273
|
-
@"voice_cloning": @NO
|
|
274
|
-
}
|
|
275
|
-
}],
|
|
276
|
-
@"allowed_message_types": @[
|
|
277
|
-
@"partial_transcription",
|
|
278
|
-
@"validated_transcription",
|
|
279
|
-
@"translated_transcription"
|
|
280
|
-
]
|
|
281
|
-
}
|
|
282
|
-
}
|
|
283
|
-
};
|
|
284
|
-
|
|
285
|
-
NSError *error;
|
|
286
|
-
NSData *jsonData = [NSJSONSerialization dataWithJSONObject:msg options:0 error:&error];
|
|
287
|
-
if (error) {
|
|
288
|
-
NSLog(@"palabra_set_task_error: %@", error);
|
|
289
|
-
return;
|
|
290
|
-
}
|
|
291
|
-
|
|
292
|
-
NSString *payload = [[NSString alloc] initWithData:jsonData encoding:NSUTF8StringEncoding];
|
|
293
|
-
NSLog(@"palabra_set_task: %@", payload);
|
|
294
|
-
|
|
295
|
-
NSURLSessionWebSocketMessage *wsMsg = [[NSURLSessionWebSocketMessage alloc] initWithString:payload];
|
|
296
|
-
[self.webSocket sendMessage:wsMsg completionHandler:^(NSError *error) {
|
|
297
|
-
if (error) {
|
|
298
|
-
NSLog(@"palabra_send_error: %@", error);
|
|
299
|
-
}
|
|
300
|
-
}];
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
- (void)handleMessage:(NSString *)text {
|
|
304
|
-
NSError *error;
|
|
305
|
-
NSDictionary *json = [NSJSONSerialization JSONObjectWithData:[text dataUsingEncoding:NSUTF8StringEncoding]
|
|
306
|
-
options:0
|
|
307
|
-
error:&error];
|
|
308
|
-
if (error) {
|
|
309
|
-
NSLog(@"palabra_parse_error: %@", error);
|
|
310
|
-
return;
|
|
311
|
-
}
|
|
312
|
-
|
|
313
|
-
NSString *type = json[@"message_type"] ?: @"";
|
|
314
|
-
NSLog(@"palabra_msg: %@", type);
|
|
315
|
-
|
|
316
|
-
if ([type isEqualToString:@"output_audio_data"]) {
|
|
317
|
-
NSDictionary *data = json[@"data"];
|
|
318
|
-
NSString *audioBase64 = data[@"data"] ?: @"";
|
|
319
|
-
if (audioBase64.length > 0) {
|
|
320
|
-
NSData *audioData = [[NSData alloc] initWithBase64EncodedString:audioBase64 options:0];
|
|
321
|
-
NSLog(@"palabra_audio_out: %lu bytes", (unsigned long)audioData.length);
|
|
322
|
-
[self playAudio:audioData];
|
|
323
|
-
}
|
|
324
|
-
} else if ([type containsString:@"transcription"]) {
|
|
325
|
-
NSDictionary *data = json[@"data"];
|
|
326
|
-
NSDictionary *transcription = data[@"transcription"];
|
|
327
|
-
if (transcription) {
|
|
328
|
-
NSString *text = transcription[@"text"] ?: @"";
|
|
329
|
-
NSString *lang = transcription[@"language"] ?: @"";
|
|
330
|
-
BOOL isFinal = ![type isEqualToString:@"partial_transcription"];
|
|
331
|
-
NSLog(@"palabra_transcription: %@ (%@)", text, lang);
|
|
332
|
-
|
|
333
|
-
dispatch_async(dispatch_get_main_queue(), ^{
|
|
334
|
-
[self notifyTranscription:text language:lang isFinal:isFinal];
|
|
335
|
-
});
|
|
336
|
-
}
|
|
337
|
-
} else if ([type isEqualToString:@"error"]) {
|
|
338
|
-
NSDictionary *data = json[@"data"];
|
|
339
|
-
NSString *desc = data[@"desc"] ?: @"unknown";
|
|
340
|
-
NSLog(@"palabra_error: %@", desc);
|
|
341
|
-
NSError *err = [NSError errorWithDomain:@"PalabraClient" code:500 userInfo:@{NSLocalizedDescriptionKey: desc}];
|
|
342
|
-
dispatch_async(dispatch_get_main_queue(), ^{
|
|
343
|
-
[self notifyError:err];
|
|
344
|
-
});
|
|
345
|
-
} else if ([type isEqualToString:@"task_ready"]) {
|
|
346
|
-
NSLog(@"palabra_task_ready");
|
|
347
|
-
}
|
|
348
|
-
}
|
|
349
|
-
|
|
350
|
-
- (void)playAudio:(NSData *)audioData {
|
|
351
|
-
if (!self.translating || !self.playerNode) return;
|
|
352
|
-
|
|
353
|
-
AVAudioFrameCount frameCount = (AVAudioFrameCount)(audioData.length / 2);
|
|
354
|
-
AVAudioPCMBuffer *buffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:self.audioFormat frameCapacity:frameCount];
|
|
355
|
-
buffer.frameLength = frameCount;
|
|
356
|
-
|
|
357
|
-
memcpy(buffer.int16ChannelData[0], audioData.bytes, audioData.length);
|
|
358
|
-
|
|
359
|
-
[self.playerNode scheduleBuffer:buffer completionHandler:nil];
|
|
360
|
-
}
|
|
361
|
-
|
|
362
|
-
- (void)stop {
|
|
363
|
-
if (!self.translating) return;
|
|
364
|
-
|
|
365
|
-
self.translating = NO;
|
|
366
|
-
self.connected = NO;
|
|
367
|
-
|
|
368
|
-
if (self.remoteTrack) {
|
|
369
|
-
[self.remoteTrack removeRenderer:self];
|
|
370
|
-
self.remoteTrack.source.volume = self.originalVolume;
|
|
371
|
-
}
|
|
372
|
-
|
|
373
|
-
if (self.webSocket) {
|
|
374
|
-
NSDictionary *endMsg = @{
|
|
375
|
-
@"message_type": @"end_task",
|
|
376
|
-
@"data": @{@"force": @NO}
|
|
377
|
-
};
|
|
378
|
-
NSData *jsonData = [NSJSONSerialization dataWithJSONObject:endMsg options:0 error:nil];
|
|
379
|
-
NSString *payload = [[NSString alloc] initWithData:jsonData encoding:NSUTF8StringEncoding];
|
|
380
|
-
NSURLSessionWebSocketMessage *msg = [[NSURLSessionWebSocketMessage alloc] initWithString:payload];
|
|
381
|
-
[self.webSocket sendMessage:msg completionHandler:nil];
|
|
382
|
-
|
|
383
|
-
[self.webSocket cancelWithCloseCode:NSURLSessionWebSocketCloseCodeNormalClosure reason:nil];
|
|
384
|
-
self.webSocket = nil;
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
[self.playerNode stop];
|
|
388
|
-
[self.audioEngine stop];
|
|
389
|
-
|
|
390
|
-
[self.bufferLock lock];
|
|
391
|
-
[self.audioBuffer setLength:0];
|
|
392
|
-
[self.bufferLock unlock];
|
|
393
|
-
|
|
394
|
-
self.remoteTrack = nil;
|
|
395
|
-
[self notifyConnectionState:@"disconnected"];
|
|
396
|
-
}
|
|
397
|
-
|
|
398
|
-
- (void)setTranslatedVolume:(double)volume {
|
|
399
|
-
if (self.playerNode) {
|
|
400
|
-
self.playerNode.volume = volume;
|
|
401
|
-
}
|
|
402
|
-
}
|
|
403
|
-
|
|
404
|
-
- (void)sendAudioToPalabra:(AVAudioPCMBuffer *)buffer {
|
|
405
|
-
}
|
|
406
|
-
|
|
407
|
-
#pragma mark - RTCAudioRenderer
|
|
408
|
-
|
|
409
|
-
- (void)renderPCMBuffer:(AVAudioPCMBuffer *)buffer {
|
|
410
|
-
if (!self.translating || !self.webSocket) {
|
|
411
|
-
return;
|
|
412
|
-
}
|
|
413
|
-
|
|
414
|
-
static int logCounter = 0;
|
|
415
|
-
if (logCounter++ % 100 == 0) {
|
|
416
|
-
NSLog(@"palabra_render_pcm: frames=%u rate=%.0f channels=%u",
|
|
417
|
-
(unsigned int)buffer.frameLength,
|
|
418
|
-
buffer.format.sampleRate,
|
|
419
|
-
(unsigned int)buffer.format.channelCount);
|
|
420
|
-
}
|
|
421
|
-
|
|
422
|
-
NSData *resampled = [self resampleBuffer:buffer toRate:kSampleRateIn channels:kChannels];
|
|
423
|
-
if (resampled.length == 0) {
|
|
424
|
-
NSLog(@"palabra_resample_empty");
|
|
425
|
-
return;
|
|
426
|
-
}
|
|
427
|
-
|
|
428
|
-
[self.bufferLock lock];
|
|
429
|
-
[self.audioBuffer appendData:resampled];
|
|
430
|
-
|
|
431
|
-
while (self.audioBuffer.length >= kChunkBytes) {
|
|
432
|
-
NSData *chunk = [self.audioBuffer subdataWithRange:NSMakeRange(0, kChunkBytes)];
|
|
433
|
-
NSMutableData *remaining = [NSMutableData dataWithData:[self.audioBuffer subdataWithRange:NSMakeRange(kChunkBytes, self.audioBuffer.length - kChunkBytes)]];
|
|
434
|
-
self.audioBuffer = remaining;
|
|
435
|
-
|
|
436
|
-
[self sendAudioChunk:chunk];
|
|
437
|
-
}
|
|
438
|
-
[self.bufferLock unlock];
|
|
439
|
-
}
|
|
440
|
-
|
|
441
|
-
- (NSData *)resampleBuffer:(AVAudioPCMBuffer *)buffer toRate:(int)dstRate channels:(int)dstChannels {
|
|
442
|
-
int srcRate = (int)buffer.format.sampleRate;
|
|
443
|
-
int srcChannels = (int)buffer.format.channelCount;
|
|
444
|
-
int srcSamples = (int)buffer.frameLength;
|
|
445
|
-
|
|
446
|
-
if (srcSamples == 0) {
|
|
447
|
-
return [NSData data];
|
|
448
|
-
}
|
|
449
|
-
|
|
450
|
-
AVAudioCommonFormat format = buffer.format.commonFormat;
|
|
451
|
-
|
|
452
|
-
if (srcRate == dstRate && srcChannels == dstChannels && format == AVAudioPCMFormatInt16) {
|
|
453
|
-
return [NSData dataWithBytes:buffer.int16ChannelData[0] length:srcSamples * 2];
|
|
454
|
-
}
|
|
455
|
-
|
|
456
|
-
float *srcData = NULL;
|
|
457
|
-
float *tempFloat = NULL;
|
|
458
|
-
|
|
459
|
-
if (format == AVAudioPCMFormatFloat32) {
|
|
460
|
-
srcData = buffer.floatChannelData[0];
|
|
461
|
-
} else if (format == AVAudioPCMFormatInt16) {
|
|
462
|
-
tempFloat = malloc(srcSamples * sizeof(float));
|
|
463
|
-
int16_t *srcInt16 = buffer.int16ChannelData[0];
|
|
464
|
-
for (int i = 0; i < srcSamples; i++) {
|
|
465
|
-
tempFloat[i] = (float)srcInt16[i] / 32768.0f;
|
|
466
|
-
}
|
|
467
|
-
srcData = tempFloat;
|
|
468
|
-
} else {
|
|
469
|
-
NSLog(@"palabra_unsupported_format: %ld", (long)format);
|
|
470
|
-
return [NSData data];
|
|
471
|
-
}
|
|
472
|
-
|
|
473
|
-
float *monoSrc = srcData;
|
|
474
|
-
float *monoBuffer = NULL;
|
|
475
|
-
if (srcChannels == 2) {
|
|
476
|
-
monoBuffer = malloc(srcSamples * sizeof(float));
|
|
477
|
-
if (format == AVAudioPCMFormatFloat32 && buffer.floatChannelData[1]) {
|
|
478
|
-
for (int i = 0; i < srcSamples; i++) {
|
|
479
|
-
monoBuffer[i] = (buffer.floatChannelData[0][i] + buffer.floatChannelData[1][i]) / 2.0f;
|
|
480
|
-
}
|
|
481
|
-
} else if (format == AVAudioPCMFormatInt16 && buffer.int16ChannelData[1]) {
|
|
482
|
-
for (int i = 0; i < srcSamples; i++) {
|
|
483
|
-
float ch0 = (float)buffer.int16ChannelData[0][i] / 32768.0f;
|
|
484
|
-
float ch1 = (float)buffer.int16ChannelData[1][i] / 32768.0f;
|
|
485
|
-
monoBuffer[i] = (ch0 + ch1) / 2.0f;
|
|
486
|
-
}
|
|
487
|
-
} else {
|
|
488
|
-
for (int i = 0; i < srcSamples; i++) {
|
|
489
|
-
monoBuffer[i] = srcData[i];
|
|
490
|
-
}
|
|
491
|
-
}
|
|
492
|
-
monoSrc = monoBuffer;
|
|
493
|
-
}
|
|
494
|
-
|
|
495
|
-
int dstSamples = (int)((long)srcSamples * dstRate / srcRate);
|
|
496
|
-
if (dstSamples == 0) dstSamples = 1;
|
|
497
|
-
|
|
498
|
-
int16_t *dstData = malloc(dstSamples * sizeof(int16_t));
|
|
499
|
-
|
|
500
|
-
for (int i = 0; i < dstSamples; i++) {
|
|
501
|
-
float srcIdx = (float)i * (srcSamples - 1) / (dstSamples - 1);
|
|
502
|
-
int idx0 = (int)srcIdx;
|
|
503
|
-
int idx1 = MIN(idx0 + 1, srcSamples - 1);
|
|
504
|
-
float frac = srcIdx - idx0;
|
|
505
|
-
float sample = monoSrc[idx0] * (1 - frac) + monoSrc[idx1] * frac;
|
|
506
|
-
sample = fmaxf(-1.0f, fminf(1.0f, sample));
|
|
507
|
-
dstData[i] = (int16_t)(sample * 32767.0f);
|
|
508
|
-
}
|
|
509
|
-
|
|
510
|
-
if (monoBuffer) free(monoBuffer);
|
|
511
|
-
if (tempFloat) free(tempFloat);
|
|
512
|
-
|
|
513
|
-
NSData *result = [NSData dataWithBytes:dstData length:dstSamples * 2];
|
|
514
|
-
free(dstData);
|
|
515
|
-
return result;
|
|
516
|
-
}
|
|
517
|
-
|
|
518
|
-
- (void)sendAudioChunk:(NSData *)chunk {
|
|
519
|
-
if (!self.webSocket || !self.connected) return;
|
|
520
|
-
|
|
521
|
-
static int sendCounter = 0;
|
|
522
|
-
if (sendCounter++ % 50 == 0) {
|
|
523
|
-
NSLog(@"palabra_send_chunk: %lu bytes (count=%d)", (unsigned long)chunk.length, sendCounter);
|
|
524
|
-
}
|
|
525
|
-
|
|
526
|
-
NSString *base64 = [chunk base64EncodedStringWithOptions:0];
|
|
527
|
-
NSDictionary *msg = @{
|
|
528
|
-
@"message_type": @"input_audio_data",
|
|
529
|
-
@"data": @{
|
|
530
|
-
@"data": base64
|
|
531
|
-
}
|
|
532
|
-
};
|
|
533
|
-
|
|
534
|
-
NSData *jsonData = [NSJSONSerialization dataWithJSONObject:msg options:0 error:nil];
|
|
535
|
-
NSString *payload = [[NSString alloc] initWithData:jsonData encoding:NSUTF8StringEncoding];
|
|
536
|
-
|
|
537
|
-
NSURLSessionWebSocketMessage *wsMsg = [[NSURLSessionWebSocketMessage alloc] initWithString:payload];
|
|
538
|
-
[self.webSocket sendMessage:wsMsg completionHandler:^(NSError *error) {
|
|
539
|
-
if (error) {
|
|
540
|
-
NSLog(@"palabra_chunk_send_error: %@", error);
|
|
541
|
-
}
|
|
542
|
-
}];
|
|
543
|
-
}
|
|
544
|
-
|
|
545
|
-
#pragma mark - Notifications
|
|
546
|
-
|
|
547
|
-
- (void)notifyConnectionState:(NSString *)state {
|
|
548
|
-
if ([self.delegate respondsToSelector:@selector(palabraDidChangeConnectionState:)]) {
|
|
549
|
-
[self.delegate palabraDidChangeConnectionState:state];
|
|
550
|
-
}
|
|
551
|
-
}
|
|
552
|
-
|
|
553
|
-
- (void)notifyError:(NSError *)error {
|
|
554
|
-
if ([self.delegate respondsToSelector:@selector(palabraDidFailWithError:)]) {
|
|
555
|
-
[self.delegate palabraDidFailWithError:error];
|
|
556
|
-
}
|
|
557
|
-
}
|
|
558
|
-
|
|
559
|
-
- (void)notifyTranscription:(NSString *)text language:(NSString *)lang isFinal:(BOOL)isFinal {
|
|
560
|
-
if ([self.delegate respondsToSelector:@selector(palabraDidReceiveTranscription:)]) {
|
|
561
|
-
[self.delegate palabraDidReceiveTranscription:@{
|
|
562
|
-
@"text": text,
|
|
563
|
-
@"language": lang,
|
|
564
|
-
@"isFinal": @(isFinal)
|
|
565
|
-
}];
|
|
566
|
-
}
|
|
567
|
-
}
|
|
568
|
-
|
|
569
|
-
#pragma mark - NSURLSessionWebSocketDelegate
|
|
570
|
-
|
|
571
|
-
- (void)URLSession:(NSURLSession *)session webSocketTask:(NSURLSessionWebSocketTask *)webSocketTask didOpenWithProtocol:(NSString *)protocol {
|
|
572
|
-
NSLog(@"palabra_ws_opened");
|
|
573
|
-
}
|
|
574
|
-
|
|
575
|
-
- (void)URLSession:(NSURLSession *)session webSocketTask:(NSURLSessionWebSocketTask *)webSocketTask didCloseWithCode:(NSURLSessionWebSocketCloseCode)closeCode reason:(NSData *)reason {
|
|
576
|
-
NSLog(@"palabra_ws_closed: %ld", (long)closeCode);
|
|
577
|
-
dispatch_async(dispatch_get_main_queue(), ^{
|
|
578
|
-
if (self.translating) {
|
|
579
|
-
[self stop];
|
|
580
|
-
}
|
|
581
|
-
});
|
|
582
|
-
}
|
|
583
|
-
|
|
584
|
-
@end
|
|
1
|
+
#import "PalabraClient.h"
|
|
2
|
+
#import "WebRTCModule.h"
|
|
3
|
+
#import <AVFoundation/AVFoundation.h>
|
|
4
|
+
|
|
5
|
+
static const int kSampleRateIn = 16000;
|
|
6
|
+
static const int kSampleRateOut = 24000;
|
|
7
|
+
static const int kChannels = 1;
|
|
8
|
+
static const int kChunkMs = 320;
|
|
9
|
+
static const int kChunkSamples = kSampleRateIn * kChunkMs / 1000;
|
|
10
|
+
static const int kChunkBytes = kChunkSamples * 2;
|
|
11
|
+
|
|
12
|
+
@interface PalabraClient () <NSURLSessionWebSocketDelegate, RTCAudioRenderer>
|
|
13
|
+
|
|
14
|
+
@property (nonatomic, strong) NSString *clientId;
|
|
15
|
+
@property (nonatomic, strong) NSString *clientSecret;
|
|
16
|
+
@property (nonatomic, strong) NSString *apiUrl;
|
|
17
|
+
@property (nonatomic, strong) NSString *sourceLang;
|
|
18
|
+
@property (nonatomic, strong) NSString *targetLang;
|
|
19
|
+
|
|
20
|
+
@property (nonatomic, strong) RTCAudioTrack *remoteTrack;
|
|
21
|
+
|
|
22
|
+
@property (nonatomic, strong) NSString *sessionId;
|
|
23
|
+
@property (nonatomic, strong) NSString *wsUrl;
|
|
24
|
+
@property (nonatomic, strong) NSString *publisherToken;
|
|
25
|
+
|
|
26
|
+
@property (nonatomic, strong) NSURLSession *urlSession;
|
|
27
|
+
@property (nonatomic, strong) NSURLSessionWebSocketTask *webSocket;
|
|
28
|
+
|
|
29
|
+
@property (nonatomic, strong) AVAudioEngine *audioEngine;
|
|
30
|
+
@property (nonatomic, strong) AVAudioPlayerNode *playerNode;
|
|
31
|
+
@property (nonatomic, strong) AVAudioFormat *audioFormat;
|
|
32
|
+
|
|
33
|
+
@property (nonatomic, assign) BOOL connected;
|
|
34
|
+
@property (nonatomic, assign) BOOL translating;
|
|
35
|
+
@property (nonatomic, assign) double originalVolume;
|
|
36
|
+
|
|
37
|
+
@property (nonatomic, strong) NSMutableData *audioBuffer;
|
|
38
|
+
@property (nonatomic, strong) NSLock *bufferLock;
|
|
39
|
+
|
|
40
|
+
@end
|
|
41
|
+
|
|
42
|
+
@implementation PalabraClient
|
|
43
|
+
|
|
44
|
+
- (instancetype)initWithClientId:(NSString *)clientId
|
|
45
|
+
clientSecret:(NSString *)clientSecret
|
|
46
|
+
apiUrl:(NSString *)apiUrl
|
|
47
|
+
module:(WebRTCModule *)module {
|
|
48
|
+
self = [super init];
|
|
49
|
+
if (self) {
|
|
50
|
+
_clientId = clientId;
|
|
51
|
+
_clientSecret = clientSecret;
|
|
52
|
+
_apiUrl = apiUrl;
|
|
53
|
+
_module = module;
|
|
54
|
+
_connected = NO;
|
|
55
|
+
_translating = NO;
|
|
56
|
+
_originalVolume = 1.0;
|
|
57
|
+
_audioBuffer = [NSMutableData new];
|
|
58
|
+
_bufferLock = [NSLock new];
|
|
59
|
+
|
|
60
|
+
NSURLSessionConfiguration *config = [NSURLSessionConfiguration defaultSessionConfiguration];
|
|
61
|
+
_urlSession = [NSURLSession sessionWithConfiguration:config delegate:self delegateQueue:nil];
|
|
62
|
+
|
|
63
|
+
[self setupAudioEngine];
|
|
64
|
+
}
|
|
65
|
+
return self;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
- (BOOL)isConnected {
|
|
69
|
+
return _connected;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
- (BOOL)isTranslating {
|
|
73
|
+
return _translating;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
- (void)setupAudioEngine {
|
|
77
|
+
self.audioEngine = [[AVAudioEngine alloc] init];
|
|
78
|
+
self.playerNode = [[AVAudioPlayerNode alloc] init];
|
|
79
|
+
[self.audioEngine attachNode:self.playerNode];
|
|
80
|
+
|
|
81
|
+
self.audioFormat = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16
|
|
82
|
+
sampleRate:kSampleRateOut
|
|
83
|
+
channels:kChannels
|
|
84
|
+
interleaved:YES];
|
|
85
|
+
|
|
86
|
+
[self.audioEngine connect:self.playerNode
|
|
87
|
+
to:self.audioEngine.mainMixerNode
|
|
88
|
+
format:self.audioFormat];
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
- (void)startWithTrack:(RTCAudioTrack *)remoteAudioTrack
|
|
92
|
+
sourceLang:(NSString *)sourceLang
|
|
93
|
+
targetLang:(NSString *)targetLang {
|
|
94
|
+
|
|
95
|
+
if (self.translating) {
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
self.remoteTrack = remoteAudioTrack;
|
|
100
|
+
self.sourceLang = sourceLang;
|
|
101
|
+
self.targetLang = targetLang;
|
|
102
|
+
|
|
103
|
+
self.originalVolume = remoteAudioTrack.source.volume;
|
|
104
|
+
remoteAudioTrack.source.volume = 0;
|
|
105
|
+
|
|
106
|
+
[self notifyConnectionState:@"connecting"];
|
|
107
|
+
|
|
108
|
+
__weak typeof(self) weakSelf = self;
|
|
109
|
+
[self createSessionWithCompletion:^(NSDictionary *session, NSError *error) {
|
|
110
|
+
__strong typeof(weakSelf) strongSelf = weakSelf;
|
|
111
|
+
if (!strongSelf) return;
|
|
112
|
+
|
|
113
|
+
if (error) {
|
|
114
|
+
[strongSelf notifyError:error];
|
|
115
|
+
strongSelf.remoteTrack.source.volume = strongSelf.originalVolume;
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
NSDictionary *data = session[@"data"];
|
|
120
|
+
strongSelf.sessionId = data[@"id"];
|
|
121
|
+
strongSelf.wsUrl = data[@"ws_url"];
|
|
122
|
+
strongSelf.publisherToken = data[@"publisher"];
|
|
123
|
+
|
|
124
|
+
NSLog(@"palabra_ws_url: %@", strongSelf.wsUrl);
|
|
125
|
+
|
|
126
|
+
dispatch_async(dispatch_get_main_queue(), ^{
|
|
127
|
+
[strongSelf connectWebSocket];
|
|
128
|
+
});
|
|
129
|
+
}];
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
- (void)createSessionWithCompletion:(void (^)(NSDictionary *, NSError *))completion {
|
|
133
|
+
NSString *urlStr = [NSString stringWithFormat:@"%@/session-storage/session", self.apiUrl];
|
|
134
|
+
NSURL *url = [NSURL URLWithString:urlStr];
|
|
135
|
+
|
|
136
|
+
NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:url];
|
|
137
|
+
request.HTTPMethod = @"POST";
|
|
138
|
+
[request setValue:self.clientId forHTTPHeaderField:@"ClientId"];
|
|
139
|
+
[request setValue:self.clientSecret forHTTPHeaderField:@"ClientSecret"];
|
|
140
|
+
[request setValue:@"application/json" forHTTPHeaderField:@"Content-Type"];
|
|
141
|
+
|
|
142
|
+
NSDictionary *body = @{
|
|
143
|
+
@"data": @{
|
|
144
|
+
@"subscriber_count": @0,
|
|
145
|
+
@"publisher_can_subscribe": @YES
|
|
146
|
+
}
|
|
147
|
+
};
|
|
148
|
+
|
|
149
|
+
NSError *jsonError;
|
|
150
|
+
request.HTTPBody = [NSJSONSerialization dataWithJSONObject:body options:0 error:&jsonError];
|
|
151
|
+
|
|
152
|
+
if (jsonError) {
|
|
153
|
+
completion(nil, jsonError);
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
NSURLSessionDataTask *task = [[NSURLSession sharedSession] dataTaskWithRequest:request
|
|
158
|
+
completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) {
|
|
159
|
+
if (error) {
|
|
160
|
+
dispatch_async(dispatch_get_main_queue(), ^{
|
|
161
|
+
completion(nil, error);
|
|
162
|
+
});
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
NSHTTPURLResponse *httpResponse = (NSHTTPURLResponse *)response;
|
|
167
|
+
if (httpResponse.statusCode < 200 || httpResponse.statusCode >= 300) {
|
|
168
|
+
NSError *httpError = [NSError errorWithDomain:@"PalabraClient"
|
|
169
|
+
code:httpResponse.statusCode
|
|
170
|
+
userInfo:@{NSLocalizedDescriptionKey: [NSString stringWithFormat:@"session_http_error_%ld", (long)httpResponse.statusCode]}];
|
|
171
|
+
dispatch_async(dispatch_get_main_queue(), ^{
|
|
172
|
+
completion(nil, httpError);
|
|
173
|
+
});
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
NSError *parseError;
|
|
178
|
+
NSDictionary *json = [NSJSONSerialization JSONObjectWithData:data options:0 error:&parseError];
|
|
179
|
+
|
|
180
|
+
dispatch_async(dispatch_get_main_queue(), ^{
|
|
181
|
+
completion(json, parseError);
|
|
182
|
+
});
|
|
183
|
+
}];
|
|
184
|
+
|
|
185
|
+
[task resume];
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
- (void)connectWebSocket {
|
|
189
|
+
NSString *endpoint = [NSString stringWithFormat:@"%@?token=%@", self.wsUrl, self.publisherToken];
|
|
190
|
+
NSLog(@"palabra_connecting_ws: %@", endpoint);
|
|
191
|
+
|
|
192
|
+
NSURL *url = [NSURL URLWithString:endpoint];
|
|
193
|
+
self.webSocket = [self.urlSession webSocketTaskWithURL:url];
|
|
194
|
+
[self.webSocket resume];
|
|
195
|
+
|
|
196
|
+
[self receiveMessage];
|
|
197
|
+
|
|
198
|
+
self.connected = YES;
|
|
199
|
+
self.translating = YES;
|
|
200
|
+
|
|
201
|
+
[self.remoteTrack addRenderer:self];
|
|
202
|
+
|
|
203
|
+
NSError *audioError;
|
|
204
|
+
[self.audioEngine startAndReturnError:&audioError];
|
|
205
|
+
if (audioError) {
|
|
206
|
+
NSLog(@"palabra_audio_engine_error: %@", audioError);
|
|
207
|
+
} else {
|
|
208
|
+
[self.playerNode play];
|
|
209
|
+
NSLog(@"palabra_audio_engine_started");
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
[self notifyConnectionState:@"connected"];
|
|
213
|
+
|
|
214
|
+
dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.5 * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{
|
|
215
|
+
[self sendSetTask];
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
- (void)receiveMessage {
|
|
220
|
+
if (!self.webSocket) return;
|
|
221
|
+
|
|
222
|
+
__weak typeof(self) weakSelf = self;
|
|
223
|
+
[self.webSocket receiveMessageWithCompletionHandler:^(NSURLSessionWebSocketMessage *message, NSError *error) {
|
|
224
|
+
__strong typeof(weakSelf) strongSelf = weakSelf;
|
|
225
|
+
if (!strongSelf) return;
|
|
226
|
+
|
|
227
|
+
if (error) {
|
|
228
|
+
NSLog(@"palabra_ws_error: %@", error);
|
|
229
|
+
dispatch_async(dispatch_get_main_queue(), ^{
|
|
230
|
+
[strongSelf stop];
|
|
231
|
+
[strongSelf notifyError:error];
|
|
232
|
+
});
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
if (message.type == NSURLSessionWebSocketMessageTypeString) {
|
|
237
|
+
[strongSelf handleMessage:message.string];
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
[strongSelf receiveMessage];
|
|
241
|
+
}];
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
- (void)sendSetTask {
|
|
245
|
+
if (!self.webSocket || !self.connected) return;
|
|
246
|
+
|
|
247
|
+
NSDictionary *msg = @{
|
|
248
|
+
@"message_type": @"set_task",
|
|
249
|
+
@"data": @{
|
|
250
|
+
@"input_stream": @{
|
|
251
|
+
@"content_type": @"audio",
|
|
252
|
+
@"source": @{
|
|
253
|
+
@"type": @"ws",
|
|
254
|
+
@"format": @"pcm_s16le",
|
|
255
|
+
@"sample_rate": @(kSampleRateIn),
|
|
256
|
+
@"channels": @(kChannels)
|
|
257
|
+
}
|
|
258
|
+
},
|
|
259
|
+
@"output_stream": @{
|
|
260
|
+
@"content_type": @"audio",
|
|
261
|
+
@"target": @{
|
|
262
|
+
@"type": @"ws",
|
|
263
|
+
@"format": @"pcm_s16le"
|
|
264
|
+
}
|
|
265
|
+
},
|
|
266
|
+
@"pipeline": @{
|
|
267
|
+
@"transcription": @{
|
|
268
|
+
@"source_language": self.sourceLang
|
|
269
|
+
},
|
|
270
|
+
@"translations": @[@{
|
|
271
|
+
@"target_language": self.targetLang,
|
|
272
|
+
@"speech_generation": @{
|
|
273
|
+
@"voice_cloning": @NO
|
|
274
|
+
}
|
|
275
|
+
}],
|
|
276
|
+
@"allowed_message_types": @[
|
|
277
|
+
@"partial_transcription",
|
|
278
|
+
@"validated_transcription",
|
|
279
|
+
@"translated_transcription"
|
|
280
|
+
]
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
};
|
|
284
|
+
|
|
285
|
+
NSError *error;
|
|
286
|
+
NSData *jsonData = [NSJSONSerialization dataWithJSONObject:msg options:0 error:&error];
|
|
287
|
+
if (error) {
|
|
288
|
+
NSLog(@"palabra_set_task_error: %@", error);
|
|
289
|
+
return;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
NSString *payload = [[NSString alloc] initWithData:jsonData encoding:NSUTF8StringEncoding];
|
|
293
|
+
NSLog(@"palabra_set_task: %@", payload);
|
|
294
|
+
|
|
295
|
+
NSURLSessionWebSocketMessage *wsMsg = [[NSURLSessionWebSocketMessage alloc] initWithString:payload];
|
|
296
|
+
[self.webSocket sendMessage:wsMsg completionHandler:^(NSError *error) {
|
|
297
|
+
if (error) {
|
|
298
|
+
NSLog(@"palabra_send_error: %@", error);
|
|
299
|
+
}
|
|
300
|
+
}];
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
- (void)handleMessage:(NSString *)text {
|
|
304
|
+
NSError *error;
|
|
305
|
+
NSDictionary *json = [NSJSONSerialization JSONObjectWithData:[text dataUsingEncoding:NSUTF8StringEncoding]
|
|
306
|
+
options:0
|
|
307
|
+
error:&error];
|
|
308
|
+
if (error) {
|
|
309
|
+
NSLog(@"palabra_parse_error: %@", error);
|
|
310
|
+
return;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
NSString *type = json[@"message_type"] ?: @"";
|
|
314
|
+
NSLog(@"palabra_msg: %@", type);
|
|
315
|
+
|
|
316
|
+
if ([type isEqualToString:@"output_audio_data"]) {
|
|
317
|
+
NSDictionary *data = json[@"data"];
|
|
318
|
+
NSString *audioBase64 = data[@"data"] ?: @"";
|
|
319
|
+
if (audioBase64.length > 0) {
|
|
320
|
+
NSData *audioData = [[NSData alloc] initWithBase64EncodedString:audioBase64 options:0];
|
|
321
|
+
NSLog(@"palabra_audio_out: %lu bytes", (unsigned long)audioData.length);
|
|
322
|
+
[self playAudio:audioData];
|
|
323
|
+
}
|
|
324
|
+
} else if ([type containsString:@"transcription"]) {
|
|
325
|
+
NSDictionary *data = json[@"data"];
|
|
326
|
+
NSDictionary *transcription = data[@"transcription"];
|
|
327
|
+
if (transcription) {
|
|
328
|
+
NSString *text = transcription[@"text"] ?: @"";
|
|
329
|
+
NSString *lang = transcription[@"language"] ?: @"";
|
|
330
|
+
BOOL isFinal = ![type isEqualToString:@"partial_transcription"];
|
|
331
|
+
NSLog(@"palabra_transcription: %@ (%@)", text, lang);
|
|
332
|
+
|
|
333
|
+
dispatch_async(dispatch_get_main_queue(), ^{
|
|
334
|
+
[self notifyTranscription:text language:lang isFinal:isFinal];
|
|
335
|
+
});
|
|
336
|
+
}
|
|
337
|
+
} else if ([type isEqualToString:@"error"]) {
|
|
338
|
+
NSDictionary *data = json[@"data"];
|
|
339
|
+
NSString *desc = data[@"desc"] ?: @"unknown";
|
|
340
|
+
NSLog(@"palabra_error: %@", desc);
|
|
341
|
+
NSError *err = [NSError errorWithDomain:@"PalabraClient" code:500 userInfo:@{NSLocalizedDescriptionKey: desc}];
|
|
342
|
+
dispatch_async(dispatch_get_main_queue(), ^{
|
|
343
|
+
[self notifyError:err];
|
|
344
|
+
});
|
|
345
|
+
} else if ([type isEqualToString:@"task_ready"]) {
|
|
346
|
+
NSLog(@"palabra_task_ready");
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
- (void)playAudio:(NSData *)audioData {
|
|
351
|
+
if (!self.translating || !self.playerNode) return;
|
|
352
|
+
|
|
353
|
+
AVAudioFrameCount frameCount = (AVAudioFrameCount)(audioData.length / 2);
|
|
354
|
+
AVAudioPCMBuffer *buffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:self.audioFormat frameCapacity:frameCount];
|
|
355
|
+
buffer.frameLength = frameCount;
|
|
356
|
+
|
|
357
|
+
memcpy(buffer.int16ChannelData[0], audioData.bytes, audioData.length);
|
|
358
|
+
|
|
359
|
+
[self.playerNode scheduleBuffer:buffer completionHandler:nil];
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
- (void)stop {
|
|
363
|
+
if (!self.translating) return;
|
|
364
|
+
|
|
365
|
+
self.translating = NO;
|
|
366
|
+
self.connected = NO;
|
|
367
|
+
|
|
368
|
+
if (self.remoteTrack) {
|
|
369
|
+
[self.remoteTrack removeRenderer:self];
|
|
370
|
+
self.remoteTrack.source.volume = self.originalVolume;
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
if (self.webSocket) {
|
|
374
|
+
NSDictionary *endMsg = @{
|
|
375
|
+
@"message_type": @"end_task",
|
|
376
|
+
@"data": @{@"force": @NO}
|
|
377
|
+
};
|
|
378
|
+
NSData *jsonData = [NSJSONSerialization dataWithJSONObject:endMsg options:0 error:nil];
|
|
379
|
+
NSString *payload = [[NSString alloc] initWithData:jsonData encoding:NSUTF8StringEncoding];
|
|
380
|
+
NSURLSessionWebSocketMessage *msg = [[NSURLSessionWebSocketMessage alloc] initWithString:payload];
|
|
381
|
+
[self.webSocket sendMessage:msg completionHandler:nil];
|
|
382
|
+
|
|
383
|
+
[self.webSocket cancelWithCloseCode:NSURLSessionWebSocketCloseCodeNormalClosure reason:nil];
|
|
384
|
+
self.webSocket = nil;
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
[self.playerNode stop];
|
|
388
|
+
[self.audioEngine stop];
|
|
389
|
+
|
|
390
|
+
[self.bufferLock lock];
|
|
391
|
+
[self.audioBuffer setLength:0];
|
|
392
|
+
[self.bufferLock unlock];
|
|
393
|
+
|
|
394
|
+
self.remoteTrack = nil;
|
|
395
|
+
[self notifyConnectionState:@"disconnected"];
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
- (void)setTranslatedVolume:(double)volume {
|
|
399
|
+
if (self.playerNode) {
|
|
400
|
+
self.playerNode.volume = volume;
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
- (void)sendAudioToPalabra:(AVAudioPCMBuffer *)buffer {
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
#pragma mark - RTCAudioRenderer
|
|
408
|
+
|
|
409
|
+
- (void)renderPCMBuffer:(AVAudioPCMBuffer *)buffer {
|
|
410
|
+
if (!self.translating || !self.webSocket) {
|
|
411
|
+
return;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
static int logCounter = 0;
|
|
415
|
+
if (logCounter++ % 100 == 0) {
|
|
416
|
+
NSLog(@"palabra_render_pcm: frames=%u rate=%.0f channels=%u",
|
|
417
|
+
(unsigned int)buffer.frameLength,
|
|
418
|
+
buffer.format.sampleRate,
|
|
419
|
+
(unsigned int)buffer.format.channelCount);
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
NSData *resampled = [self resampleBuffer:buffer toRate:kSampleRateIn channels:kChannels];
|
|
423
|
+
if (resampled.length == 0) {
|
|
424
|
+
NSLog(@"palabra_resample_empty");
|
|
425
|
+
return;
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
[self.bufferLock lock];
|
|
429
|
+
[self.audioBuffer appendData:resampled];
|
|
430
|
+
|
|
431
|
+
while (self.audioBuffer.length >= kChunkBytes) {
|
|
432
|
+
NSData *chunk = [self.audioBuffer subdataWithRange:NSMakeRange(0, kChunkBytes)];
|
|
433
|
+
NSMutableData *remaining = [NSMutableData dataWithData:[self.audioBuffer subdataWithRange:NSMakeRange(kChunkBytes, self.audioBuffer.length - kChunkBytes)]];
|
|
434
|
+
self.audioBuffer = remaining;
|
|
435
|
+
|
|
436
|
+
[self sendAudioChunk:chunk];
|
|
437
|
+
}
|
|
438
|
+
[self.bufferLock unlock];
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
- (NSData *)resampleBuffer:(AVAudioPCMBuffer *)buffer toRate:(int)dstRate channels:(int)dstChannels {
|
|
442
|
+
int srcRate = (int)buffer.format.sampleRate;
|
|
443
|
+
int srcChannels = (int)buffer.format.channelCount;
|
|
444
|
+
int srcSamples = (int)buffer.frameLength;
|
|
445
|
+
|
|
446
|
+
if (srcSamples == 0) {
|
|
447
|
+
return [NSData data];
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
AVAudioCommonFormat format = buffer.format.commonFormat;
|
|
451
|
+
|
|
452
|
+
if (srcRate == dstRate && srcChannels == dstChannels && format == AVAudioPCMFormatInt16) {
|
|
453
|
+
return [NSData dataWithBytes:buffer.int16ChannelData[0] length:srcSamples * 2];
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
float *srcData = NULL;
|
|
457
|
+
float *tempFloat = NULL;
|
|
458
|
+
|
|
459
|
+
if (format == AVAudioPCMFormatFloat32) {
|
|
460
|
+
srcData = buffer.floatChannelData[0];
|
|
461
|
+
} else if (format == AVAudioPCMFormatInt16) {
|
|
462
|
+
tempFloat = malloc(srcSamples * sizeof(float));
|
|
463
|
+
int16_t *srcInt16 = buffer.int16ChannelData[0];
|
|
464
|
+
for (int i = 0; i < srcSamples; i++) {
|
|
465
|
+
tempFloat[i] = (float)srcInt16[i] / 32768.0f;
|
|
466
|
+
}
|
|
467
|
+
srcData = tempFloat;
|
|
468
|
+
} else {
|
|
469
|
+
NSLog(@"palabra_unsupported_format: %ld", (long)format);
|
|
470
|
+
return [NSData data];
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
float *monoSrc = srcData;
|
|
474
|
+
float *monoBuffer = NULL;
|
|
475
|
+
if (srcChannels == 2) {
|
|
476
|
+
monoBuffer = malloc(srcSamples * sizeof(float));
|
|
477
|
+
if (format == AVAudioPCMFormatFloat32 && buffer.floatChannelData[1]) {
|
|
478
|
+
for (int i = 0; i < srcSamples; i++) {
|
|
479
|
+
monoBuffer[i] = (buffer.floatChannelData[0][i] + buffer.floatChannelData[1][i]) / 2.0f;
|
|
480
|
+
}
|
|
481
|
+
} else if (format == AVAudioPCMFormatInt16 && buffer.int16ChannelData[1]) {
|
|
482
|
+
for (int i = 0; i < srcSamples; i++) {
|
|
483
|
+
float ch0 = (float)buffer.int16ChannelData[0][i] / 32768.0f;
|
|
484
|
+
float ch1 = (float)buffer.int16ChannelData[1][i] / 32768.0f;
|
|
485
|
+
monoBuffer[i] = (ch0 + ch1) / 2.0f;
|
|
486
|
+
}
|
|
487
|
+
} else {
|
|
488
|
+
for (int i = 0; i < srcSamples; i++) {
|
|
489
|
+
monoBuffer[i] = srcData[i];
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
monoSrc = monoBuffer;
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
int dstSamples = (int)((long)srcSamples * dstRate / srcRate);
|
|
496
|
+
if (dstSamples == 0) dstSamples = 1;
|
|
497
|
+
|
|
498
|
+
int16_t *dstData = malloc(dstSamples * sizeof(int16_t));
|
|
499
|
+
|
|
500
|
+
for (int i = 0; i < dstSamples; i++) {
|
|
501
|
+
float srcIdx = (float)i * (srcSamples - 1) / (dstSamples - 1);
|
|
502
|
+
int idx0 = (int)srcIdx;
|
|
503
|
+
int idx1 = MIN(idx0 + 1, srcSamples - 1);
|
|
504
|
+
float frac = srcIdx - idx0;
|
|
505
|
+
float sample = monoSrc[idx0] * (1 - frac) + monoSrc[idx1] * frac;
|
|
506
|
+
sample = fmaxf(-1.0f, fminf(1.0f, sample));
|
|
507
|
+
dstData[i] = (int16_t)(sample * 32767.0f);
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
if (monoBuffer) free(monoBuffer);
|
|
511
|
+
if (tempFloat) free(tempFloat);
|
|
512
|
+
|
|
513
|
+
NSData *result = [NSData dataWithBytes:dstData length:dstSamples * 2];
|
|
514
|
+
free(dstData);
|
|
515
|
+
return result;
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
- (void)sendAudioChunk:(NSData *)chunk {
|
|
519
|
+
if (!self.webSocket || !self.connected) return;
|
|
520
|
+
|
|
521
|
+
static int sendCounter = 0;
|
|
522
|
+
if (sendCounter++ % 50 == 0) {
|
|
523
|
+
NSLog(@"palabra_send_chunk: %lu bytes (count=%d)", (unsigned long)chunk.length, sendCounter);
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
NSString *base64 = [chunk base64EncodedStringWithOptions:0];
|
|
527
|
+
NSDictionary *msg = @{
|
|
528
|
+
@"message_type": @"input_audio_data",
|
|
529
|
+
@"data": @{
|
|
530
|
+
@"data": base64
|
|
531
|
+
}
|
|
532
|
+
};
|
|
533
|
+
|
|
534
|
+
NSData *jsonData = [NSJSONSerialization dataWithJSONObject:msg options:0 error:nil];
|
|
535
|
+
NSString *payload = [[NSString alloc] initWithData:jsonData encoding:NSUTF8StringEncoding];
|
|
536
|
+
|
|
537
|
+
NSURLSessionWebSocketMessage *wsMsg = [[NSURLSessionWebSocketMessage alloc] initWithString:payload];
|
|
538
|
+
[self.webSocket sendMessage:wsMsg completionHandler:^(NSError *error) {
|
|
539
|
+
if (error) {
|
|
540
|
+
NSLog(@"palabra_chunk_send_error: %@", error);
|
|
541
|
+
}
|
|
542
|
+
}];
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
#pragma mark - Notifications
|
|
546
|
+
|
|
547
|
+
- (void)notifyConnectionState:(NSString *)state {
|
|
548
|
+
if ([self.delegate respondsToSelector:@selector(palabraDidChangeConnectionState:)]) {
|
|
549
|
+
[self.delegate palabraDidChangeConnectionState:state];
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
- (void)notifyError:(NSError *)error {
|
|
554
|
+
if ([self.delegate respondsToSelector:@selector(palabraDidFailWithError:)]) {
|
|
555
|
+
[self.delegate palabraDidFailWithError:error];
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
- (void)notifyTranscription:(NSString *)text language:(NSString *)lang isFinal:(BOOL)isFinal {
|
|
560
|
+
if ([self.delegate respondsToSelector:@selector(palabraDidReceiveTranscription:)]) {
|
|
561
|
+
[self.delegate palabraDidReceiveTranscription:@{
|
|
562
|
+
@"text": text,
|
|
563
|
+
@"language": lang,
|
|
564
|
+
@"isFinal": @(isFinal)
|
|
565
|
+
}];
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
#pragma mark - NSURLSessionWebSocketDelegate
|
|
570
|
+
|
|
571
|
+
- (void)URLSession:(NSURLSession *)session webSocketTask:(NSURLSessionWebSocketTask *)webSocketTask didOpenWithProtocol:(NSString *)protocol {
|
|
572
|
+
NSLog(@"palabra_ws_opened");
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
- (void)URLSession:(NSURLSession *)session webSocketTask:(NSURLSessionWebSocketTask *)webSocketTask didCloseWithCode:(NSURLSessionWebSocketCloseCode)closeCode reason:(NSData *)reason {
|
|
576
|
+
NSLog(@"palabra_ws_closed: %ld", (long)closeCode);
|
|
577
|
+
dispatch_async(dispatch_get_main_queue(), ^{
|
|
578
|
+
if (self.translating) {
|
|
579
|
+
[self stop];
|
|
580
|
+
}
|
|
581
|
+
});
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
@end
|