@sbhjt-gr/react-native-webrtc 124.0.3 → 124.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (361) hide show
  1. package/.clang-format +11 -11
  2. package/.eslintignore +6 -6
  3. package/.nvmrc +1 -1
  4. package/ISSUE_TEMPLATE.md +40 -40
  5. package/LICENSE +22 -22
  6. package/README.md +103 -103
  7. package/android/build.gradle +37 -37
  8. package/android/consumer-rules.pro +3 -3
  9. package/android/src/main/AndroidManifest.xml +11 -11
  10. package/android/src/main/java/com/oney/WebRTCModule/AbstractVideoCaptureController.java +113 -113
  11. package/android/src/main/java/com/oney/WebRTCModule/CameraCaptureController.java +338 -338
  12. package/android/src/main/java/com/oney/WebRTCModule/CameraEventsHandler.java +49 -49
  13. package/android/src/main/java/com/oney/WebRTCModule/DataChannelWrapper.java +99 -99
  14. package/android/src/main/java/com/oney/WebRTCModule/DataPacketCryptorManager.java +62 -62
  15. package/android/src/main/java/com/oney/WebRTCModule/DisplayUtils.java +16 -16
  16. package/android/src/main/java/com/oney/WebRTCModule/EglUtils.java +66 -66
  17. package/android/src/main/java/com/oney/WebRTCModule/GetUserMediaImpl.java +539 -539
  18. package/android/src/main/java/com/oney/WebRTCModule/LibraryLoader.java +21 -21
  19. package/android/src/main/java/com/oney/WebRTCModule/MediaProjectionNotification.java +70 -70
  20. package/android/src/main/java/com/oney/WebRTCModule/MediaProjectionService.java +82 -82
  21. package/android/src/main/java/com/oney/WebRTCModule/PeerConnectionObserver.java +588 -588
  22. package/android/src/main/java/com/oney/WebRTCModule/RTCCryptoManager.java +493 -493
  23. package/android/src/main/java/com/oney/WebRTCModule/RTCVideoViewManager.java +98 -98
  24. package/android/src/main/java/com/oney/WebRTCModule/ReactBridgeUtil.java +35 -35
  25. package/android/src/main/java/com/oney/WebRTCModule/ScreenCaptureController.java +94 -94
  26. package/android/src/main/java/com/oney/WebRTCModule/SerializeUtils.java +342 -342
  27. package/android/src/main/java/com/oney/WebRTCModule/StringUtils.java +100 -100
  28. package/android/src/main/java/com/oney/WebRTCModule/ThreadUtils.java +41 -41
  29. package/android/src/main/java/com/oney/WebRTCModule/TrackCapturerEventsEmitter.java +34 -34
  30. package/android/src/main/java/com/oney/WebRTCModule/VideoTrackAdapter.java +137 -137
  31. package/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java +1649 -1643
  32. package/android/src/main/java/com/oney/WebRTCModule/WebRTCModuleOptions.java +33 -33
  33. package/android/src/main/java/com/oney/WebRTCModule/WebRTCModulePackage.java +21 -21
  34. package/android/src/main/java/com/oney/WebRTCModule/WebRTCView.java +583 -583
  35. package/android/src/main/java/com/oney/WebRTCModule/palabra/PalabraClient.java +529 -501
  36. package/android/src/main/java/com/oney/WebRTCModule/palabra/PalabraConfig.java +17 -17
  37. package/android/src/main/java/com/oney/WebRTCModule/palabra/PalabraListener.java +7 -7
  38. package/android/src/main/java/com/oney/WebRTCModule/videoEffects/ProcessorProvider.java +38 -38
  39. package/android/src/main/java/com/oney/WebRTCModule/videoEffects/VideoEffectProcessor.java +59 -59
  40. package/android/src/main/java/com/oney/WebRTCModule/videoEffects/VideoFrameProcessor.java +19 -19
  41. package/android/src/main/java/com/oney/WebRTCModule/videoEffects/VideoFrameProcessorFactoryInterface.java +12 -12
  42. package/android/src/main/java/com/oney/WebRTCModule/webrtcutils/H264AndSoftwareVideoDecoderFactory.java +73 -73
  43. package/android/src/main/java/com/oney/WebRTCModule/webrtcutils/H264AndSoftwareVideoEncoderFactory.java +73 -73
  44. package/android/src/main/java/com/oney/WebRTCModule/webrtcutils/SoftwareVideoDecoderFactoryProxy.java +36 -36
  45. package/android/src/main/java/com/oney/WebRTCModule/webrtcutils/SoftwareVideoEncoderFactoryProxy.java +36 -36
  46. package/android/src/main/java/org/webrtc/Camera1Helper.java +54 -54
  47. package/android/src/main/java/org/webrtc/Camera2Helper.java +52 -52
  48. package/android/src/main/res/values/strings.xml +5 -5
  49. package/android/src/main/res/values/styles.xml +8 -8
  50. package/ios/RCTWebRTC/CaptureController.h +18 -18
  51. package/ios/RCTWebRTC/CaptureController.m +28 -28
  52. package/ios/RCTWebRTC/CapturerEventsDelegate.h +12 -12
  53. package/ios/RCTWebRTC/DataChannelWrapper.h +27 -27
  54. package/ios/RCTWebRTC/DataChannelWrapper.m +42 -42
  55. package/ios/RCTWebRTC/I420Converter.h +22 -22
  56. package/ios/RCTWebRTC/I420Converter.m +164 -164
  57. package/ios/RCTWebRTC/PIPController.h +24 -24
  58. package/ios/RCTWebRTC/PIPController.m +234 -234
  59. package/ios/RCTWebRTC/PalabraAudioSink.h +13 -13
  60. package/ios/RCTWebRTC/PalabraAudioSink.m +18 -18
  61. package/ios/RCTWebRTC/PalabraClient.h +42 -36
  62. package/ios/RCTWebRTC/PalabraClient.m +680 -644
  63. package/ios/RCTWebRTC/RCTConvert+WebRTC.h +16 -16
  64. package/ios/RCTWebRTC/RCTConvert+WebRTC.m +206 -206
  65. package/ios/RCTWebRTC/RTCMediaStreamTrack+React.h +10 -10
  66. package/ios/RCTWebRTC/RTCMediaStreamTrack+React.m +16 -16
  67. package/ios/RCTWebRTC/RTCVideoViewManager.h +29 -29
  68. package/ios/RCTWebRTC/RTCVideoViewManager.m +411 -411
  69. package/ios/RCTWebRTC/SampleBufferVideoCallView.h +12 -12
  70. package/ios/RCTWebRTC/SampleBufferVideoCallView.m +178 -178
  71. package/ios/RCTWebRTC/ScreenCaptureController.h +20 -20
  72. package/ios/RCTWebRTC/ScreenCaptureController.m +82 -82
  73. package/ios/RCTWebRTC/ScreenCapturePickerViewManager.h +7 -7
  74. package/ios/RCTWebRTC/ScreenCapturePickerViewManager.m +59 -59
  75. package/ios/RCTWebRTC/ScreenCapturer.h +19 -19
  76. package/ios/RCTWebRTC/ScreenCapturer.m +263 -263
  77. package/ios/RCTWebRTC/SerializeUtils.h +28 -28
  78. package/ios/RCTWebRTC/SerializeUtils.m +314 -314
  79. package/ios/RCTWebRTC/SocketConnection.h +13 -13
  80. package/ios/RCTWebRTC/SocketConnection.m +137 -137
  81. package/ios/RCTWebRTC/TrackCapturerEventsEmitter.h +14 -14
  82. package/ios/RCTWebRTC/TrackCapturerEventsEmitter.m +36 -36
  83. package/ios/RCTWebRTC/VideoCaptureController.h +21 -21
  84. package/ios/RCTWebRTC/VideoCaptureController.m +328 -328
  85. package/ios/RCTWebRTC/WLVAudioDevice.h +12 -12
  86. package/ios/RCTWebRTC/WLVAudioDevice.m +137 -137
  87. package/ios/RCTWebRTC/WebRTCModule+Palabra.h +4 -4
  88. package/ios/RCTWebRTC/WebRTCModule+Palabra.m +92 -83
  89. package/ios/RCTWebRTC/WebRTCModule+Permissions.m +75 -75
  90. package/ios/RCTWebRTC/WebRTCModule+RTCAudioSession.m +20 -20
  91. package/ios/RCTWebRTC/WebRTCModule+RTCDataChannel.h +14 -14
  92. package/ios/RCTWebRTC/WebRTCModule+RTCDataChannel.m +165 -165
  93. package/ios/RCTWebRTC/WebRTCModule+RTCFrameCryptor.m +611 -611
  94. package/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.h +13 -13
  95. package/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m +728 -728
  96. package/ios/RCTWebRTC/WebRTCModule+RTCPeerConnection.h +24 -24
  97. package/ios/RCTWebRTC/WebRTCModule+RTCPeerConnection.m +1004 -1004
  98. package/ios/RCTWebRTC/WebRTCModule+Transceivers.m +267 -267
  99. package/ios/RCTWebRTC/WebRTCModule+VideoTrackAdapter.h +12 -12
  100. package/ios/RCTWebRTC/WebRTCModule+VideoTrackAdapter.m +166 -166
  101. package/ios/RCTWebRTC/WebRTCModule.h +58 -58
  102. package/ios/RCTWebRTC/WebRTCModule.m +169 -169
  103. package/ios/RCTWebRTC/WebRTCModuleOptions.h +24 -24
  104. package/ios/RCTWebRTC/WebRTCModuleOptions.m +31 -31
  105. package/ios/RCTWebRTC/videoEffects/ProcessorProvider.h +9 -9
  106. package/ios/RCTWebRTC/videoEffects/ProcessorProvider.m +23 -23
  107. package/ios/RCTWebRTC/videoEffects/VideoEffectProcessor.h +13 -13
  108. package/ios/RCTWebRTC/videoEffects/VideoEffectProcessor.m +23 -23
  109. package/ios/RCTWebRTC/videoEffects/VideoFrameProcessor.h +8 -8
  110. package/ios/RCTWebRTC.xcodeproj/project.pbxproj +404 -404
  111. package/ios/RCTWebRTC.xcworkspace/contents.xcworkspacedata +10 -10
  112. package/lib/commonjs/Constraints.js.map +1 -1
  113. package/lib/commonjs/EventEmitter.js.map +1 -1
  114. package/lib/commonjs/Logger.js.map +1 -1
  115. package/lib/commonjs/MediaDevices.js +17 -17
  116. package/lib/commonjs/MediaDevices.js.map +1 -1
  117. package/lib/commonjs/MediaStream.js +19 -19
  118. package/lib/commonjs/MediaStream.js.map +1 -1
  119. package/lib/commonjs/MediaStreamError.js.map +1 -1
  120. package/lib/commonjs/MediaStreamErrorEvent.js.map +1 -1
  121. package/lib/commonjs/MediaStreamTrack.js +28 -28
  122. package/lib/commonjs/MediaStreamTrack.js.map +1 -1
  123. package/lib/commonjs/MediaStreamTrackEvent.js +6 -6
  124. package/lib/commonjs/MediaStreamTrackEvent.js.map +1 -1
  125. package/lib/commonjs/MessageEvent.js +7 -7
  126. package/lib/commonjs/MessageEvent.js.map +1 -1
  127. package/lib/commonjs/Permissions.js +28 -28
  128. package/lib/commonjs/Permissions.js.map +1 -1
  129. package/lib/commonjs/RTCAudioSession.js +4 -4
  130. package/lib/commonjs/RTCAudioSession.js.map +1 -1
  131. package/lib/commonjs/RTCDataChannel.js +2 -2
  132. package/lib/commonjs/RTCDataChannel.js.map +1 -1
  133. package/lib/commonjs/RTCDataChannelEvent.js +6 -6
  134. package/lib/commonjs/RTCDataChannelEvent.js.map +1 -1
  135. package/lib/commonjs/RTCDataPacketCryptor.js.map +1 -1
  136. package/lib/commonjs/RTCDataPacketCryptorFactory.js.map +1 -1
  137. package/lib/commonjs/RTCErrorEvent.js +3 -3
  138. package/lib/commonjs/RTCErrorEvent.js.map +1 -1
  139. package/lib/commonjs/RTCFrameCryptor.js +8 -8
  140. package/lib/commonjs/RTCFrameCryptor.js.map +1 -1
  141. package/lib/commonjs/RTCFrameCryptorFactory.js.map +1 -1
  142. package/lib/commonjs/RTCIceCandidate.js.map +1 -1
  143. package/lib/commonjs/RTCIceCandidateEvent.js +7 -7
  144. package/lib/commonjs/RTCIceCandidateEvent.js.map +1 -1
  145. package/lib/commonjs/RTCKeyProvider.js.map +1 -1
  146. package/lib/commonjs/RTCPIPView.js +2 -2
  147. package/lib/commonjs/RTCPIPView.js.map +1 -1
  148. package/lib/commonjs/RTCPIPView.web.js.map +1 -1
  149. package/lib/commonjs/RTCPeerConnection.js +36 -36
  150. package/lib/commonjs/RTCPeerConnection.js.map +1 -1
  151. package/lib/commonjs/RTCRtcpParameters.js.map +1 -1
  152. package/lib/commonjs/RTCRtpCapabilities.js +2 -2
  153. package/lib/commonjs/RTCRtpCapabilities.js.map +1 -1
  154. package/lib/commonjs/RTCRtpCodecCapability.js.map +1 -1
  155. package/lib/commonjs/RTCRtpCodecParameters.js.map +1 -1
  156. package/lib/commonjs/RTCRtpEncodingParameters.js.map +1 -1
  157. package/lib/commonjs/RTCRtpHeaderExtension.js.map +1 -1
  158. package/lib/commonjs/RTCRtpParameters.js.map +1 -1
  159. package/lib/commonjs/RTCRtpReceiveParameters.js.map +1 -1
  160. package/lib/commonjs/RTCRtpReceiver.js +7 -7
  161. package/lib/commonjs/RTCRtpReceiver.js.map +1 -1
  162. package/lib/commonjs/RTCRtpSendParameters.js +3 -3
  163. package/lib/commonjs/RTCRtpSendParameters.js.map +1 -1
  164. package/lib/commonjs/RTCRtpSender.js +7 -7
  165. package/lib/commonjs/RTCRtpSender.js.map +1 -1
  166. package/lib/commonjs/RTCRtpTransceiver.js.map +1 -1
  167. package/lib/commonjs/RTCSessionDescription.js.map +1 -1
  168. package/lib/commonjs/RTCTrackEvent.js +6 -6
  169. package/lib/commonjs/RTCTrackEvent.js.map +1 -1
  170. package/lib/commonjs/RTCUtil.js +28 -28
  171. package/lib/commonjs/RTCUtil.js.map +1 -1
  172. package/lib/commonjs/RTCView.js +5 -5
  173. package/lib/commonjs/RTCView.js.map +1 -1
  174. package/lib/commonjs/RTCView.web.js.map +1 -1
  175. package/lib/commonjs/ScreenCapturePickerView.js.map +1 -1
  176. package/lib/commonjs/ScreenCapturePickerView.web.js.map +1 -1
  177. package/lib/commonjs/getDisplayMedia.js.map +1 -1
  178. package/lib/commonjs/getUserMedia.js.map +1 -1
  179. package/lib/commonjs/index.js.map +1 -1
  180. package/lib/commonjs/index.web.js.map +1 -1
  181. package/lib/commonjs/webStream.js.map +1 -1
  182. package/lib/module/Constraints.js.map +1 -1
  183. package/lib/module/EventEmitter.js.map +1 -1
  184. package/lib/module/Logger.js.map +1 -1
  185. package/lib/module/MediaDevices.js +17 -17
  186. package/lib/module/MediaDevices.js.map +1 -1
  187. package/lib/module/MediaStream.js +19 -19
  188. package/lib/module/MediaStream.js.map +1 -1
  189. package/lib/module/MediaStreamError.js.map +1 -1
  190. package/lib/module/MediaStreamErrorEvent.js.map +1 -1
  191. package/lib/module/MediaStreamTrack.js +28 -28
  192. package/lib/module/MediaStreamTrack.js.map +1 -1
  193. package/lib/module/MediaStreamTrackEvent.js +6 -6
  194. package/lib/module/MediaStreamTrackEvent.js.map +1 -1
  195. package/lib/module/MessageEvent.js +7 -7
  196. package/lib/module/MessageEvent.js.map +1 -1
  197. package/lib/module/Permissions.js +28 -28
  198. package/lib/module/Permissions.js.map +1 -1
  199. package/lib/module/RTCAudioSession.js +4 -4
  200. package/lib/module/RTCAudioSession.js.map +1 -1
  201. package/lib/module/RTCDataChannel.js +2 -2
  202. package/lib/module/RTCDataChannel.js.map +1 -1
  203. package/lib/module/RTCDataChannelEvent.js +6 -6
  204. package/lib/module/RTCDataChannelEvent.js.map +1 -1
  205. package/lib/module/RTCDataPacketCryptor.js.map +1 -1
  206. package/lib/module/RTCDataPacketCryptorFactory.js.map +1 -1
  207. package/lib/module/RTCErrorEvent.js +3 -3
  208. package/lib/module/RTCErrorEvent.js.map +1 -1
  209. package/lib/module/RTCFrameCryptor.js +8 -8
  210. package/lib/module/RTCFrameCryptor.js.map +1 -1
  211. package/lib/module/RTCFrameCryptorFactory.js.map +1 -1
  212. package/lib/module/RTCIceCandidate.js.map +1 -1
  213. package/lib/module/RTCIceCandidateEvent.js +7 -7
  214. package/lib/module/RTCIceCandidateEvent.js.map +1 -1
  215. package/lib/module/RTCKeyProvider.js.map +1 -1
  216. package/lib/module/RTCPIPView.js +2 -2
  217. package/lib/module/RTCPIPView.js.map +1 -1
  218. package/lib/module/RTCPIPView.web.js.map +1 -1
  219. package/lib/module/RTCPeerConnection.js +36 -36
  220. package/lib/module/RTCPeerConnection.js.map +1 -1
  221. package/lib/module/RTCRtcpParameters.js.map +1 -1
  222. package/lib/module/RTCRtpCapabilities.js +2 -2
  223. package/lib/module/RTCRtpCapabilities.js.map +1 -1
  224. package/lib/module/RTCRtpCodecCapability.js.map +1 -1
  225. package/lib/module/RTCRtpCodecParameters.js.map +1 -1
  226. package/lib/module/RTCRtpEncodingParameters.js.map +1 -1
  227. package/lib/module/RTCRtpHeaderExtension.js.map +1 -1
  228. package/lib/module/RTCRtpParameters.js.map +1 -1
  229. package/lib/module/RTCRtpReceiveParameters.js.map +1 -1
  230. package/lib/module/RTCRtpReceiver.js +7 -7
  231. package/lib/module/RTCRtpReceiver.js.map +1 -1
  232. package/lib/module/RTCRtpSendParameters.js +3 -3
  233. package/lib/module/RTCRtpSendParameters.js.map +1 -1
  234. package/lib/module/RTCRtpSender.js +7 -7
  235. package/lib/module/RTCRtpSender.js.map +1 -1
  236. package/lib/module/RTCRtpTransceiver.js.map +1 -1
  237. package/lib/module/RTCSessionDescription.js.map +1 -1
  238. package/lib/module/RTCTrackEvent.js +6 -6
  239. package/lib/module/RTCTrackEvent.js.map +1 -1
  240. package/lib/module/RTCUtil.js +28 -28
  241. package/lib/module/RTCUtil.js.map +1 -1
  242. package/lib/module/RTCView.js +5 -5
  243. package/lib/module/RTCView.js.map +1 -1
  244. package/lib/module/RTCView.web.js.map +1 -1
  245. package/lib/module/ScreenCapturePickerView.js.map +1 -1
  246. package/lib/module/ScreenCapturePickerView.web.js.map +1 -1
  247. package/lib/module/getDisplayMedia.js.map +1 -1
  248. package/lib/module/getUserMedia.js.map +1 -1
  249. package/lib/module/index.js.map +1 -1
  250. package/lib/module/index.web.js.map +1 -1
  251. package/lib/module/webStream.js.map +1 -1
  252. package/lib/typescript/Constraints.d.ts +19 -19
  253. package/lib/typescript/EventEmitter.d.ts +6 -6
  254. package/lib/typescript/Logger.d.ts +13 -13
  255. package/lib/typescript/MediaDevices.d.ts +30 -30
  256. package/lib/typescript/MediaStream.d.ts +48 -48
  257. package/lib/typescript/MediaStreamError.d.ts +6 -6
  258. package/lib/typescript/MediaStreamErrorEvent.d.ts +6 -6
  259. package/lib/typescript/MediaStreamTrack.d.ts +101 -101
  260. package/lib/typescript/MediaStreamTrackEvent.d.ts +19 -19
  261. package/lib/typescript/MessageEvent.d.ts +20 -20
  262. package/lib/typescript/Permissions.d.ts +55 -55
  263. package/lib/typescript/RTCAudioSession.d.ts +10 -10
  264. package/lib/typescript/RTCDataChannel.d.ts +43 -43
  265. package/lib/typescript/RTCDataChannelEvent.d.ts +19 -19
  266. package/lib/typescript/RTCDataPacketCryptor.d.ts +12 -12
  267. package/lib/typescript/RTCDataPacketCryptorFactory.d.ts +6 -6
  268. package/lib/typescript/RTCErrorEvent.d.ts +12 -12
  269. package/lib/typescript/RTCFrameCryptor.d.ts +47 -47
  270. package/lib/typescript/RTCFrameCryptorFactory.d.ts +21 -21
  271. package/lib/typescript/RTCIceCandidate.d.ts +17 -17
  272. package/lib/typescript/RTCIceCandidateEvent.d.ts +20 -20
  273. package/lib/typescript/RTCKeyProvider.d.ts +21 -21
  274. package/lib/typescript/RTCPIPView.d.ts +15 -15
  275. package/lib/typescript/RTCPIPView.web.d.ts +13 -13
  276. package/lib/typescript/RTCPeerConnection.d.ts +117 -117
  277. package/lib/typescript/RTCRtcpParameters.d.ts +10 -10
  278. package/lib/typescript/RTCRtpCapabilities.d.ts +9 -9
  279. package/lib/typescript/RTCRtpCodecCapability.d.ts +7 -7
  280. package/lib/typescript/RTCRtpCodecParameters.d.ts +16 -16
  281. package/lib/typescript/RTCRtpEncodingParameters.d.ts +23 -23
  282. package/lib/typescript/RTCRtpHeaderExtension.d.ts +12 -12
  283. package/lib/typescript/RTCRtpParameters.d.ts +19 -19
  284. package/lib/typescript/RTCRtpReceiveParameters.d.ts +4 -4
  285. package/lib/typescript/RTCRtpReceiver.d.ts +21 -21
  286. package/lib/typescript/RTCRtpSendParameters.d.ts +20 -20
  287. package/lib/typescript/RTCRtpSender.d.ts +22 -22
  288. package/lib/typescript/RTCRtpTransceiver.d.ts +31 -31
  289. package/lib/typescript/RTCSessionDescription.d.ts +12 -12
  290. package/lib/typescript/RTCTrackEvent.d.ts +29 -29
  291. package/lib/typescript/RTCUtil.d.ts +37 -37
  292. package/lib/typescript/RTCView.d.ts +117 -117
  293. package/lib/typescript/RTCView.web.d.ts +25 -25
  294. package/lib/typescript/ScreenCapturePickerView.d.ts +2 -2
  295. package/lib/typescript/ScreenCapturePickerView.web.d.ts +1 -1
  296. package/lib/typescript/getDisplayMedia.d.ts +2 -2
  297. package/lib/typescript/getUserMedia.d.ts +7 -7
  298. package/lib/typescript/index.d.ts +22 -22
  299. package/lib/typescript/index.web.d.ts +101 -101
  300. package/lib/typescript/webStream.d.ts +3 -3
  301. package/livekit-react-native-webrtc.podspec +29 -29
  302. package/macos/RCTWebRTC.xcodeproj/project.pbxproj +324 -324
  303. package/macos/RCTWebRTC.xcodeproj/project.xcworkspace/contents.xcworkspacedata +7 -7
  304. package/macos/RCTWebRTC.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +8 -8
  305. package/metro.config.js +7 -7
  306. package/metro.config.macos.js +14 -14
  307. package/package.json +66 -66
  308. package/react-native.config.js +11 -11
  309. package/src/.eslintrc.cjs +67 -67
  310. package/src/Constraints.ts +20 -20
  311. package/src/EventEmitter.ts +65 -65
  312. package/src/Logger.ts +49 -49
  313. package/src/MediaDevices.ts +53 -53
  314. package/src/MediaStream.ts +161 -161
  315. package/src/MediaStreamError.ts +12 -12
  316. package/src/MediaStreamErrorEvent.ts +11 -11
  317. package/src/MediaStreamTrack.ts +282 -282
  318. package/src/MediaStreamTrackEvent.ts +25 -25
  319. package/src/MessageEvent.ts +26 -26
  320. package/src/Permissions.ts +133 -133
  321. package/src/RTCAudioSession.ts +25 -25
  322. package/src/RTCDataChannel.ts +190 -190
  323. package/src/RTCDataChannelEvent.ts +28 -28
  324. package/src/RTCDataPacketCryptor.ts +90 -90
  325. package/src/RTCDataPacketCryptorFactory.ts +24 -24
  326. package/src/RTCErrorEvent.ts +20 -20
  327. package/src/RTCFrameCryptor.ts +162 -162
  328. package/src/RTCFrameCryptorFactory.ts +101 -101
  329. package/src/RTCIceCandidate.ts +29 -29
  330. package/src/RTCIceCandidateEvent.ts +26 -26
  331. package/src/RTCKeyProvider.ts +117 -117
  332. package/src/RTCPIPView.tsx +46 -46
  333. package/src/RTCPIPView.web.tsx +18 -18
  334. package/src/RTCPeerConnection.ts +935 -935
  335. package/src/RTCRtcpParameters.ts +23 -23
  336. package/src/RTCRtpCapabilities.ts +16 -16
  337. package/src/RTCRtpCodecCapability.ts +12 -12
  338. package/src/RTCRtpCodecParameters.ts +44 -44
  339. package/src/RTCRtpEncodingParameters.ts +90 -90
  340. package/src/RTCRtpHeaderExtension.ts +27 -27
  341. package/src/RTCRtpParameters.ts +37 -37
  342. package/src/RTCRtpReceiveParameters.ts +7 -7
  343. package/src/RTCRtpReceiver.ts +60 -60
  344. package/src/RTCRtpSendParameters.ts +63 -63
  345. package/src/RTCRtpSender.ts +78 -78
  346. package/src/RTCRtpTransceiver.ts +107 -107
  347. package/src/RTCSessionDescription.ts +30 -30
  348. package/src/RTCTrackEvent.ts +42 -42
  349. package/src/RTCUtil.ts +211 -211
  350. package/src/RTCView.ts +122 -122
  351. package/src/RTCView.web.tsx +80 -80
  352. package/src/ScreenCapturePickerView.ts +4 -4
  353. package/src/ScreenCapturePickerView.web.tsx +3 -3
  354. package/src/getDisplayMedia.ts +30 -30
  355. package/src/getUserMedia.ts +136 -136
  356. package/src/index.ts +107 -107
  357. package/src/index.web.ts +191 -191
  358. package/src/webStream.ts +31 -31
  359. package/tools/format.sh +6 -6
  360. package/tools/release.sh +45 -45
  361. package/tsconfig.json +17 -17
@@ -1,644 +1,680 @@
1
- #import "PalabraClient.h"
2
- #import "WebRTCModule.h"
3
- #import <AVFoundation/AVFoundation.h>
4
-
5
- static const int kSampleRateIn = 16000;
6
- static const int kSampleRateOut = 24000;
7
- static const int kChannels = 1;
8
- static const int kChunkMs = 320;
9
- static const int kChunkSamples = kSampleRateIn * kChunkMs / 1000;
10
- static const int kChunkBytes = kChunkSamples * 2;
11
-
12
- @interface PalabraClient () <NSURLSessionWebSocketDelegate, RTCAudioRenderer>
13
-
14
- @property (nonatomic, strong) NSString *clientId;
15
- @property (nonatomic, strong) NSString *clientSecret;
16
- @property (nonatomic, strong) NSString *apiUrl;
17
- @property (nonatomic, strong) NSString *sourceLang;
18
- @property (nonatomic, strong) NSString *targetLang;
19
-
20
- @property (nonatomic, strong) RTCAudioTrack *remoteTrack;
21
-
22
- @property (nonatomic, strong) NSString *sessionId;
23
- @property (nonatomic, strong) NSString *wsUrl;
24
- @property (nonatomic, strong) NSString *publisherToken;
25
-
26
- @property (nonatomic, strong) NSURLSession *urlSession;
27
- @property (nonatomic, strong) NSURLSessionWebSocketTask *webSocket;
28
-
29
- @property (nonatomic, strong) AVAudioEngine *audioEngine;
30
- @property (nonatomic, strong) AVAudioPlayerNode *playerNode;
31
- @property (nonatomic, strong) AVAudioFormat *audioFormat;
32
-
33
- @property (nonatomic, assign) BOOL connected;
34
- @property (nonatomic, assign) BOOL translating;
35
- @property (nonatomic, assign) double originalVolume;
36
-
37
- @property (nonatomic, strong) NSMutableData *audioBuffer;
38
- @property (nonatomic, strong) NSLock *bufferLock;
39
-
40
- @end
41
-
42
- @implementation PalabraClient
43
-
44
- - (instancetype)initWithClientId:(NSString *)clientId
45
- clientSecret:(NSString *)clientSecret
46
- apiUrl:(NSString *)apiUrl
47
- module:(WebRTCModule *)module {
48
- self = [super init];
49
- if (self) {
50
- _clientId = clientId;
51
- _clientSecret = clientSecret;
52
- _apiUrl = apiUrl;
53
- _module = module;
54
- _connected = NO;
55
- _translating = NO;
56
- _originalVolume = 1.0;
57
- _audioBuffer = [NSMutableData new];
58
- _bufferLock = [NSLock new];
59
-
60
- NSURLSessionConfiguration *config = [NSURLSessionConfiguration defaultSessionConfiguration];
61
- _urlSession = [NSURLSession sessionWithConfiguration:config delegate:self delegateQueue:nil];
62
-
63
- [self setupAudioEngine];
64
- }
65
- return self;
66
- }
67
-
68
- - (BOOL)isConnected {
69
- return _connected;
70
- }
71
-
72
- - (BOOL)isTranslating {
73
- return _translating;
74
- }
75
-
76
- - (void)setupAudioEngine {
77
- self.audioEngine = [[AVAudioEngine alloc] init];
78
- self.playerNode = [[AVAudioPlayerNode alloc] init];
79
- [self.audioEngine attachNode:self.playerNode];
80
-
81
- self.audioFormat = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16
82
- sampleRate:kSampleRateOut
83
- channels:kChannels
84
- interleaved:YES];
85
-
86
- [self.audioEngine connect:self.playerNode
87
- to:self.audioEngine.mainMixerNode
88
- format:self.audioFormat];
89
- }
90
-
91
- - (void)startWithTrack:(RTCAudioTrack *)remoteAudioTrack
92
- sourceLang:(NSString *)sourceLang
93
- targetLang:(NSString *)targetLang {
94
-
95
- if (self.translating) {
96
- return;
97
- }
98
-
99
- self.remoteTrack = remoteAudioTrack;
100
- self.sourceLang = sourceLang;
101
- self.targetLang = targetLang;
102
-
103
- self.originalVolume = remoteAudioTrack.source.volume;
104
- remoteAudioTrack.source.volume = 0;
105
-
106
- [self notifyConnectionState:@"connecting"];
107
-
108
- __weak typeof(self) weakSelf = self;
109
- [self createSessionWithCompletion:^(NSDictionary *session, NSError *error) {
110
- __strong typeof(weakSelf) strongSelf = weakSelf;
111
- if (!strongSelf) return;
112
-
113
- if (error) {
114
- [strongSelf notifyError:error];
115
- strongSelf.remoteTrack.source.volume = strongSelf.originalVolume;
116
- return;
117
- }
118
-
119
- NSDictionary *data = session[@"data"];
120
- strongSelf.sessionId = data[@"id"];
121
- strongSelf.wsUrl = data[@"ws_url"];
122
- strongSelf.publisherToken = data[@"publisher"];
123
-
124
- NSLog(@"palabra_ws_url: %@", strongSelf.wsUrl);
125
-
126
- dispatch_async(dispatch_get_main_queue(), ^{
127
- [strongSelf connectWebSocket];
128
- });
129
- }];
130
- }
131
-
132
- - (void)createSessionWithCompletion:(void (^)(NSDictionary *, NSError *))completion {
133
- NSString *urlStr = [NSString stringWithFormat:@"%@/session-storage/session", self.apiUrl];
134
- NSURL *url = [NSURL URLWithString:urlStr];
135
-
136
- NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:url];
137
- request.HTTPMethod = @"POST";
138
- [request setValue:self.clientId forHTTPHeaderField:@"ClientId"];
139
- [request setValue:self.clientSecret forHTTPHeaderField:@"ClientSecret"];
140
- [request setValue:@"application/json" forHTTPHeaderField:@"Content-Type"];
141
-
142
- NSDictionary *body = @{
143
- @"data": @{
144
- @"subscriber_count": @0,
145
- @"publisher_can_subscribe": @YES
146
- }
147
- };
148
-
149
- NSError *jsonError;
150
- request.HTTPBody = [NSJSONSerialization dataWithJSONObject:body options:0 error:&jsonError];
151
-
152
- if (jsonError) {
153
- completion(nil, jsonError);
154
- return;
155
- }
156
-
157
- NSURLSessionDataTask *task = [[NSURLSession sharedSession] dataTaskWithRequest:request
158
- completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) {
159
- if (error) {
160
- dispatch_async(dispatch_get_main_queue(), ^{
161
- completion(nil, error);
162
- });
163
- return;
164
- }
165
-
166
- NSHTTPURLResponse *httpResponse = (NSHTTPURLResponse *)response;
167
- if (httpResponse.statusCode < 200 || httpResponse.statusCode >= 300) {
168
- NSString *message = [self sessionErrorMessageForStatusCode:httpResponse.statusCode data:data];
169
- NSError *httpError = [NSError errorWithDomain:@"PalabraClient"
170
- code:httpResponse.statusCode
171
- userInfo:@{NSLocalizedDescriptionKey: message}];
172
- dispatch_async(dispatch_get_main_queue(), ^{
173
- completion(nil, httpError);
174
- });
175
- return;
176
- }
177
-
178
- NSError *parseError;
179
- NSDictionary *json = [NSJSONSerialization JSONObjectWithData:data options:0 error:&parseError];
180
-
181
- dispatch_async(dispatch_get_main_queue(), ^{
182
- completion(json, parseError);
183
- });
184
- }];
185
-
186
- [task resume];
187
- }
188
-
189
- - (NSString *)sessionErrorMessageForStatusCode:(NSInteger)statusCode data:(NSData *)data {
190
- NSString *fallback = [NSString stringWithFormat:@"session_http_error_%ld", (long)statusCode];
191
- if (!data || data.length == 0) {
192
- return fallback;
193
- }
194
-
195
- NSError *parseError;
196
- id json = [NSJSONSerialization JSONObjectWithData:data options:0 error:&parseError];
197
- if (parseError || ![json isKindOfClass:[NSDictionary class]]) {
198
- return fallback;
199
- }
200
-
201
- NSArray *errors = ((NSDictionary *)json)[@"errors"];
202
- if (![errors isKindOfClass:[NSArray class]] || errors.count == 0) {
203
- return fallback;
204
- }
205
-
206
- NSDictionary *first = errors.firstObject;
207
- if (![first isKindOfClass:[NSDictionary class]]) {
208
- return fallback;
209
- }
210
-
211
- NSString *detail = first[@"detail"];
212
- if (![detail isKindOfClass:[NSString class]] || detail.length == 0) {
213
- return fallback;
214
- }
215
-
216
- return [NSString stringWithFormat:@"%@: %@", fallback, detail];
217
- }
218
-
219
- - (void)connectWebSocket {
220
- NSString *endpoint = [NSString stringWithFormat:@"%@?token=%@", self.wsUrl, self.publisherToken];
221
- NSLog(@"palabra_connecting_ws: %@", endpoint);
222
-
223
- NSURL *url = [NSURL URLWithString:endpoint];
224
- self.webSocket = [self.urlSession webSocketTaskWithURL:url];
225
- [self.webSocket resume];
226
-
227
- [self receiveMessage];
228
- }
229
-
230
- - (void)handleWebSocketOpened {
231
- if (self.connected || self.translating || !self.webSocket) {
232
- return;
233
- }
234
-
235
- self.connected = YES;
236
- self.translating = YES;
237
-
238
- [self.remoteTrack addRenderer:self];
239
-
240
- NSError *audioError;
241
- [self.audioEngine startAndReturnError:&audioError];
242
- if (audioError) {
243
- NSLog(@"palabra_audio_engine_error: %@", audioError);
244
- [self.remoteTrack removeRenderer:self];
245
- self.translating = NO;
246
- self.connected = NO;
247
- [self.webSocket cancelWithCloseCode:NSURLSessionWebSocketCloseCodeInternalServerError reason:nil];
248
- self.webSocket = nil;
249
- [self notifyError:audioError];
250
- return;
251
- }
252
-
253
- [self.playerNode play];
254
- NSLog(@"palabra_audio_engine_started");
255
-
256
- [self notifyConnectionState:@"connected"];
257
-
258
- dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.5 * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{
259
- [self sendSetTask];
260
- });
261
- }
262
-
263
- - (void)receiveMessage {
264
- if (!self.webSocket) return;
265
-
266
- __weak typeof(self) weakSelf = self;
267
- [self.webSocket receiveMessageWithCompletionHandler:^(NSURLSessionWebSocketMessage *message, NSError *error) {
268
- __strong typeof(weakSelf) strongSelf = weakSelf;
269
- if (!strongSelf) return;
270
-
271
- if (error) {
272
- NSLog(@"palabra_ws_error: %@", error);
273
- dispatch_async(dispatch_get_main_queue(), ^{
274
- [strongSelf stop];
275
- [strongSelf notifyError:error];
276
- });
277
- return;
278
- }
279
-
280
- if (message.type == NSURLSessionWebSocketMessageTypeString) {
281
- [strongSelf handleMessage:message.string];
282
- }
283
-
284
- [strongSelf receiveMessage];
285
- }];
286
- }
287
-
288
- - (void)sendSetTask {
289
- if (!self.webSocket || !self.connected) return;
290
-
291
- NSDictionary *msg = @{
292
- @"message_type": @"set_task",
293
- @"data": @{
294
- @"input_stream": @{
295
- @"content_type": @"audio",
296
- @"source": @{
297
- @"type": @"ws",
298
- @"format": @"pcm_s16le",
299
- @"sample_rate": @(kSampleRateIn),
300
- @"channels": @(kChannels)
301
- }
302
- },
303
- @"output_stream": @{
304
- @"content_type": @"audio",
305
- @"target": @{
306
- @"type": @"ws",
307
- @"format": @"pcm_s16le"
308
- }
309
- },
310
- @"pipeline": @{
311
- @"transcription": @{
312
- @"source_language": self.sourceLang
313
- },
314
- @"translations": @[@{
315
- @"target_language": self.targetLang,
316
- @"speech_generation": @{
317
- @"voice_cloning": @NO
318
- }
319
- }],
320
- @"allowed_message_types": @[
321
- @"partial_transcription",
322
- @"validated_transcription",
323
- @"translated_transcription"
324
- ]
325
- }
326
- }
327
- };
328
-
329
- NSError *error;
330
- NSData *jsonData = [NSJSONSerialization dataWithJSONObject:msg options:0 error:&error];
331
- if (error) {
332
- NSLog(@"palabra_set_task_error: %@", error);
333
- return;
334
- }
335
-
336
- NSString *payload = [[NSString alloc] initWithData:jsonData encoding:NSUTF8StringEncoding];
337
- NSLog(@"palabra_set_task: %@", payload);
338
-
339
- NSURLSessionWebSocketMessage *wsMsg = [[NSURLSessionWebSocketMessage alloc] initWithString:payload];
340
- [self.webSocket sendMessage:wsMsg completionHandler:^(NSError *error) {
341
- if (error) {
342
- NSLog(@"palabra_send_error: %@", error);
343
- }
344
- }];
345
- }
346
-
347
- - (void)handleMessage:(NSString *)text {
348
- NSError *error;
349
- NSDictionary *json = [NSJSONSerialization JSONObjectWithData:[text dataUsingEncoding:NSUTF8StringEncoding]
350
- options:0
351
- error:&error];
352
- if (error) {
353
- NSLog(@"palabra_parse_error: %@", error);
354
- return;
355
- }
356
-
357
- NSString *type = json[@"message_type"] ?: @"";
358
- NSLog(@"palabra_msg: %@", type);
359
-
360
- if ([type isEqualToString:@"output_audio_data"]) {
361
- NSDictionary *data = json[@"data"];
362
- NSString *audioBase64 = data[@"data"] ?: @"";
363
- if (audioBase64.length > 0) {
364
- NSData *audioData = [[NSData alloc] initWithBase64EncodedString:audioBase64 options:0];
365
- NSLog(@"palabra_audio_out: %lu bytes", (unsigned long)audioData.length);
366
- [self playAudio:audioData];
367
- }
368
- } else if ([type containsString:@"transcription"]) {
369
- NSDictionary *data = json[@"data"];
370
- NSDictionary *transcription = data[@"transcription"];
371
- if (transcription) {
372
- NSString *text = transcription[@"text"] ?: @"";
373
- NSString *lang = transcription[@"language"] ?: @"";
374
- BOOL isFinal = ![type isEqualToString:@"partial_transcription"];
375
- NSLog(@"palabra_transcription: %@ (%@)", text, lang);
376
-
377
- dispatch_async(dispatch_get_main_queue(), ^{
378
- [self notifyTranscription:text language:lang isFinal:isFinal];
379
- });
380
- }
381
- } else if ([type isEqualToString:@"error"]) {
382
- NSDictionary *data = json[@"data"];
383
- NSString *desc = data[@"desc"] ?: @"unknown";
384
- NSLog(@"palabra_error: %@", desc);
385
- NSError *err = [NSError errorWithDomain:@"PalabraClient" code:500 userInfo:@{NSLocalizedDescriptionKey: desc}];
386
- dispatch_async(dispatch_get_main_queue(), ^{
387
- [self notifyError:err];
388
- });
389
- } else if ([type isEqualToString:@"task_ready"]) {
390
- NSLog(@"palabra_task_ready");
391
- }
392
- }
393
-
394
- - (void)playAudio:(NSData *)audioData {
395
- if (!self.translating || !self.playerNode) return;
396
-
397
- AVAudioFrameCount frameCount = (AVAudioFrameCount)(audioData.length / 2);
398
- AVAudioPCMBuffer *buffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:self.audioFormat frameCapacity:frameCount];
399
- buffer.frameLength = frameCount;
400
-
401
- memcpy(buffer.int16ChannelData[0], audioData.bytes, audioData.length);
402
-
403
- [self.playerNode scheduleBuffer:buffer completionHandler:nil];
404
- }
405
-
406
- - (void)stop {
407
- if (!self.translating) return;
408
-
409
- self.translating = NO;
410
- self.connected = NO;
411
-
412
- if (self.remoteTrack) {
413
- [self.remoteTrack removeRenderer:self];
414
- self.remoteTrack.source.volume = self.originalVolume;
415
- }
416
-
417
- if (self.webSocket) {
418
- NSDictionary *endMsg = @{
419
- @"message_type": @"end_task",
420
- @"data": @{@"force": @NO}
421
- };
422
- NSData *jsonData = [NSJSONSerialization dataWithJSONObject:endMsg options:0 error:nil];
423
- NSString *payload = [[NSString alloc] initWithData:jsonData encoding:NSUTF8StringEncoding];
424
- NSURLSessionWebSocketMessage *msg = [[NSURLSessionWebSocketMessage alloc] initWithString:payload];
425
- [self.webSocket sendMessage:msg completionHandler:nil];
426
-
427
- [self.webSocket cancelWithCloseCode:NSURLSessionWebSocketCloseCodeNormalClosure reason:nil];
428
- self.webSocket = nil;
429
- }
430
-
431
- [self.playerNode stop];
432
- [self.audioEngine stop];
433
-
434
- [self.bufferLock lock];
435
- [self.audioBuffer setLength:0];
436
- [self.bufferLock unlock];
437
-
438
- self.remoteTrack = nil;
439
- [self notifyConnectionState:@"disconnected"];
440
- }
441
-
442
- - (void)setTranslatedVolume:(double)volume {
443
- if (self.playerNode) {
444
- self.playerNode.volume = volume;
445
- }
446
- }
447
-
448
- - (void)sendAudioToPalabra:(AVAudioPCMBuffer *)buffer {
449
- }
450
-
451
- #pragma mark - RTCAudioRenderer
452
-
453
- - (void)renderPCMBuffer:(AVAudioPCMBuffer *)buffer {
454
- if (!self.translating || !self.webSocket) {
455
- return;
456
- }
457
-
458
- static int logCounter = 0;
459
- if (logCounter++ % 100 == 0) {
460
- NSLog(@"palabra_render_pcm: frames=%u rate=%.0f channels=%u",
461
- (unsigned int)buffer.frameLength,
462
- buffer.format.sampleRate,
463
- (unsigned int)buffer.format.channelCount);
464
- }
465
-
466
- NSData *resampled = [self resampleBuffer:buffer toRate:kSampleRateIn channels:kChannels];
467
- if (resampled.length == 0) {
468
- NSLog(@"palabra_resample_empty");
469
- return;
470
- }
471
-
472
- [self.bufferLock lock];
473
- [self.audioBuffer appendData:resampled];
474
-
475
- while (self.audioBuffer.length >= kChunkBytes) {
476
- NSData *chunk = [self.audioBuffer subdataWithRange:NSMakeRange(0, kChunkBytes)];
477
- NSMutableData *remaining = [NSMutableData dataWithData:[self.audioBuffer subdataWithRange:NSMakeRange(kChunkBytes, self.audioBuffer.length - kChunkBytes)]];
478
- self.audioBuffer = remaining;
479
-
480
- [self sendAudioChunk:chunk];
481
- }
482
- [self.bufferLock unlock];
483
- }
484
-
485
- - (NSData *)resampleBuffer:(AVAudioPCMBuffer *)buffer toRate:(int)dstRate channels:(int)dstChannels {
486
- int srcRate = (int)buffer.format.sampleRate;
487
- int srcChannels = (int)buffer.format.channelCount;
488
- int srcSamples = (int)buffer.frameLength;
489
-
490
- if (srcSamples == 0) {
491
- return [NSData data];
492
- }
493
-
494
- AVAudioCommonFormat format = buffer.format.commonFormat;
495
-
496
- if (srcRate == dstRate && srcChannels == dstChannels && format == AVAudioPCMFormatInt16) {
497
- return [NSData dataWithBytes:buffer.int16ChannelData[0] length:srcSamples * 2];
498
- }
499
-
500
- float *srcData = NULL;
501
- float *tempFloat = NULL;
502
-
503
- if (format == AVAudioPCMFormatFloat32) {
504
- srcData = buffer.floatChannelData[0];
505
- } else if (format == AVAudioPCMFormatInt16) {
506
- tempFloat = malloc(srcSamples * sizeof(float));
507
- int16_t *srcInt16 = buffer.int16ChannelData[0];
508
- for (int i = 0; i < srcSamples; i++) {
509
- tempFloat[i] = (float)srcInt16[i] / 32768.0f;
510
- }
511
- srcData = tempFloat;
512
- } else {
513
- NSLog(@"palabra_unsupported_format: %ld", (long)format);
514
- return [NSData data];
515
- }
516
-
517
- float *monoSrc = srcData;
518
- float *monoBuffer = NULL;
519
- if (srcChannels == 2) {
520
- monoBuffer = malloc(srcSamples * sizeof(float));
521
- if (format == AVAudioPCMFormatFloat32 && buffer.floatChannelData[1]) {
522
- for (int i = 0; i < srcSamples; i++) {
523
- monoBuffer[i] = (buffer.floatChannelData[0][i] + buffer.floatChannelData[1][i]) / 2.0f;
524
- }
525
- } else if (format == AVAudioPCMFormatInt16 && buffer.int16ChannelData[1]) {
526
- for (int i = 0; i < srcSamples; i++) {
527
- float ch0 = (float)buffer.int16ChannelData[0][i] / 32768.0f;
528
- float ch1 = (float)buffer.int16ChannelData[1][i] / 32768.0f;
529
- monoBuffer[i] = (ch0 + ch1) / 2.0f;
530
- }
531
- } else {
532
- for (int i = 0; i < srcSamples; i++) {
533
- monoBuffer[i] = srcData[i];
534
- }
535
- }
536
- monoSrc = monoBuffer;
537
- }
538
-
539
- int dstSamples = (int)((long)srcSamples * dstRate / srcRate);
540
- if (dstSamples == 0) dstSamples = 1;
541
-
542
- int16_t *dstData = malloc(dstSamples * sizeof(int16_t));
543
-
544
- if (dstSamples == 1 || srcSamples == 1) {
545
- float sample = monoSrc[0];
546
- sample = fmaxf(-1.0f, fminf(1.0f, sample));
547
- dstData[0] = (int16_t)(sample * 32767.0f);
548
-
549
- if (monoBuffer) free(monoBuffer);
550
- if (tempFloat) free(tempFloat);
551
-
552
- NSData *result = [NSData dataWithBytes:dstData length:sizeof(int16_t)];
553
- free(dstData);
554
- return result;
555
- }
556
-
557
- for (int i = 0; i < dstSamples; i++) {
558
- float srcIdx = (float)i * (srcSamples - 1) / (dstSamples - 1);
559
- int idx0 = (int)srcIdx;
560
- int idx1 = MIN(idx0 + 1, srcSamples - 1);
561
- float frac = srcIdx - idx0;
562
- float sample = monoSrc[idx0] * (1 - frac) + monoSrc[idx1] * frac;
563
- sample = fmaxf(-1.0f, fminf(1.0f, sample));
564
- dstData[i] = (int16_t)(sample * 32767.0f);
565
- }
566
-
567
- if (monoBuffer) free(monoBuffer);
568
- if (tempFloat) free(tempFloat);
569
-
570
- NSData *result = [NSData dataWithBytes:dstData length:dstSamples * 2];
571
- free(dstData);
572
- return result;
573
- }
574
-
575
- - (void)sendAudioChunk:(NSData *)chunk {
576
- if (!self.webSocket || !self.connected) return;
577
-
578
- static int sendCounter = 0;
579
- if (sendCounter++ % 50 == 0) {
580
- NSLog(@"palabra_send_chunk: %lu bytes (count=%d)", (unsigned long)chunk.length, sendCounter);
581
- }
582
-
583
- NSString *base64 = [chunk base64EncodedStringWithOptions:0];
584
- NSDictionary *msg = @{
585
- @"message_type": @"input_audio_data",
586
- @"data": @{
587
- @"data": base64
588
- }
589
- };
590
-
591
- NSData *jsonData = [NSJSONSerialization dataWithJSONObject:msg options:0 error:nil];
592
- NSString *payload = [[NSString alloc] initWithData:jsonData encoding:NSUTF8StringEncoding];
593
-
594
- NSURLSessionWebSocketMessage *wsMsg = [[NSURLSessionWebSocketMessage alloc] initWithString:payload];
595
- [self.webSocket sendMessage:wsMsg completionHandler:^(NSError *error) {
596
- if (error) {
597
- NSLog(@"palabra_chunk_send_error: %@", error);
598
- }
599
- }];
600
- }
601
-
602
- #pragma mark - Notifications
603
-
604
- - (void)notifyConnectionState:(NSString *)state {
605
- if ([self.delegate respondsToSelector:@selector(palabraDidChangeConnectionState:)]) {
606
- [self.delegate palabraDidChangeConnectionState:state];
607
- }
608
- }
609
-
610
- - (void)notifyError:(NSError *)error {
611
- if ([self.delegate respondsToSelector:@selector(palabraDidFailWithError:)]) {
612
- [self.delegate palabraDidFailWithError:error];
613
- }
614
- }
615
-
616
- - (void)notifyTranscription:(NSString *)text language:(NSString *)lang isFinal:(BOOL)isFinal {
617
- if ([self.delegate respondsToSelector:@selector(palabraDidReceiveTranscription:)]) {
618
- [self.delegate palabraDidReceiveTranscription:@{
619
- @"text": text,
620
- @"lang": lang,
621
- @"isFinal": @(isFinal)
622
- }];
623
- }
624
- }
625
-
626
- #pragma mark - NSURLSessionWebSocketDelegate
627
-
628
- - (void)URLSession:(NSURLSession *)session webSocketTask:(NSURLSessionWebSocketTask *)webSocketTask didOpenWithProtocol:(NSString *)protocol {
629
- NSLog(@"palabra_ws_opened");
630
- dispatch_async(dispatch_get_main_queue(), ^{
631
- [self handleWebSocketOpened];
632
- });
633
- }
634
-
635
- - (void)URLSession:(NSURLSession *)session webSocketTask:(NSURLSessionWebSocketTask *)webSocketTask didCloseWithCode:(NSURLSessionWebSocketCloseCode)closeCode reason:(NSData *)reason {
636
- NSLog(@"palabra_ws_closed: %ld", (long)closeCode);
637
- dispatch_async(dispatch_get_main_queue(), ^{
638
- if (self.translating) {
639
- [self stop];
640
- }
641
- });
642
- }
643
-
644
- @end
1
+ #import "PalabraClient.h"
2
+ #import "WebRTCModule.h"
3
+ #import <AVFoundation/AVFoundation.h>
4
+
5
+ static const int kSampleRateIn = 16000;
6
+ static const int kSampleRateOut = 24000;
7
+ static const int kChannels = 1;
8
+ static const int kChunkMs = 320;
9
+ static const int kChunkSamples = kSampleRateIn * kChunkMs / 1000;
10
+ static const int kChunkBytes = kChunkSamples * 2;
11
+
12
+ @interface PalabraClient () <NSURLSessionWebSocketDelegate, RTCAudioRenderer>
13
+
14
+ @property (nonatomic, strong) NSString *clientId;
15
+ @property (nonatomic, strong) NSString *clientSecret;
16
+ @property (nonatomic, strong) NSString *apiUrl;
17
+ @property (nonatomic, strong) NSString *sourceLang;
18
+ @property (nonatomic, strong) NSString *targetLang;
19
+
20
+ @property (nonatomic, strong) RTCAudioTrack *remoteTrack;
21
+
22
+ @property (nonatomic, strong) NSString *sessionId;
23
+ @property (nonatomic, strong) NSString *wsUrl;
24
+ @property (nonatomic, strong) NSString *publisherToken;
25
+ @property (nonatomic, strong) NSString *remoteTrackId;
26
+
27
+ @property (nonatomic, strong) NSURLSession *urlSession;
28
+ @property (nonatomic, strong) NSURLSessionWebSocketTask *webSocket;
29
+
30
+ @property (nonatomic, strong) AVAudioEngine *audioEngine;
31
+ @property (nonatomic, strong) AVAudioPlayerNode *playerNode;
32
+ @property (nonatomic, strong) AVAudioFormat *audioFormat;
33
+
34
+ @property (nonatomic, assign) BOOL connected;
35
+ @property (nonatomic, assign) BOOL starting;
36
+ @property (nonatomic, assign) BOOL translating;
37
+ @property (nonatomic, assign) double originalVolume;
38
+
39
+ @property (nonatomic, strong) NSMutableData *audioBuffer;
40
+ @property (nonatomic, strong) NSLock *bufferLock;
41
+
42
+ @end
43
+
44
+ @implementation PalabraClient
45
+
46
+ - (instancetype)initWithClientId:(NSString *)clientId
47
+ clientSecret:(NSString *)clientSecret
48
+ apiUrl:(NSString *)apiUrl
49
+ module:(WebRTCModule *)module {
50
+ self = [super init];
51
+ if (self) {
52
+ _clientId = clientId;
53
+ _clientSecret = clientSecret;
54
+ _apiUrl = apiUrl;
55
+ _module = module;
56
+ _connected = NO;
57
+ _translating = NO;
58
+ _originalVolume = 1.0;
59
+ _audioBuffer = [NSMutableData new];
60
+ _bufferLock = [NSLock new];
61
+
62
+ NSURLSessionConfiguration *config = [NSURLSessionConfiguration defaultSessionConfiguration];
63
+ _urlSession = [NSURLSession sessionWithConfiguration:config delegate:self delegateQueue:nil];
64
+
65
+ [self setupAudioEngine];
66
+ }
67
+ return self;
68
+ }
69
+
70
+ - (BOOL)isConnected {
71
+ return _connected;
72
+ }
73
+
74
+ - (BOOL)isTranslating {
75
+ return _translating;
76
+ }
77
+
78
+ - (BOOL)isRunning {
79
+ return self.starting || self.connected || self.translating;
80
+ }
81
+
82
+ - (BOOL)matchesSessionWithTrackId:(NSString *)trackId
83
+ sourceLang:(NSString *)sourceLang
84
+ targetLang:(NSString *)targetLang
85
+ apiUrl:(NSString *)apiUrl {
86
+ return self.remoteTrackId != nil
87
+ && [self.remoteTrackId isEqualToString:trackId]
88
+ && [self.sourceLang isEqualToString:sourceLang]
89
+ && [self.targetLang isEqualToString:targetLang]
90
+ && [self.apiUrl isEqualToString:apiUrl];
91
+ }
92
+
93
+ - (void)setupAudioEngine {
94
+ self.audioEngine = [[AVAudioEngine alloc] init];
95
+ self.playerNode = [[AVAudioPlayerNode alloc] init];
96
+ [self.audioEngine attachNode:self.playerNode];
97
+
98
+ self.audioFormat = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16
99
+ sampleRate:kSampleRateOut
100
+ channels:kChannels
101
+ interleaved:YES];
102
+
103
+ [self.audioEngine connect:self.playerNode
104
+ to:self.audioEngine.mainMixerNode
105
+ format:self.audioFormat];
106
+ }
107
+
108
+ - (void)startWithTrack:(RTCAudioTrack *)remoteAudioTrack
109
+ sourceLang:(NSString *)sourceLang
110
+ targetLang:(NSString *)targetLang {
111
+
112
+ if (self.starting || self.translating) {
113
+ NSLog(@"palabra_start_skip_busy");
114
+ return;
115
+ }
116
+
117
+ self.starting = YES;
118
+ self.remoteTrack = remoteAudioTrack;
119
+ self.remoteTrackId = remoteAudioTrack.trackId;
120
+ self.sourceLang = sourceLang;
121
+ self.targetLang = targetLang;
122
+
123
+ self.originalVolume = remoteAudioTrack.source.volume;
124
+ remoteAudioTrack.source.volume = 0;
125
+ NSLog(@"palabra_start: %@", self.remoteTrackId);
126
+
127
+ [self notifyConnectionState:@"connecting"];
128
+
129
+ __weak typeof(self) weakSelf = self;
130
+ [self createSessionWithCompletion:^(NSDictionary *session, NSError *error) {
131
+ __strong typeof(weakSelf) strongSelf = weakSelf;
132
+ if (!strongSelf) return;
133
+
134
+ if (!strongSelf.starting) {
135
+ NSLog(@"palabra_start_cancelled");
136
+ return;
137
+ }
138
+
139
+ if (error) {
140
+ strongSelf.starting = NO;
141
+ [strongSelf notifyError:error];
142
+ strongSelf.remoteTrack.source.volume = strongSelf.originalVolume;
143
+ return;
144
+ }
145
+
146
+ NSDictionary *data = session[@"data"];
147
+ strongSelf.sessionId = data[@"id"];
148
+ strongSelf.wsUrl = data[@"ws_url"];
149
+ strongSelf.publisherToken = data[@"publisher"];
150
+
151
+ NSLog(@"palabra_ws_url: %@", strongSelf.wsUrl);
152
+
153
+ dispatch_async(dispatch_get_main_queue(), ^{
154
+ [strongSelf connectWebSocket];
155
+ });
156
+ }];
157
+ }
158
+
159
+ - (void)createSessionWithCompletion:(void (^)(NSDictionary *, NSError *))completion {
160
+ NSString *urlStr = [NSString stringWithFormat:@"%@/session-storage/session", self.apiUrl];
161
+ NSURL *url = [NSURL URLWithString:urlStr];
162
+
163
+ NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:url];
164
+ request.HTTPMethod = @"POST";
165
+ [request setValue:self.clientId forHTTPHeaderField:@"ClientId"];
166
+ [request setValue:self.clientSecret forHTTPHeaderField:@"ClientSecret"];
167
+ [request setValue:@"application/json" forHTTPHeaderField:@"Content-Type"];
168
+
169
+ NSDictionary *body = @{
170
+ @"data": @{
171
+ @"subscriber_count": @0,
172
+ @"publisher_can_subscribe": @YES
173
+ }
174
+ };
175
+
176
+ NSError *jsonError;
177
+ request.HTTPBody = [NSJSONSerialization dataWithJSONObject:body options:0 error:&jsonError];
178
+
179
+ if (jsonError) {
180
+ completion(nil, jsonError);
181
+ return;
182
+ }
183
+
184
+ NSURLSessionDataTask *task = [[NSURLSession sharedSession] dataTaskWithRequest:request
185
+ completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) {
186
+ if (error) {
187
+ dispatch_async(dispatch_get_main_queue(), ^{
188
+ completion(nil, error);
189
+ });
190
+ return;
191
+ }
192
+
193
+ NSHTTPURLResponse *httpResponse = (NSHTTPURLResponse *)response;
194
+ if (httpResponse.statusCode < 200 || httpResponse.statusCode >= 300) {
195
+ NSString *message = [self sessionErrorMessageForStatusCode:httpResponse.statusCode data:data];
196
+ NSError *httpError = [NSError errorWithDomain:@"PalabraClient"
197
+ code:httpResponse.statusCode
198
+ userInfo:@{NSLocalizedDescriptionKey: message}];
199
+ dispatch_async(dispatch_get_main_queue(), ^{
200
+ completion(nil, httpError);
201
+ });
202
+ return;
203
+ }
204
+
205
+ NSError *parseError;
206
+ NSDictionary *json = [NSJSONSerialization JSONObjectWithData:data options:0 error:&parseError];
207
+
208
+ dispatch_async(dispatch_get_main_queue(), ^{
209
+ completion(json, parseError);
210
+ });
211
+ }];
212
+
213
+ [task resume];
214
+ }
215
+
216
+ - (NSString *)sessionErrorMessageForStatusCode:(NSInteger)statusCode data:(NSData *)data {
217
+ NSString *fallback = [NSString stringWithFormat:@"session_http_error_%ld", (long)statusCode];
218
+ if (!data || data.length == 0) {
219
+ return fallback;
220
+ }
221
+
222
+ NSError *parseError;
223
+ id json = [NSJSONSerialization JSONObjectWithData:data options:0 error:&parseError];
224
+ if (parseError || ![json isKindOfClass:[NSDictionary class]]) {
225
+ return fallback;
226
+ }
227
+
228
+ NSArray *errors = ((NSDictionary *)json)[@"errors"];
229
+ if (![errors isKindOfClass:[NSArray class]] || errors.count == 0) {
230
+ return fallback;
231
+ }
232
+
233
+ NSDictionary *first = errors.firstObject;
234
+ if (![first isKindOfClass:[NSDictionary class]]) {
235
+ return fallback;
236
+ }
237
+
238
+ NSString *detail = first[@"detail"];
239
+ if (![detail isKindOfClass:[NSString class]] || detail.length == 0) {
240
+ return fallback;
241
+ }
242
+
243
+ return [NSString stringWithFormat:@"%@: %@", fallback, detail];
244
+ }
245
+
246
+ - (void)connectWebSocket {
247
+ if (!self.starting && !self.translating) {
248
+ NSLog(@"palabra_connect_skip");
249
+ return;
250
+ }
251
+
252
+ NSString *endpoint = [NSString stringWithFormat:@"%@?token=%@", self.wsUrl, self.publisherToken];
253
+ NSLog(@"palabra_connecting_ws: %@", endpoint);
254
+
255
+ NSURL *url = [NSURL URLWithString:endpoint];
256
+ self.webSocket = [self.urlSession webSocketTaskWithURL:url];
257
+ [self.webSocket resume];
258
+
259
+ [self receiveMessage];
260
+ }
261
+
262
+ - (void)handleWebSocketOpened {
263
+ if (self.connected || self.translating || !self.webSocket) {
264
+ return;
265
+ }
266
+
267
+ self.starting = NO;
268
+ self.connected = YES;
269
+ self.translating = YES;
270
+
271
+ [self.remoteTrack addRenderer:self];
272
+
273
+ NSError *audioError;
274
+ [self.audioEngine startAndReturnError:&audioError];
275
+ if (audioError) {
276
+ NSLog(@"palabra_audio_engine_error: %@", audioError);
277
+ [self.remoteTrack removeRenderer:self];
278
+ self.starting = NO;
279
+ self.translating = NO;
280
+ self.connected = NO;
281
+ [self.webSocket cancelWithCloseCode:NSURLSessionWebSocketCloseCodeInternalServerError reason:nil];
282
+ self.webSocket = nil;
283
+ [self notifyError:audioError];
284
+ return;
285
+ }
286
+
287
+ [self.playerNode play];
288
+ NSLog(@"palabra_audio_engine_started");
289
+
290
+ [self notifyConnectionState:@"connected"];
291
+
292
+ dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.5 * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{
293
+ [self sendSetTask];
294
+ });
295
+ }
296
+
297
+ - (void)receiveMessage {
298
+ if (!self.webSocket) return;
299
+
300
+ __weak typeof(self) weakSelf = self;
301
+ [self.webSocket receiveMessageWithCompletionHandler:^(NSURLSessionWebSocketMessage *message, NSError *error) {
302
+ __strong typeof(weakSelf) strongSelf = weakSelf;
303
+ if (!strongSelf) return;
304
+
305
+ if (error) {
306
+ NSLog(@"palabra_ws_error: %@", error);
307
+ dispatch_async(dispatch_get_main_queue(), ^{
308
+ [strongSelf stop];
309
+ [strongSelf notifyError:error];
310
+ });
311
+ return;
312
+ }
313
+
314
+ if (message.type == NSURLSessionWebSocketMessageTypeString) {
315
+ [strongSelf handleMessage:message.string];
316
+ }
317
+
318
+ [strongSelf receiveMessage];
319
+ }];
320
+ }
321
+
322
+ - (void)sendSetTask {
323
+ if (!self.webSocket || !self.connected) return;
324
+
325
+ NSDictionary *msg = @{
326
+ @"message_type": @"set_task",
327
+ @"data": @{
328
+ @"input_stream": @{
329
+ @"content_type": @"audio",
330
+ @"source": @{
331
+ @"type": @"ws",
332
+ @"format": @"pcm_s16le",
333
+ @"sample_rate": @(kSampleRateIn),
334
+ @"channels": @(kChannels)
335
+ }
336
+ },
337
+ @"output_stream": @{
338
+ @"content_type": @"audio",
339
+ @"target": @{
340
+ @"type": @"ws",
341
+ @"format": @"pcm_s16le"
342
+ }
343
+ },
344
+ @"pipeline": @{
345
+ @"transcription": @{
346
+ @"source_language": self.sourceLang
347
+ },
348
+ @"translations": @[@{
349
+ @"target_language": self.targetLang,
350
+ @"speech_generation": @{
351
+ @"voice_cloning": @NO
352
+ }
353
+ }],
354
+ @"allowed_message_types": @[
355
+ @"partial_transcription",
356
+ @"validated_transcription",
357
+ @"translated_transcription"
358
+ ]
359
+ }
360
+ }
361
+ };
362
+
363
+ NSError *error;
364
+ NSData *jsonData = [NSJSONSerialization dataWithJSONObject:msg options:0 error:&error];
365
+ if (error) {
366
+ NSLog(@"palabra_set_task_error: %@", error);
367
+ return;
368
+ }
369
+
370
+ NSString *payload = [[NSString alloc] initWithData:jsonData encoding:NSUTF8StringEncoding];
371
+ NSLog(@"palabra_set_task: %@", payload);
372
+
373
+ NSURLSessionWebSocketMessage *wsMsg = [[NSURLSessionWebSocketMessage alloc] initWithString:payload];
374
+ [self.webSocket sendMessage:wsMsg completionHandler:^(NSError *error) {
375
+ if (error) {
376
+ NSLog(@"palabra_send_error: %@", error);
377
+ }
378
+ }];
379
+ }
380
+
381
+ - (void)handleMessage:(NSString *)text {
382
+ NSError *error;
383
+ NSDictionary *json = [NSJSONSerialization JSONObjectWithData:[text dataUsingEncoding:NSUTF8StringEncoding]
384
+ options:0
385
+ error:&error];
386
+ if (error) {
387
+ NSLog(@"palabra_parse_error: %@", error);
388
+ return;
389
+ }
390
+
391
+ NSString *type = json[@"message_type"] ?: @"";
392
+ NSLog(@"palabra_msg: %@", type);
393
+
394
+ if ([type isEqualToString:@"output_audio_data"]) {
395
+ NSDictionary *data = json[@"data"];
396
+ NSString *audioBase64 = data[@"data"] ?: @"";
397
+ if (audioBase64.length > 0) {
398
+ NSData *audioData = [[NSData alloc] initWithBase64EncodedString:audioBase64 options:0];
399
+ NSLog(@"palabra_audio_out: %lu bytes", (unsigned long)audioData.length);
400
+ [self playAudio:audioData];
401
+ }
402
+ } else if ([type containsString:@"transcription"]) {
403
+ NSDictionary *data = json[@"data"];
404
+ NSDictionary *transcription = data[@"transcription"];
405
+ if (transcription) {
406
+ NSString *text = transcription[@"text"] ?: @"";
407
+ NSString *lang = transcription[@"language"] ?: @"";
408
+ BOOL isFinal = ![type isEqualToString:@"partial_transcription"];
409
+ NSLog(@"palabra_transcription: %@ (%@)", text, lang);
410
+
411
+ dispatch_async(dispatch_get_main_queue(), ^{
412
+ [self notifyTranscription:text language:lang isFinal:isFinal];
413
+ });
414
+ }
415
+ } else if ([type isEqualToString:@"error"]) {
416
+ NSDictionary *data = json[@"data"];
417
+ NSString *desc = data[@"desc"] ?: @"unknown";
418
+ NSLog(@"palabra_error: %@", desc);
419
+ NSError *err = [NSError errorWithDomain:@"PalabraClient" code:500 userInfo:@{NSLocalizedDescriptionKey: desc}];
420
+ dispatch_async(dispatch_get_main_queue(), ^{
421
+ [self notifyError:err];
422
+ });
423
+ } else if ([type isEqualToString:@"task_ready"]) {
424
+ NSLog(@"palabra_task_ready");
425
+ }
426
+ }
427
+
428
+ - (void)playAudio:(NSData *)audioData {
429
+ if (!self.translating || !self.playerNode) return;
430
+
431
+ AVAudioFrameCount frameCount = (AVAudioFrameCount)(audioData.length / 2);
432
+ AVAudioPCMBuffer *buffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:self.audioFormat frameCapacity:frameCount];
433
+ buffer.frameLength = frameCount;
434
+
435
+ memcpy(buffer.int16ChannelData[0], audioData.bytes, audioData.length);
436
+
437
+ [self.playerNode scheduleBuffer:buffer completionHandler:nil];
438
+ }
439
+
440
+ - (void)stop {
441
+ if (!self.starting && !self.translating && !self.connected && !self.remoteTrack && !self.webSocket) return;
442
+
443
+ self.starting = NO;
444
+ self.translating = NO;
445
+ self.connected = NO;
446
+
447
+ if (self.remoteTrack) {
448
+ [self.remoteTrack removeRenderer:self];
449
+ self.remoteTrack.source.volume = self.originalVolume;
450
+ }
451
+
452
+ if (self.webSocket) {
453
+ NSDictionary *endMsg = @{
454
+ @"message_type": @"end_task",
455
+ @"data": @{@"force": @NO}
456
+ };
457
+ NSData *jsonData = [NSJSONSerialization dataWithJSONObject:endMsg options:0 error:nil];
458
+ NSString *payload = [[NSString alloc] initWithData:jsonData encoding:NSUTF8StringEncoding];
459
+ NSURLSessionWebSocketMessage *msg = [[NSURLSessionWebSocketMessage alloc] initWithString:payload];
460
+ [self.webSocket sendMessage:msg completionHandler:nil];
461
+
462
+ [self.webSocket cancelWithCloseCode:NSURLSessionWebSocketCloseCodeNormalClosure reason:nil];
463
+ self.webSocket = nil;
464
+ }
465
+
466
+ [self.playerNode stop];
467
+ [self.audioEngine stop];
468
+
469
+ [self.bufferLock lock];
470
+ [self.audioBuffer setLength:0];
471
+ [self.bufferLock unlock];
472
+
473
+ self.remoteTrack = nil;
474
+ self.remoteTrackId = nil;
475
+ [self notifyConnectionState:@"disconnected"];
476
+ }
477
+
478
+ - (void)setTranslatedVolume:(double)volume {
479
+ if (self.playerNode) {
480
+ self.playerNode.volume = volume;
481
+ }
482
+ }
483
+
484
+ - (void)sendAudioToPalabra:(AVAudioPCMBuffer *)buffer {
485
+ }
486
+
487
+ #pragma mark - RTCAudioRenderer
488
+
489
+ - (void)renderPCMBuffer:(AVAudioPCMBuffer *)buffer {
490
+ if (!self.translating || !self.webSocket) {
491
+ return;
492
+ }
493
+
494
+ static int logCounter = 0;
495
+ if (logCounter++ % 100 == 0) {
496
+ NSLog(@"palabra_render_pcm: frames=%u rate=%.0f channels=%u",
497
+ (unsigned int)buffer.frameLength,
498
+ buffer.format.sampleRate,
499
+ (unsigned int)buffer.format.channelCount);
500
+ }
501
+
502
+ NSData *resampled = [self resampleBuffer:buffer toRate:kSampleRateIn channels:kChannels];
503
+ if (resampled.length == 0) {
504
+ NSLog(@"palabra_resample_empty");
505
+ return;
506
+ }
507
+
508
+ [self.bufferLock lock];
509
+ [self.audioBuffer appendData:resampled];
510
+
511
+ while (self.audioBuffer.length >= kChunkBytes) {
512
+ NSData *chunk = [self.audioBuffer subdataWithRange:NSMakeRange(0, kChunkBytes)];
513
+ NSMutableData *remaining = [NSMutableData dataWithData:[self.audioBuffer subdataWithRange:NSMakeRange(kChunkBytes, self.audioBuffer.length - kChunkBytes)]];
514
+ self.audioBuffer = remaining;
515
+
516
+ [self sendAudioChunk:chunk];
517
+ }
518
+ [self.bufferLock unlock];
519
+ }
520
+
521
+ - (NSData *)resampleBuffer:(AVAudioPCMBuffer *)buffer toRate:(int)dstRate channels:(int)dstChannels {
522
+ int srcRate = (int)buffer.format.sampleRate;
523
+ int srcChannels = (int)buffer.format.channelCount;
524
+ int srcSamples = (int)buffer.frameLength;
525
+
526
+ if (srcSamples == 0) {
527
+ return [NSData data];
528
+ }
529
+
530
+ AVAudioCommonFormat format = buffer.format.commonFormat;
531
+
532
+ if (srcRate == dstRate && srcChannels == dstChannels && format == AVAudioPCMFormatInt16) {
533
+ return [NSData dataWithBytes:buffer.int16ChannelData[0] length:srcSamples * 2];
534
+ }
535
+
536
+ float *srcData = NULL;
537
+ float *tempFloat = NULL;
538
+
539
+ if (format == AVAudioPCMFormatFloat32) {
540
+ srcData = buffer.floatChannelData[0];
541
+ } else if (format == AVAudioPCMFormatInt16) {
542
+ tempFloat = malloc(srcSamples * sizeof(float));
543
+ int16_t *srcInt16 = buffer.int16ChannelData[0];
544
+ for (int i = 0; i < srcSamples; i++) {
545
+ tempFloat[i] = (float)srcInt16[i] / 32768.0f;
546
+ }
547
+ srcData = tempFloat;
548
+ } else {
549
+ NSLog(@"palabra_unsupported_format: %ld", (long)format);
550
+ return [NSData data];
551
+ }
552
+
553
+ float *monoSrc = srcData;
554
+ float *monoBuffer = NULL;
555
+ if (srcChannels == 2) {
556
+ monoBuffer = malloc(srcSamples * sizeof(float));
557
+ if (format == AVAudioPCMFormatFloat32 && buffer.floatChannelData[1]) {
558
+ for (int i = 0; i < srcSamples; i++) {
559
+ monoBuffer[i] = (buffer.floatChannelData[0][i] + buffer.floatChannelData[1][i]) / 2.0f;
560
+ }
561
+ } else if (format == AVAudioPCMFormatInt16 && buffer.int16ChannelData[1]) {
562
+ for (int i = 0; i < srcSamples; i++) {
563
+ float ch0 = (float)buffer.int16ChannelData[0][i] / 32768.0f;
564
+ float ch1 = (float)buffer.int16ChannelData[1][i] / 32768.0f;
565
+ monoBuffer[i] = (ch0 + ch1) / 2.0f;
566
+ }
567
+ } else {
568
+ for (int i = 0; i < srcSamples; i++) {
569
+ monoBuffer[i] = srcData[i];
570
+ }
571
+ }
572
+ monoSrc = monoBuffer;
573
+ }
574
+
575
+ int dstSamples = (int)((long)srcSamples * dstRate / srcRate);
576
+ if (dstSamples == 0) dstSamples = 1;
577
+
578
+ int16_t *dstData = malloc(dstSamples * sizeof(int16_t));
579
+
580
+ if (dstSamples == 1 || srcSamples == 1) {
581
+ float sample = monoSrc[0];
582
+ sample = fmaxf(-1.0f, fminf(1.0f, sample));
583
+ dstData[0] = (int16_t)(sample * 32767.0f);
584
+
585
+ if (monoBuffer) free(monoBuffer);
586
+ if (tempFloat) free(tempFloat);
587
+
588
+ NSData *result = [NSData dataWithBytes:dstData length:sizeof(int16_t)];
589
+ free(dstData);
590
+ return result;
591
+ }
592
+
593
+ for (int i = 0; i < dstSamples; i++) {
594
+ float srcIdx = (float)i * (srcSamples - 1) / (dstSamples - 1);
595
+ int idx0 = (int)srcIdx;
596
+ int idx1 = MIN(idx0 + 1, srcSamples - 1);
597
+ float frac = srcIdx - idx0;
598
+ float sample = monoSrc[idx0] * (1 - frac) + monoSrc[idx1] * frac;
599
+ sample = fmaxf(-1.0f, fminf(1.0f, sample));
600
+ dstData[i] = (int16_t)(sample * 32767.0f);
601
+ }
602
+
603
+ if (monoBuffer) free(monoBuffer);
604
+ if (tempFloat) free(tempFloat);
605
+
606
+ NSData *result = [NSData dataWithBytes:dstData length:dstSamples * 2];
607
+ free(dstData);
608
+ return result;
609
+ }
610
+
611
+ - (void)sendAudioChunk:(NSData *)chunk {
612
+ if (!self.webSocket || !self.connected) return;
613
+
614
+ static int sendCounter = 0;
615
+ if (sendCounter++ % 50 == 0) {
616
+ NSLog(@"palabra_send_chunk: %lu bytes (count=%d)", (unsigned long)chunk.length, sendCounter);
617
+ }
618
+
619
+ NSString *base64 = [chunk base64EncodedStringWithOptions:0];
620
+ NSDictionary *msg = @{
621
+ @"message_type": @"input_audio_data",
622
+ @"data": @{
623
+ @"data": base64
624
+ }
625
+ };
626
+
627
+ NSData *jsonData = [NSJSONSerialization dataWithJSONObject:msg options:0 error:nil];
628
+ NSString *payload = [[NSString alloc] initWithData:jsonData encoding:NSUTF8StringEncoding];
629
+
630
+ NSURLSessionWebSocketMessage *wsMsg = [[NSURLSessionWebSocketMessage alloc] initWithString:payload];
631
+ [self.webSocket sendMessage:wsMsg completionHandler:^(NSError *error) {
632
+ if (error) {
633
+ NSLog(@"palabra_chunk_send_error: %@", error);
634
+ }
635
+ }];
636
+ }
637
+
638
+ #pragma mark - Notifications
639
+
640
+ - (void)notifyConnectionState:(NSString *)state {
641
+ if ([self.delegate respondsToSelector:@selector(palabraDidChangeConnectionState:)]) {
642
+ [self.delegate palabraDidChangeConnectionState:state];
643
+ }
644
+ }
645
+
646
+ - (void)notifyError:(NSError *)error {
647
+ if ([self.delegate respondsToSelector:@selector(palabraDidFailWithError:)]) {
648
+ [self.delegate palabraDidFailWithError:error];
649
+ }
650
+ }
651
+
652
+ - (void)notifyTranscription:(NSString *)text language:(NSString *)lang isFinal:(BOOL)isFinal {
653
+ if ([self.delegate respondsToSelector:@selector(palabraDidReceiveTranscription:)]) {
654
+ [self.delegate palabraDidReceiveTranscription:@{
655
+ @"text": text,
656
+ @"lang": lang,
657
+ @"isFinal": @(isFinal)
658
+ }];
659
+ }
660
+ }
661
+
662
+ #pragma mark - NSURLSessionWebSocketDelegate
663
+
664
+ - (void)URLSession:(NSURLSession *)session webSocketTask:(NSURLSessionWebSocketTask *)webSocketTask didOpenWithProtocol:(NSString *)protocol {
665
+ NSLog(@"palabra_ws_opened");
666
+ dispatch_async(dispatch_get_main_queue(), ^{
667
+ [self handleWebSocketOpened];
668
+ });
669
+ }
670
+
671
+ - (void)URLSession:(NSURLSession *)session webSocketTask:(NSURLSessionWebSocketTask *)webSocketTask didCloseWithCode:(NSURLSessionWebSocketCloseCode)closeCode reason:(NSData *)reason {
672
+ NSLog(@"palabra_ws_closed: %ld", (long)closeCode);
673
+ dispatch_async(dispatch_get_main_queue(), ^{
674
+ if (self.translating) {
675
+ [self stop];
676
+ }
677
+ });
678
+ }
679
+
680
+ @end