@sbhjt-gr/react-native-webrtc 124.0.2 → 124.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (363) hide show
  1. package/.clang-format +11 -11
  2. package/.eslintignore +6 -6
  3. package/.nvmrc +1 -1
  4. package/ISSUE_TEMPLATE.md +40 -40
  5. package/LICENSE +22 -22
  6. package/README.md +103 -103
  7. package/android/build.gradle +37 -37
  8. package/android/consumer-rules.pro +3 -3
  9. package/android/src/main/AndroidManifest.xml +11 -11
  10. package/android/src/main/java/com/oney/WebRTCModule/AbstractVideoCaptureController.java +113 -113
  11. package/android/src/main/java/com/oney/WebRTCModule/CameraCaptureController.java +338 -338
  12. package/android/src/main/java/com/oney/WebRTCModule/CameraEventsHandler.java +49 -49
  13. package/android/src/main/java/com/oney/WebRTCModule/DataChannelWrapper.java +99 -99
  14. package/android/src/main/java/com/oney/WebRTCModule/DataPacketCryptorManager.java +62 -62
  15. package/android/src/main/java/com/oney/WebRTCModule/DisplayUtils.java +16 -16
  16. package/android/src/main/java/com/oney/WebRTCModule/EglUtils.java +66 -66
  17. package/android/src/main/java/com/oney/WebRTCModule/GetUserMediaImpl.java +539 -539
  18. package/android/src/main/java/com/oney/WebRTCModule/LibraryLoader.java +21 -21
  19. package/android/src/main/java/com/oney/WebRTCModule/MediaProjectionNotification.java +70 -70
  20. package/android/src/main/java/com/oney/WebRTCModule/MediaProjectionService.java +82 -82
  21. package/android/src/main/java/com/oney/WebRTCModule/PeerConnectionObserver.java +588 -588
  22. package/android/src/main/java/com/oney/WebRTCModule/RTCCryptoManager.java +493 -493
  23. package/android/src/main/java/com/oney/WebRTCModule/RTCVideoViewManager.java +98 -98
  24. package/android/src/main/java/com/oney/WebRTCModule/ReactBridgeUtil.java +35 -35
  25. package/android/src/main/java/com/oney/WebRTCModule/ScreenCaptureController.java +94 -94
  26. package/android/src/main/java/com/oney/WebRTCModule/SerializeUtils.java +342 -342
  27. package/android/src/main/java/com/oney/WebRTCModule/StringUtils.java +100 -100
  28. package/android/src/main/java/com/oney/WebRTCModule/ThreadUtils.java +41 -41
  29. package/android/src/main/java/com/oney/WebRTCModule/TrackCapturerEventsEmitter.java +34 -34
  30. package/android/src/main/java/com/oney/WebRTCModule/VideoTrackAdapter.java +137 -137
  31. package/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java +1649 -1643
  32. package/android/src/main/java/com/oney/WebRTCModule/WebRTCModuleOptions.java +33 -33
  33. package/android/src/main/java/com/oney/WebRTCModule/WebRTCModulePackage.java +21 -21
  34. package/android/src/main/java/com/oney/WebRTCModule/WebRTCView.java +583 -583
  35. package/android/src/main/java/com/oney/WebRTCModule/palabra/PalabraClient.java +529 -464
  36. package/android/src/main/java/com/oney/WebRTCModule/palabra/PalabraConfig.java +17 -17
  37. package/android/src/main/java/com/oney/WebRTCModule/palabra/PalabraListener.java +7 -7
  38. package/android/src/main/java/com/oney/WebRTCModule/videoEffects/ProcessorProvider.java +38 -38
  39. package/android/src/main/java/com/oney/WebRTCModule/videoEffects/VideoEffectProcessor.java +59 -59
  40. package/android/src/main/java/com/oney/WebRTCModule/videoEffects/VideoFrameProcessor.java +19 -19
  41. package/android/src/main/java/com/oney/WebRTCModule/videoEffects/VideoFrameProcessorFactoryInterface.java +12 -12
  42. package/android/src/main/java/com/oney/WebRTCModule/webrtcutils/H264AndSoftwareVideoDecoderFactory.java +73 -73
  43. package/android/src/main/java/com/oney/WebRTCModule/webrtcutils/H264AndSoftwareVideoEncoderFactory.java +73 -73
  44. package/android/src/main/java/com/oney/WebRTCModule/webrtcutils/SoftwareVideoDecoderFactoryProxy.java +36 -36
  45. package/android/src/main/java/com/oney/WebRTCModule/webrtcutils/SoftwareVideoEncoderFactoryProxy.java +36 -36
  46. package/android/src/main/java/org/webrtc/Camera1Helper.java +54 -54
  47. package/android/src/main/java/org/webrtc/Camera2Helper.java +52 -52
  48. package/android/src/main/res/values/strings.xml +5 -5
  49. package/android/src/main/res/values/styles.xml +8 -8
  50. package/ios/RCTWebRTC/CaptureController.h +18 -18
  51. package/ios/RCTWebRTC/CaptureController.m +28 -28
  52. package/ios/RCTWebRTC/CapturerEventsDelegate.h +12 -12
  53. package/ios/RCTWebRTC/DataChannelWrapper.h +27 -27
  54. package/ios/RCTWebRTC/DataChannelWrapper.m +42 -42
  55. package/ios/RCTWebRTC/I420Converter.h +22 -22
  56. package/ios/RCTWebRTC/I420Converter.m +164 -164
  57. package/ios/RCTWebRTC/PIPController.h +24 -24
  58. package/ios/RCTWebRTC/PIPController.m +234 -234
  59. package/ios/RCTWebRTC/PalabraAudioSink.h +13 -13
  60. package/ios/RCTWebRTC/PalabraAudioSink.m +18 -18
  61. package/ios/RCTWebRTC/PalabraClient.h +42 -36
  62. package/ios/RCTWebRTC/PalabraClient.m +680 -584
  63. package/ios/RCTWebRTC/RCTConvert+WebRTC.h +16 -16
  64. package/ios/RCTWebRTC/RCTConvert+WebRTC.m +206 -206
  65. package/ios/RCTWebRTC/RTCMediaStreamTrack+React.h +10 -10
  66. package/ios/RCTWebRTC/RTCMediaStreamTrack+React.m +16 -16
  67. package/ios/RCTWebRTC/RTCVideoViewManager.h +29 -29
  68. package/ios/RCTWebRTC/RTCVideoViewManager.m +411 -411
  69. package/ios/RCTWebRTC/SampleBufferVideoCallView.h +12 -12
  70. package/ios/RCTWebRTC/SampleBufferVideoCallView.m +178 -178
  71. package/ios/RCTWebRTC/ScreenCaptureController.h +20 -20
  72. package/ios/RCTWebRTC/ScreenCaptureController.m +82 -82
  73. package/ios/RCTWebRTC/ScreenCapturePickerViewManager.h +7 -7
  74. package/ios/RCTWebRTC/ScreenCapturePickerViewManager.m +59 -59
  75. package/ios/RCTWebRTC/ScreenCapturer.h +19 -19
  76. package/ios/RCTWebRTC/ScreenCapturer.m +263 -263
  77. package/ios/RCTWebRTC/SerializeUtils.h +28 -28
  78. package/ios/RCTWebRTC/SerializeUtils.m +314 -314
  79. package/ios/RCTWebRTC/SocketConnection.h +13 -13
  80. package/ios/RCTWebRTC/SocketConnection.m +137 -137
  81. package/ios/RCTWebRTC/TrackCapturerEventsEmitter.h +14 -14
  82. package/ios/RCTWebRTC/TrackCapturerEventsEmitter.m +36 -36
  83. package/ios/RCTWebRTC/VideoCaptureController.h +21 -21
  84. package/ios/RCTWebRTC/VideoCaptureController.m +328 -328
  85. package/ios/RCTWebRTC/WLVAudioDevice.h +12 -12
  86. package/ios/RCTWebRTC/WLVAudioDevice.m +137 -137
  87. package/ios/RCTWebRTC/WebRTCModule+Palabra.h +4 -4
  88. package/ios/RCTWebRTC/WebRTCModule+Palabra.m +92 -83
  89. package/ios/RCTWebRTC/WebRTCModule+Permissions.m +75 -75
  90. package/ios/RCTWebRTC/WebRTCModule+RTCAudioSession.m +20 -20
  91. package/ios/RCTWebRTC/WebRTCModule+RTCDataChannel.h +14 -14
  92. package/ios/RCTWebRTC/WebRTCModule+RTCDataChannel.m +165 -165
  93. package/ios/RCTWebRTC/WebRTCModule+RTCFrameCryptor.m +611 -611
  94. package/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.h +13 -13
  95. package/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m +728 -728
  96. package/ios/RCTWebRTC/WebRTCModule+RTCPeerConnection.h +24 -24
  97. package/ios/RCTWebRTC/WebRTCModule+RTCPeerConnection.m +1004 -1004
  98. package/ios/RCTWebRTC/WebRTCModule+Transceivers.m +267 -267
  99. package/ios/RCTWebRTC/WebRTCModule+VideoTrackAdapter.h +12 -12
  100. package/ios/RCTWebRTC/WebRTCModule+VideoTrackAdapter.m +166 -166
  101. package/ios/RCTWebRTC/WebRTCModule.h +58 -58
  102. package/ios/RCTWebRTC/WebRTCModule.m +169 -169
  103. package/ios/RCTWebRTC/WebRTCModuleOptions.h +24 -24
  104. package/ios/RCTWebRTC/WebRTCModuleOptions.m +31 -31
  105. package/ios/RCTWebRTC/videoEffects/ProcessorProvider.h +9 -9
  106. package/ios/RCTWebRTC/videoEffects/ProcessorProvider.m +23 -23
  107. package/ios/RCTWebRTC/videoEffects/VideoEffectProcessor.h +13 -13
  108. package/ios/RCTWebRTC/videoEffects/VideoEffectProcessor.m +23 -23
  109. package/ios/RCTWebRTC/videoEffects/VideoFrameProcessor.h +8 -8
  110. package/ios/RCTWebRTC.xcodeproj/project.pbxproj +404 -404
  111. package/ios/RCTWebRTC.xcworkspace/contents.xcworkspacedata +10 -10
  112. package/lib/commonjs/Constraints.js.map +1 -1
  113. package/lib/commonjs/EventEmitter.js.map +1 -1
  114. package/lib/commonjs/Logger.js.map +1 -1
  115. package/lib/commonjs/MediaDevices.js +17 -17
  116. package/lib/commonjs/MediaDevices.js.map +1 -1
  117. package/lib/commonjs/MediaStream.js +19 -19
  118. package/lib/commonjs/MediaStream.js.map +1 -1
  119. package/lib/commonjs/MediaStreamError.js.map +1 -1
  120. package/lib/commonjs/MediaStreamErrorEvent.js.map +1 -1
  121. package/lib/commonjs/MediaStreamTrack.js +28 -28
  122. package/lib/commonjs/MediaStreamTrack.js.map +1 -1
  123. package/lib/commonjs/MediaStreamTrackEvent.js +6 -6
  124. package/lib/commonjs/MediaStreamTrackEvent.js.map +1 -1
  125. package/lib/commonjs/MessageEvent.js +7 -7
  126. package/lib/commonjs/MessageEvent.js.map +1 -1
  127. package/lib/commonjs/Permissions.js +28 -28
  128. package/lib/commonjs/Permissions.js.map +1 -1
  129. package/lib/commonjs/RTCAudioSession.js +4 -4
  130. package/lib/commonjs/RTCAudioSession.js.map +1 -1
  131. package/lib/commonjs/RTCDataChannel.js +2 -2
  132. package/lib/commonjs/RTCDataChannel.js.map +1 -1
  133. package/lib/commonjs/RTCDataChannelEvent.js +6 -6
  134. package/lib/commonjs/RTCDataChannelEvent.js.map +1 -1
  135. package/lib/commonjs/RTCDataPacketCryptor.js.map +1 -1
  136. package/lib/commonjs/RTCDataPacketCryptorFactory.js.map +1 -1
  137. package/lib/commonjs/RTCErrorEvent.js +3 -3
  138. package/lib/commonjs/RTCErrorEvent.js.map +1 -1
  139. package/lib/commonjs/RTCFrameCryptor.js +8 -8
  140. package/lib/commonjs/RTCFrameCryptor.js.map +1 -1
  141. package/lib/commonjs/RTCFrameCryptorFactory.js.map +1 -1
  142. package/lib/commonjs/RTCIceCandidate.js.map +1 -1
  143. package/lib/commonjs/RTCIceCandidateEvent.js +7 -7
  144. package/lib/commonjs/RTCIceCandidateEvent.js.map +1 -1
  145. package/lib/commonjs/RTCKeyProvider.js.map +1 -1
  146. package/lib/commonjs/RTCPIPView.js +2 -2
  147. package/lib/commonjs/RTCPIPView.js.map +1 -1
  148. package/lib/commonjs/RTCPIPView.web.js.map +1 -1
  149. package/lib/commonjs/RTCPeerConnection.js +146 -41
  150. package/lib/commonjs/RTCPeerConnection.js.map +1 -1
  151. package/lib/commonjs/RTCRtcpParameters.js.map +1 -1
  152. package/lib/commonjs/RTCRtpCapabilities.js +2 -2
  153. package/lib/commonjs/RTCRtpCapabilities.js.map +1 -1
  154. package/lib/commonjs/RTCRtpCodecCapability.js.map +1 -1
  155. package/lib/commonjs/RTCRtpCodecParameters.js.map +1 -1
  156. package/lib/commonjs/RTCRtpEncodingParameters.js.map +1 -1
  157. package/lib/commonjs/RTCRtpHeaderExtension.js.map +1 -1
  158. package/lib/commonjs/RTCRtpParameters.js.map +1 -1
  159. package/lib/commonjs/RTCRtpReceiveParameters.js.map +1 -1
  160. package/lib/commonjs/RTCRtpReceiver.js +7 -7
  161. package/lib/commonjs/RTCRtpReceiver.js.map +1 -1
  162. package/lib/commonjs/RTCRtpSendParameters.js +3 -3
  163. package/lib/commonjs/RTCRtpSendParameters.js.map +1 -1
  164. package/lib/commonjs/RTCRtpSender.js +7 -7
  165. package/lib/commonjs/RTCRtpSender.js.map +1 -1
  166. package/lib/commonjs/RTCRtpTransceiver.js.map +1 -1
  167. package/lib/commonjs/RTCSessionDescription.js.map +1 -1
  168. package/lib/commonjs/RTCTrackEvent.js +6 -6
  169. package/lib/commonjs/RTCTrackEvent.js.map +1 -1
  170. package/lib/commonjs/RTCUtil.js +28 -28
  171. package/lib/commonjs/RTCUtil.js.map +1 -1
  172. package/lib/commonjs/RTCView.js +5 -5
  173. package/lib/commonjs/RTCView.js.map +1 -1
  174. package/lib/commonjs/RTCView.web.js.map +1 -1
  175. package/lib/commonjs/ScreenCapturePickerView.js.map +1 -1
  176. package/lib/commonjs/ScreenCapturePickerView.web.js.map +1 -1
  177. package/lib/commonjs/getDisplayMedia.js.map +1 -1
  178. package/lib/commonjs/getUserMedia.js +23 -0
  179. package/lib/commonjs/getUserMedia.js.map +1 -1
  180. package/lib/commonjs/index.js.map +1 -1
  181. package/lib/commonjs/index.web.js.map +1 -1
  182. package/lib/commonjs/webStream.js.map +1 -1
  183. package/lib/module/Constraints.js.map +1 -1
  184. package/lib/module/EventEmitter.js.map +1 -1
  185. package/lib/module/Logger.js.map +1 -1
  186. package/lib/module/MediaDevices.js +17 -17
  187. package/lib/module/MediaDevices.js.map +1 -1
  188. package/lib/module/MediaStream.js +19 -19
  189. package/lib/module/MediaStream.js.map +1 -1
  190. package/lib/module/MediaStreamError.js.map +1 -1
  191. package/lib/module/MediaStreamErrorEvent.js.map +1 -1
  192. package/lib/module/MediaStreamTrack.js +28 -28
  193. package/lib/module/MediaStreamTrack.js.map +1 -1
  194. package/lib/module/MediaStreamTrackEvent.js +6 -6
  195. package/lib/module/MediaStreamTrackEvent.js.map +1 -1
  196. package/lib/module/MessageEvent.js +7 -7
  197. package/lib/module/MessageEvent.js.map +1 -1
  198. package/lib/module/Permissions.js +28 -28
  199. package/lib/module/Permissions.js.map +1 -1
  200. package/lib/module/RTCAudioSession.js +4 -4
  201. package/lib/module/RTCAudioSession.js.map +1 -1
  202. package/lib/module/RTCDataChannel.js +2 -2
  203. package/lib/module/RTCDataChannel.js.map +1 -1
  204. package/lib/module/RTCDataChannelEvent.js +6 -6
  205. package/lib/module/RTCDataChannelEvent.js.map +1 -1
  206. package/lib/module/RTCDataPacketCryptor.js.map +1 -1
  207. package/lib/module/RTCDataPacketCryptorFactory.js.map +1 -1
  208. package/lib/module/RTCErrorEvent.js +3 -3
  209. package/lib/module/RTCErrorEvent.js.map +1 -1
  210. package/lib/module/RTCFrameCryptor.js +8 -8
  211. package/lib/module/RTCFrameCryptor.js.map +1 -1
  212. package/lib/module/RTCFrameCryptorFactory.js.map +1 -1
  213. package/lib/module/RTCIceCandidate.js.map +1 -1
  214. package/lib/module/RTCIceCandidateEvent.js +7 -7
  215. package/lib/module/RTCIceCandidateEvent.js.map +1 -1
  216. package/lib/module/RTCKeyProvider.js.map +1 -1
  217. package/lib/module/RTCPIPView.js +2 -2
  218. package/lib/module/RTCPIPView.js.map +1 -1
  219. package/lib/module/RTCPIPView.web.js.map +1 -1
  220. package/lib/module/RTCPeerConnection.js +146 -41
  221. package/lib/module/RTCPeerConnection.js.map +1 -1
  222. package/lib/module/RTCRtcpParameters.js.map +1 -1
  223. package/lib/module/RTCRtpCapabilities.js +2 -2
  224. package/lib/module/RTCRtpCapabilities.js.map +1 -1
  225. package/lib/module/RTCRtpCodecCapability.js.map +1 -1
  226. package/lib/module/RTCRtpCodecParameters.js.map +1 -1
  227. package/lib/module/RTCRtpEncodingParameters.js.map +1 -1
  228. package/lib/module/RTCRtpHeaderExtension.js.map +1 -1
  229. package/lib/module/RTCRtpParameters.js.map +1 -1
  230. package/lib/module/RTCRtpReceiveParameters.js.map +1 -1
  231. package/lib/module/RTCRtpReceiver.js +7 -7
  232. package/lib/module/RTCRtpReceiver.js.map +1 -1
  233. package/lib/module/RTCRtpSendParameters.js +3 -3
  234. package/lib/module/RTCRtpSendParameters.js.map +1 -1
  235. package/lib/module/RTCRtpSender.js +7 -7
  236. package/lib/module/RTCRtpSender.js.map +1 -1
  237. package/lib/module/RTCRtpTransceiver.js.map +1 -1
  238. package/lib/module/RTCSessionDescription.js.map +1 -1
  239. package/lib/module/RTCTrackEvent.js +6 -6
  240. package/lib/module/RTCTrackEvent.js.map +1 -1
  241. package/lib/module/RTCUtil.js +28 -28
  242. package/lib/module/RTCUtil.js.map +1 -1
  243. package/lib/module/RTCView.js +5 -5
  244. package/lib/module/RTCView.js.map +1 -1
  245. package/lib/module/RTCView.web.js.map +1 -1
  246. package/lib/module/ScreenCapturePickerView.js.map +1 -1
  247. package/lib/module/ScreenCapturePickerView.web.js.map +1 -1
  248. package/lib/module/getDisplayMedia.js.map +1 -1
  249. package/lib/module/getUserMedia.js +23 -0
  250. package/lib/module/getUserMedia.js.map +1 -1
  251. package/lib/module/index.js.map +1 -1
  252. package/lib/module/index.web.js.map +1 -1
  253. package/lib/module/webStream.js.map +1 -1
  254. package/lib/typescript/Constraints.d.ts +19 -19
  255. package/lib/typescript/EventEmitter.d.ts +6 -6
  256. package/lib/typescript/Logger.d.ts +13 -13
  257. package/lib/typescript/MediaDevices.d.ts +30 -30
  258. package/lib/typescript/MediaStream.d.ts +48 -48
  259. package/lib/typescript/MediaStreamError.d.ts +6 -6
  260. package/lib/typescript/MediaStreamErrorEvent.d.ts +6 -6
  261. package/lib/typescript/MediaStreamTrack.d.ts +101 -101
  262. package/lib/typescript/MediaStreamTrackEvent.d.ts +19 -19
  263. package/lib/typescript/MessageEvent.d.ts +20 -20
  264. package/lib/typescript/Permissions.d.ts +55 -55
  265. package/lib/typescript/RTCAudioSession.d.ts +10 -10
  266. package/lib/typescript/RTCDataChannel.d.ts +43 -43
  267. package/lib/typescript/RTCDataChannelEvent.d.ts +19 -19
  268. package/lib/typescript/RTCDataPacketCryptor.d.ts +12 -12
  269. package/lib/typescript/RTCDataPacketCryptorFactory.d.ts +6 -6
  270. package/lib/typescript/RTCErrorEvent.d.ts +12 -12
  271. package/lib/typescript/RTCFrameCryptor.d.ts +47 -47
  272. package/lib/typescript/RTCFrameCryptorFactory.d.ts +21 -21
  273. package/lib/typescript/RTCIceCandidate.d.ts +17 -17
  274. package/lib/typescript/RTCIceCandidateEvent.d.ts +20 -20
  275. package/lib/typescript/RTCKeyProvider.d.ts +21 -21
  276. package/lib/typescript/RTCPIPView.d.ts +15 -15
  277. package/lib/typescript/RTCPIPView.web.d.ts +13 -13
  278. package/lib/typescript/RTCPeerConnection.d.ts +117 -117
  279. package/lib/typescript/RTCRtcpParameters.d.ts +10 -10
  280. package/lib/typescript/RTCRtpCapabilities.d.ts +9 -9
  281. package/lib/typescript/RTCRtpCodecCapability.d.ts +7 -7
  282. package/lib/typescript/RTCRtpCodecParameters.d.ts +16 -16
  283. package/lib/typescript/RTCRtpEncodingParameters.d.ts +23 -23
  284. package/lib/typescript/RTCRtpHeaderExtension.d.ts +12 -12
  285. package/lib/typescript/RTCRtpParameters.d.ts +19 -19
  286. package/lib/typescript/RTCRtpReceiveParameters.d.ts +4 -4
  287. package/lib/typescript/RTCRtpReceiver.d.ts +21 -21
  288. package/lib/typescript/RTCRtpSendParameters.d.ts +20 -20
  289. package/lib/typescript/RTCRtpSender.d.ts +22 -22
  290. package/lib/typescript/RTCRtpTransceiver.d.ts +31 -31
  291. package/lib/typescript/RTCSessionDescription.d.ts +12 -12
  292. package/lib/typescript/RTCTrackEvent.d.ts +29 -29
  293. package/lib/typescript/RTCUtil.d.ts +37 -37
  294. package/lib/typescript/RTCView.d.ts +117 -117
  295. package/lib/typescript/RTCView.web.d.ts +25 -25
  296. package/lib/typescript/ScreenCapturePickerView.d.ts +2 -2
  297. package/lib/typescript/ScreenCapturePickerView.web.d.ts +1 -1
  298. package/lib/typescript/getDisplayMedia.d.ts +2 -2
  299. package/lib/typescript/getUserMedia.d.ts +7 -7
  300. package/lib/typescript/index.d.ts +22 -22
  301. package/lib/typescript/index.web.d.ts +101 -101
  302. package/lib/typescript/webStream.d.ts +3 -3
  303. package/livekit-react-native-webrtc.podspec +29 -29
  304. package/macos/RCTWebRTC.xcodeproj/project.pbxproj +324 -324
  305. package/macos/RCTWebRTC.xcodeproj/project.xcworkspace/contents.xcworkspacedata +7 -7
  306. package/macos/RCTWebRTC.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist +8 -8
  307. package/metro.config.js +7 -7
  308. package/metro.config.macos.js +14 -14
  309. package/package.json +66 -66
  310. package/react-native.config.js +11 -11
  311. package/src/.eslintrc.cjs +67 -67
  312. package/src/Constraints.ts +20 -20
  313. package/src/EventEmitter.ts +65 -65
  314. package/src/Logger.ts +49 -49
  315. package/src/MediaDevices.ts +53 -53
  316. package/src/MediaStream.ts +161 -161
  317. package/src/MediaStreamError.ts +12 -12
  318. package/src/MediaStreamErrorEvent.ts +11 -11
  319. package/src/MediaStreamTrack.ts +282 -282
  320. package/src/MediaStreamTrackEvent.ts +25 -25
  321. package/src/MessageEvent.ts +26 -26
  322. package/src/Permissions.ts +133 -133
  323. package/src/RTCAudioSession.ts +25 -25
  324. package/src/RTCDataChannel.ts +190 -190
  325. package/src/RTCDataChannelEvent.ts +28 -28
  326. package/src/RTCDataPacketCryptor.ts +90 -90
  327. package/src/RTCDataPacketCryptorFactory.ts +24 -24
  328. package/src/RTCErrorEvent.ts +20 -20
  329. package/src/RTCFrameCryptor.ts +162 -162
  330. package/src/RTCFrameCryptorFactory.ts +101 -101
  331. package/src/RTCIceCandidate.ts +29 -29
  332. package/src/RTCIceCandidateEvent.ts +26 -26
  333. package/src/RTCKeyProvider.ts +117 -117
  334. package/src/RTCPIPView.tsx +46 -46
  335. package/src/RTCPIPView.web.tsx +18 -18
  336. package/src/RTCPeerConnection.ts +935 -832
  337. package/src/RTCRtcpParameters.ts +23 -23
  338. package/src/RTCRtpCapabilities.ts +16 -16
  339. package/src/RTCRtpCodecCapability.ts +12 -12
  340. package/src/RTCRtpCodecParameters.ts +44 -44
  341. package/src/RTCRtpEncodingParameters.ts +90 -90
  342. package/src/RTCRtpHeaderExtension.ts +27 -27
  343. package/src/RTCRtpParameters.ts +37 -37
  344. package/src/RTCRtpReceiveParameters.ts +7 -7
  345. package/src/RTCRtpReceiver.ts +60 -60
  346. package/src/RTCRtpSendParameters.ts +63 -63
  347. package/src/RTCRtpSender.ts +78 -78
  348. package/src/RTCRtpTransceiver.ts +107 -107
  349. package/src/RTCSessionDescription.ts +30 -30
  350. package/src/RTCTrackEvent.ts +42 -42
  351. package/src/RTCUtil.ts +211 -211
  352. package/src/RTCView.ts +122 -122
  353. package/src/RTCView.web.tsx +80 -80
  354. package/src/ScreenCapturePickerView.ts +4 -4
  355. package/src/ScreenCapturePickerView.web.tsx +3 -3
  356. package/src/getDisplayMedia.ts +30 -30
  357. package/src/getUserMedia.ts +136 -111
  358. package/src/index.ts +107 -107
  359. package/src/index.web.ts +191 -191
  360. package/src/webStream.ts +31 -31
  361. package/tools/format.sh +6 -6
  362. package/tools/release.sh +45 -45
  363. package/tsconfig.json +17 -17
@@ -1,584 +1,680 @@
1
- #import "PalabraClient.h"
2
- #import "WebRTCModule.h"
3
- #import <AVFoundation/AVFoundation.h>
4
-
5
- static const int kSampleRateIn = 16000;
6
- static const int kSampleRateOut = 24000;
7
- static const int kChannels = 1;
8
- static const int kChunkMs = 320;
9
- static const int kChunkSamples = kSampleRateIn * kChunkMs / 1000;
10
- static const int kChunkBytes = kChunkSamples * 2;
11
-
12
- @interface PalabraClient () <NSURLSessionWebSocketDelegate, RTCAudioRenderer>
13
-
14
- @property (nonatomic, strong) NSString *clientId;
15
- @property (nonatomic, strong) NSString *clientSecret;
16
- @property (nonatomic, strong) NSString *apiUrl;
17
- @property (nonatomic, strong) NSString *sourceLang;
18
- @property (nonatomic, strong) NSString *targetLang;
19
-
20
- @property (nonatomic, strong) RTCAudioTrack *remoteTrack;
21
-
22
- @property (nonatomic, strong) NSString *sessionId;
23
- @property (nonatomic, strong) NSString *wsUrl;
24
- @property (nonatomic, strong) NSString *publisherToken;
25
-
26
- @property (nonatomic, strong) NSURLSession *urlSession;
27
- @property (nonatomic, strong) NSURLSessionWebSocketTask *webSocket;
28
-
29
- @property (nonatomic, strong) AVAudioEngine *audioEngine;
30
- @property (nonatomic, strong) AVAudioPlayerNode *playerNode;
31
- @property (nonatomic, strong) AVAudioFormat *audioFormat;
32
-
33
- @property (nonatomic, assign) BOOL connected;
34
- @property (nonatomic, assign) BOOL translating;
35
- @property (nonatomic, assign) double originalVolume;
36
-
37
- @property (nonatomic, strong) NSMutableData *audioBuffer;
38
- @property (nonatomic, strong) NSLock *bufferLock;
39
-
40
- @end
41
-
42
- @implementation PalabraClient
43
-
44
- - (instancetype)initWithClientId:(NSString *)clientId
45
- clientSecret:(NSString *)clientSecret
46
- apiUrl:(NSString *)apiUrl
47
- module:(WebRTCModule *)module {
48
- self = [super init];
49
- if (self) {
50
- _clientId = clientId;
51
- _clientSecret = clientSecret;
52
- _apiUrl = apiUrl;
53
- _module = module;
54
- _connected = NO;
55
- _translating = NO;
56
- _originalVolume = 1.0;
57
- _audioBuffer = [NSMutableData new];
58
- _bufferLock = [NSLock new];
59
-
60
- NSURLSessionConfiguration *config = [NSURLSessionConfiguration defaultSessionConfiguration];
61
- _urlSession = [NSURLSession sessionWithConfiguration:config delegate:self delegateQueue:nil];
62
-
63
- [self setupAudioEngine];
64
- }
65
- return self;
66
- }
67
-
68
- - (BOOL)isConnected {
69
- return _connected;
70
- }
71
-
72
- - (BOOL)isTranslating {
73
- return _translating;
74
- }
75
-
76
- - (void)setupAudioEngine {
77
- self.audioEngine = [[AVAudioEngine alloc] init];
78
- self.playerNode = [[AVAudioPlayerNode alloc] init];
79
- [self.audioEngine attachNode:self.playerNode];
80
-
81
- self.audioFormat = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16
82
- sampleRate:kSampleRateOut
83
- channels:kChannels
84
- interleaved:YES];
85
-
86
- [self.audioEngine connect:self.playerNode
87
- to:self.audioEngine.mainMixerNode
88
- format:self.audioFormat];
89
- }
90
-
91
- - (void)startWithTrack:(RTCAudioTrack *)remoteAudioTrack
92
- sourceLang:(NSString *)sourceLang
93
- targetLang:(NSString *)targetLang {
94
-
95
- if (self.translating) {
96
- return;
97
- }
98
-
99
- self.remoteTrack = remoteAudioTrack;
100
- self.sourceLang = sourceLang;
101
- self.targetLang = targetLang;
102
-
103
- self.originalVolume = remoteAudioTrack.source.volume;
104
- remoteAudioTrack.source.volume = 0;
105
-
106
- [self notifyConnectionState:@"connecting"];
107
-
108
- __weak typeof(self) weakSelf = self;
109
- [self createSessionWithCompletion:^(NSDictionary *session, NSError *error) {
110
- __strong typeof(weakSelf) strongSelf = weakSelf;
111
- if (!strongSelf) return;
112
-
113
- if (error) {
114
- [strongSelf notifyError:error];
115
- strongSelf.remoteTrack.source.volume = strongSelf.originalVolume;
116
- return;
117
- }
118
-
119
- NSDictionary *data = session[@"data"];
120
- strongSelf.sessionId = data[@"id"];
121
- strongSelf.wsUrl = data[@"ws_url"];
122
- strongSelf.publisherToken = data[@"publisher"];
123
-
124
- NSLog(@"palabra_ws_url: %@", strongSelf.wsUrl);
125
-
126
- dispatch_async(dispatch_get_main_queue(), ^{
127
- [strongSelf connectWebSocket];
128
- });
129
- }];
130
- }
131
-
132
- - (void)createSessionWithCompletion:(void (^)(NSDictionary *, NSError *))completion {
133
- NSString *urlStr = [NSString stringWithFormat:@"%@/session-storage/session", self.apiUrl];
134
- NSURL *url = [NSURL URLWithString:urlStr];
135
-
136
- NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:url];
137
- request.HTTPMethod = @"POST";
138
- [request setValue:self.clientId forHTTPHeaderField:@"ClientId"];
139
- [request setValue:self.clientSecret forHTTPHeaderField:@"ClientSecret"];
140
- [request setValue:@"application/json" forHTTPHeaderField:@"Content-Type"];
141
-
142
- NSDictionary *body = @{
143
- @"data": @{
144
- @"subscriber_count": @0,
145
- @"publisher_can_subscribe": @YES
146
- }
147
- };
148
-
149
- NSError *jsonError;
150
- request.HTTPBody = [NSJSONSerialization dataWithJSONObject:body options:0 error:&jsonError];
151
-
152
- if (jsonError) {
153
- completion(nil, jsonError);
154
- return;
155
- }
156
-
157
- NSURLSessionDataTask *task = [[NSURLSession sharedSession] dataTaskWithRequest:request
158
- completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) {
159
- if (error) {
160
- dispatch_async(dispatch_get_main_queue(), ^{
161
- completion(nil, error);
162
- });
163
- return;
164
- }
165
-
166
- NSHTTPURLResponse *httpResponse = (NSHTTPURLResponse *)response;
167
- if (httpResponse.statusCode < 200 || httpResponse.statusCode >= 300) {
168
- NSError *httpError = [NSError errorWithDomain:@"PalabraClient"
169
- code:httpResponse.statusCode
170
- userInfo:@{NSLocalizedDescriptionKey: [NSString stringWithFormat:@"session_http_error_%ld", (long)httpResponse.statusCode]}];
171
- dispatch_async(dispatch_get_main_queue(), ^{
172
- completion(nil, httpError);
173
- });
174
- return;
175
- }
176
-
177
- NSError *parseError;
178
- NSDictionary *json = [NSJSONSerialization JSONObjectWithData:data options:0 error:&parseError];
179
-
180
- dispatch_async(dispatch_get_main_queue(), ^{
181
- completion(json, parseError);
182
- });
183
- }];
184
-
185
- [task resume];
186
- }
187
-
188
- - (void)connectWebSocket {
189
- NSString *endpoint = [NSString stringWithFormat:@"%@?token=%@", self.wsUrl, self.publisherToken];
190
- NSLog(@"palabra_connecting_ws: %@", endpoint);
191
-
192
- NSURL *url = [NSURL URLWithString:endpoint];
193
- self.webSocket = [self.urlSession webSocketTaskWithURL:url];
194
- [self.webSocket resume];
195
-
196
- [self receiveMessage];
197
-
198
- self.connected = YES;
199
- self.translating = YES;
200
-
201
- [self.remoteTrack addRenderer:self];
202
-
203
- NSError *audioError;
204
- [self.audioEngine startAndReturnError:&audioError];
205
- if (audioError) {
206
- NSLog(@"palabra_audio_engine_error: %@", audioError);
207
- } else {
208
- [self.playerNode play];
209
- NSLog(@"palabra_audio_engine_started");
210
- }
211
-
212
- [self notifyConnectionState:@"connected"];
213
-
214
- dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.5 * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{
215
- [self sendSetTask];
216
- });
217
- }
218
-
219
- - (void)receiveMessage {
220
- if (!self.webSocket) return;
221
-
222
- __weak typeof(self) weakSelf = self;
223
- [self.webSocket receiveMessageWithCompletionHandler:^(NSURLSessionWebSocketMessage *message, NSError *error) {
224
- __strong typeof(weakSelf) strongSelf = weakSelf;
225
- if (!strongSelf) return;
226
-
227
- if (error) {
228
- NSLog(@"palabra_ws_error: %@", error);
229
- dispatch_async(dispatch_get_main_queue(), ^{
230
- [strongSelf stop];
231
- [strongSelf notifyError:error];
232
- });
233
- return;
234
- }
235
-
236
- if (message.type == NSURLSessionWebSocketMessageTypeString) {
237
- [strongSelf handleMessage:message.string];
238
- }
239
-
240
- [strongSelf receiveMessage];
241
- }];
242
- }
243
-
244
- - (void)sendSetTask {
245
- if (!self.webSocket || !self.connected) return;
246
-
247
- NSDictionary *msg = @{
248
- @"message_type": @"set_task",
249
- @"data": @{
250
- @"input_stream": @{
251
- @"content_type": @"audio",
252
- @"source": @{
253
- @"type": @"ws",
254
- @"format": @"pcm_s16le",
255
- @"sample_rate": @(kSampleRateIn),
256
- @"channels": @(kChannels)
257
- }
258
- },
259
- @"output_stream": @{
260
- @"content_type": @"audio",
261
- @"target": @{
262
- @"type": @"ws",
263
- @"format": @"pcm_s16le"
264
- }
265
- },
266
- @"pipeline": @{
267
- @"transcription": @{
268
- @"source_language": self.sourceLang
269
- },
270
- @"translations": @[@{
271
- @"target_language": self.targetLang,
272
- @"speech_generation": @{
273
- @"voice_cloning": @NO
274
- }
275
- }],
276
- @"allowed_message_types": @[
277
- @"partial_transcription",
278
- @"validated_transcription",
279
- @"translated_transcription"
280
- ]
281
- }
282
- }
283
- };
284
-
285
- NSError *error;
286
- NSData *jsonData = [NSJSONSerialization dataWithJSONObject:msg options:0 error:&error];
287
- if (error) {
288
- NSLog(@"palabra_set_task_error: %@", error);
289
- return;
290
- }
291
-
292
- NSString *payload = [[NSString alloc] initWithData:jsonData encoding:NSUTF8StringEncoding];
293
- NSLog(@"palabra_set_task: %@", payload);
294
-
295
- NSURLSessionWebSocketMessage *wsMsg = [[NSURLSessionWebSocketMessage alloc] initWithString:payload];
296
- [self.webSocket sendMessage:wsMsg completionHandler:^(NSError *error) {
297
- if (error) {
298
- NSLog(@"palabra_send_error: %@", error);
299
- }
300
- }];
301
- }
302
-
303
- - (void)handleMessage:(NSString *)text {
304
- NSError *error;
305
- NSDictionary *json = [NSJSONSerialization JSONObjectWithData:[text dataUsingEncoding:NSUTF8StringEncoding]
306
- options:0
307
- error:&error];
308
- if (error) {
309
- NSLog(@"palabra_parse_error: %@", error);
310
- return;
311
- }
312
-
313
- NSString *type = json[@"message_type"] ?: @"";
314
- NSLog(@"palabra_msg: %@", type);
315
-
316
- if ([type isEqualToString:@"output_audio_data"]) {
317
- NSDictionary *data = json[@"data"];
318
- NSString *audioBase64 = data[@"data"] ?: @"";
319
- if (audioBase64.length > 0) {
320
- NSData *audioData = [[NSData alloc] initWithBase64EncodedString:audioBase64 options:0];
321
- NSLog(@"palabra_audio_out: %lu bytes", (unsigned long)audioData.length);
322
- [self playAudio:audioData];
323
- }
324
- } else if ([type containsString:@"transcription"]) {
325
- NSDictionary *data = json[@"data"];
326
- NSDictionary *transcription = data[@"transcription"];
327
- if (transcription) {
328
- NSString *text = transcription[@"text"] ?: @"";
329
- NSString *lang = transcription[@"language"] ?: @"";
330
- BOOL isFinal = ![type isEqualToString:@"partial_transcription"];
331
- NSLog(@"palabra_transcription: %@ (%@)", text, lang);
332
-
333
- dispatch_async(dispatch_get_main_queue(), ^{
334
- [self notifyTranscription:text language:lang isFinal:isFinal];
335
- });
336
- }
337
- } else if ([type isEqualToString:@"error"]) {
338
- NSDictionary *data = json[@"data"];
339
- NSString *desc = data[@"desc"] ?: @"unknown";
340
- NSLog(@"palabra_error: %@", desc);
341
- NSError *err = [NSError errorWithDomain:@"PalabraClient" code:500 userInfo:@{NSLocalizedDescriptionKey: desc}];
342
- dispatch_async(dispatch_get_main_queue(), ^{
343
- [self notifyError:err];
344
- });
345
- } else if ([type isEqualToString:@"task_ready"]) {
346
- NSLog(@"palabra_task_ready");
347
- }
348
- }
349
-
350
- - (void)playAudio:(NSData *)audioData {
351
- if (!self.translating || !self.playerNode) return;
352
-
353
- AVAudioFrameCount frameCount = (AVAudioFrameCount)(audioData.length / 2);
354
- AVAudioPCMBuffer *buffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:self.audioFormat frameCapacity:frameCount];
355
- buffer.frameLength = frameCount;
356
-
357
- memcpy(buffer.int16ChannelData[0], audioData.bytes, audioData.length);
358
-
359
- [self.playerNode scheduleBuffer:buffer completionHandler:nil];
360
- }
361
-
362
- - (void)stop {
363
- if (!self.translating) return;
364
-
365
- self.translating = NO;
366
- self.connected = NO;
367
-
368
- if (self.remoteTrack) {
369
- [self.remoteTrack removeRenderer:self];
370
- self.remoteTrack.source.volume = self.originalVolume;
371
- }
372
-
373
- if (self.webSocket) {
374
- NSDictionary *endMsg = @{
375
- @"message_type": @"end_task",
376
- @"data": @{@"force": @NO}
377
- };
378
- NSData *jsonData = [NSJSONSerialization dataWithJSONObject:endMsg options:0 error:nil];
379
- NSString *payload = [[NSString alloc] initWithData:jsonData encoding:NSUTF8StringEncoding];
380
- NSURLSessionWebSocketMessage *msg = [[NSURLSessionWebSocketMessage alloc] initWithString:payload];
381
- [self.webSocket sendMessage:msg completionHandler:nil];
382
-
383
- [self.webSocket cancelWithCloseCode:NSURLSessionWebSocketCloseCodeNormalClosure reason:nil];
384
- self.webSocket = nil;
385
- }
386
-
387
- [self.playerNode stop];
388
- [self.audioEngine stop];
389
-
390
- [self.bufferLock lock];
391
- [self.audioBuffer setLength:0];
392
- [self.bufferLock unlock];
393
-
394
- self.remoteTrack = nil;
395
- [self notifyConnectionState:@"disconnected"];
396
- }
397
-
398
- - (void)setTranslatedVolume:(double)volume {
399
- if (self.playerNode) {
400
- self.playerNode.volume = volume;
401
- }
402
- }
403
-
404
- - (void)sendAudioToPalabra:(AVAudioPCMBuffer *)buffer {
405
- }
406
-
407
- #pragma mark - RTCAudioRenderer
408
-
409
- - (void)renderPCMBuffer:(AVAudioPCMBuffer *)buffer {
410
- if (!self.translating || !self.webSocket) {
411
- return;
412
- }
413
-
414
- static int logCounter = 0;
415
- if (logCounter++ % 100 == 0) {
416
- NSLog(@"palabra_render_pcm: frames=%u rate=%.0f channels=%u",
417
- (unsigned int)buffer.frameLength,
418
- buffer.format.sampleRate,
419
- (unsigned int)buffer.format.channelCount);
420
- }
421
-
422
- NSData *resampled = [self resampleBuffer:buffer toRate:kSampleRateIn channels:kChannels];
423
- if (resampled.length == 0) {
424
- NSLog(@"palabra_resample_empty");
425
- return;
426
- }
427
-
428
- [self.bufferLock lock];
429
- [self.audioBuffer appendData:resampled];
430
-
431
- while (self.audioBuffer.length >= kChunkBytes) {
432
- NSData *chunk = [self.audioBuffer subdataWithRange:NSMakeRange(0, kChunkBytes)];
433
- NSMutableData *remaining = [NSMutableData dataWithData:[self.audioBuffer subdataWithRange:NSMakeRange(kChunkBytes, self.audioBuffer.length - kChunkBytes)]];
434
- self.audioBuffer = remaining;
435
-
436
- [self sendAudioChunk:chunk];
437
- }
438
- [self.bufferLock unlock];
439
- }
440
-
441
- - (NSData *)resampleBuffer:(AVAudioPCMBuffer *)buffer toRate:(int)dstRate channels:(int)dstChannels {
442
- int srcRate = (int)buffer.format.sampleRate;
443
- int srcChannels = (int)buffer.format.channelCount;
444
- int srcSamples = (int)buffer.frameLength;
445
-
446
- if (srcSamples == 0) {
447
- return [NSData data];
448
- }
449
-
450
- AVAudioCommonFormat format = buffer.format.commonFormat;
451
-
452
- if (srcRate == dstRate && srcChannels == dstChannels && format == AVAudioPCMFormatInt16) {
453
- return [NSData dataWithBytes:buffer.int16ChannelData[0] length:srcSamples * 2];
454
- }
455
-
456
- float *srcData = NULL;
457
- float *tempFloat = NULL;
458
-
459
- if (format == AVAudioPCMFormatFloat32) {
460
- srcData = buffer.floatChannelData[0];
461
- } else if (format == AVAudioPCMFormatInt16) {
462
- tempFloat = malloc(srcSamples * sizeof(float));
463
- int16_t *srcInt16 = buffer.int16ChannelData[0];
464
- for (int i = 0; i < srcSamples; i++) {
465
- tempFloat[i] = (float)srcInt16[i] / 32768.0f;
466
- }
467
- srcData = tempFloat;
468
- } else {
469
- NSLog(@"palabra_unsupported_format: %ld", (long)format);
470
- return [NSData data];
471
- }
472
-
473
- float *monoSrc = srcData;
474
- float *monoBuffer = NULL;
475
- if (srcChannels == 2) {
476
- monoBuffer = malloc(srcSamples * sizeof(float));
477
- if (format == AVAudioPCMFormatFloat32 && buffer.floatChannelData[1]) {
478
- for (int i = 0; i < srcSamples; i++) {
479
- monoBuffer[i] = (buffer.floatChannelData[0][i] + buffer.floatChannelData[1][i]) / 2.0f;
480
- }
481
- } else if (format == AVAudioPCMFormatInt16 && buffer.int16ChannelData[1]) {
482
- for (int i = 0; i < srcSamples; i++) {
483
- float ch0 = (float)buffer.int16ChannelData[0][i] / 32768.0f;
484
- float ch1 = (float)buffer.int16ChannelData[1][i] / 32768.0f;
485
- monoBuffer[i] = (ch0 + ch1) / 2.0f;
486
- }
487
- } else {
488
- for (int i = 0; i < srcSamples; i++) {
489
- monoBuffer[i] = srcData[i];
490
- }
491
- }
492
- monoSrc = monoBuffer;
493
- }
494
-
495
- int dstSamples = (int)((long)srcSamples * dstRate / srcRate);
496
- if (dstSamples == 0) dstSamples = 1;
497
-
498
- int16_t *dstData = malloc(dstSamples * sizeof(int16_t));
499
-
500
- for (int i = 0; i < dstSamples; i++) {
501
- float srcIdx = (float)i * (srcSamples - 1) / (dstSamples - 1);
502
- int idx0 = (int)srcIdx;
503
- int idx1 = MIN(idx0 + 1, srcSamples - 1);
504
- float frac = srcIdx - idx0;
505
- float sample = monoSrc[idx0] * (1 - frac) + monoSrc[idx1] * frac;
506
- sample = fmaxf(-1.0f, fminf(1.0f, sample));
507
- dstData[i] = (int16_t)(sample * 32767.0f);
508
- }
509
-
510
- if (monoBuffer) free(monoBuffer);
511
- if (tempFloat) free(tempFloat);
512
-
513
- NSData *result = [NSData dataWithBytes:dstData length:dstSamples * 2];
514
- free(dstData);
515
- return result;
516
- }
517
-
518
- - (void)sendAudioChunk:(NSData *)chunk {
519
- if (!self.webSocket || !self.connected) return;
520
-
521
- static int sendCounter = 0;
522
- if (sendCounter++ % 50 == 0) {
523
- NSLog(@"palabra_send_chunk: %lu bytes (count=%d)", (unsigned long)chunk.length, sendCounter);
524
- }
525
-
526
- NSString *base64 = [chunk base64EncodedStringWithOptions:0];
527
- NSDictionary *msg = @{
528
- @"message_type": @"input_audio_data",
529
- @"data": @{
530
- @"data": base64
531
- }
532
- };
533
-
534
- NSData *jsonData = [NSJSONSerialization dataWithJSONObject:msg options:0 error:nil];
535
- NSString *payload = [[NSString alloc] initWithData:jsonData encoding:NSUTF8StringEncoding];
536
-
537
- NSURLSessionWebSocketMessage *wsMsg = [[NSURLSessionWebSocketMessage alloc] initWithString:payload];
538
- [self.webSocket sendMessage:wsMsg completionHandler:^(NSError *error) {
539
- if (error) {
540
- NSLog(@"palabra_chunk_send_error: %@", error);
541
- }
542
- }];
543
- }
544
-
545
- #pragma mark - Notifications
546
-
547
- - (void)notifyConnectionState:(NSString *)state {
548
- if ([self.delegate respondsToSelector:@selector(palabraDidChangeConnectionState:)]) {
549
- [self.delegate palabraDidChangeConnectionState:state];
550
- }
551
- }
552
-
553
- - (void)notifyError:(NSError *)error {
554
- if ([self.delegate respondsToSelector:@selector(palabraDidFailWithError:)]) {
555
- [self.delegate palabraDidFailWithError:error];
556
- }
557
- }
558
-
559
- - (void)notifyTranscription:(NSString *)text language:(NSString *)lang isFinal:(BOOL)isFinal {
560
- if ([self.delegate respondsToSelector:@selector(palabraDidReceiveTranscription:)]) {
561
- [self.delegate palabraDidReceiveTranscription:@{
562
- @"text": text,
563
- @"language": lang,
564
- @"isFinal": @(isFinal)
565
- }];
566
- }
567
- }
568
-
569
- #pragma mark - NSURLSessionWebSocketDelegate
570
-
571
- - (void)URLSession:(NSURLSession *)session webSocketTask:(NSURLSessionWebSocketTask *)webSocketTask didOpenWithProtocol:(NSString *)protocol {
572
- NSLog(@"palabra_ws_opened");
573
- }
574
-
575
- - (void)URLSession:(NSURLSession *)session webSocketTask:(NSURLSessionWebSocketTask *)webSocketTask didCloseWithCode:(NSURLSessionWebSocketCloseCode)closeCode reason:(NSData *)reason {
576
- NSLog(@"palabra_ws_closed: %ld", (long)closeCode);
577
- dispatch_async(dispatch_get_main_queue(), ^{
578
- if (self.translating) {
579
- [self stop];
580
- }
581
- });
582
- }
583
-
584
- @end
1
+ #import "PalabraClient.h"
2
+ #import "WebRTCModule.h"
3
+ #import <AVFoundation/AVFoundation.h>
4
+
5
+ static const int kSampleRateIn = 16000;
6
+ static const int kSampleRateOut = 24000;
7
+ static const int kChannels = 1;
8
+ static const int kChunkMs = 320;
9
+ static const int kChunkSamples = kSampleRateIn * kChunkMs / 1000;
10
+ static const int kChunkBytes = kChunkSamples * 2;
11
+
12
+ @interface PalabraClient () <NSURLSessionWebSocketDelegate, RTCAudioRenderer>
13
+
14
+ @property (nonatomic, strong) NSString *clientId;
15
+ @property (nonatomic, strong) NSString *clientSecret;
16
+ @property (nonatomic, strong) NSString *apiUrl;
17
+ @property (nonatomic, strong) NSString *sourceLang;
18
+ @property (nonatomic, strong) NSString *targetLang;
19
+
20
+ @property (nonatomic, strong) RTCAudioTrack *remoteTrack;
21
+
22
+ @property (nonatomic, strong) NSString *sessionId;
23
+ @property (nonatomic, strong) NSString *wsUrl;
24
+ @property (nonatomic, strong) NSString *publisherToken;
25
+ @property (nonatomic, strong) NSString *remoteTrackId;
26
+
27
+ @property (nonatomic, strong) NSURLSession *urlSession;
28
+ @property (nonatomic, strong) NSURLSessionWebSocketTask *webSocket;
29
+
30
+ @property (nonatomic, strong) AVAudioEngine *audioEngine;
31
+ @property (nonatomic, strong) AVAudioPlayerNode *playerNode;
32
+ @property (nonatomic, strong) AVAudioFormat *audioFormat;
33
+
34
+ @property (nonatomic, assign) BOOL connected;
35
+ @property (nonatomic, assign) BOOL starting;
36
+ @property (nonatomic, assign) BOOL translating;
37
+ @property (nonatomic, assign) double originalVolume;
38
+
39
+ @property (nonatomic, strong) NSMutableData *audioBuffer;
40
+ @property (nonatomic, strong) NSLock *bufferLock;
41
+
42
+ @end
43
+
44
+ @implementation PalabraClient
45
+
46
+ - (instancetype)initWithClientId:(NSString *)clientId
47
+ clientSecret:(NSString *)clientSecret
48
+ apiUrl:(NSString *)apiUrl
49
+ module:(WebRTCModule *)module {
50
+ self = [super init];
51
+ if (self) {
52
+ _clientId = clientId;
53
+ _clientSecret = clientSecret;
54
+ _apiUrl = apiUrl;
55
+ _module = module;
56
+ _connected = NO;
57
+ _translating = NO;
58
+ _originalVolume = 1.0;
59
+ _audioBuffer = [NSMutableData new];
60
+ _bufferLock = [NSLock new];
61
+
62
+ NSURLSessionConfiguration *config = [NSURLSessionConfiguration defaultSessionConfiguration];
63
+ _urlSession = [NSURLSession sessionWithConfiguration:config delegate:self delegateQueue:nil];
64
+
65
+ [self setupAudioEngine];
66
+ }
67
+ return self;
68
+ }
69
+
70
+ - (BOOL)isConnected {
71
+ return _connected;
72
+ }
73
+
74
+ - (BOOL)isTranslating {
75
+ return _translating;
76
+ }
77
+
78
+ - (BOOL)isRunning {
79
+ return self.starting || self.connected || self.translating;
80
+ }
81
+
82
+ - (BOOL)matchesSessionWithTrackId:(NSString *)trackId
83
+ sourceLang:(NSString *)sourceLang
84
+ targetLang:(NSString *)targetLang
85
+ apiUrl:(NSString *)apiUrl {
86
+ return self.remoteTrackId != nil
87
+ && [self.remoteTrackId isEqualToString:trackId]
88
+ && [self.sourceLang isEqualToString:sourceLang]
89
+ && [self.targetLang isEqualToString:targetLang]
90
+ && [self.apiUrl isEqualToString:apiUrl];
91
+ }
92
+
93
+ - (void)setupAudioEngine {
94
+ self.audioEngine = [[AVAudioEngine alloc] init];
95
+ self.playerNode = [[AVAudioPlayerNode alloc] init];
96
+ [self.audioEngine attachNode:self.playerNode];
97
+
98
+ self.audioFormat = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16
99
+ sampleRate:kSampleRateOut
100
+ channels:kChannels
101
+ interleaved:YES];
102
+
103
+ [self.audioEngine connect:self.playerNode
104
+ to:self.audioEngine.mainMixerNode
105
+ format:self.audioFormat];
106
+ }
107
+
108
+ - (void)startWithTrack:(RTCAudioTrack *)remoteAudioTrack
109
+ sourceLang:(NSString *)sourceLang
110
+ targetLang:(NSString *)targetLang {
111
+
112
+ if (self.starting || self.translating) {
113
+ NSLog(@"palabra_start_skip_busy");
114
+ return;
115
+ }
116
+
117
+ self.starting = YES;
118
+ self.remoteTrack = remoteAudioTrack;
119
+ self.remoteTrackId = remoteAudioTrack.trackId;
120
+ self.sourceLang = sourceLang;
121
+ self.targetLang = targetLang;
122
+
123
+ self.originalVolume = remoteAudioTrack.source.volume;
124
+ remoteAudioTrack.source.volume = 0;
125
+ NSLog(@"palabra_start: %@", self.remoteTrackId);
126
+
127
+ [self notifyConnectionState:@"connecting"];
128
+
129
+ __weak typeof(self) weakSelf = self;
130
+ [self createSessionWithCompletion:^(NSDictionary *session, NSError *error) {
131
+ __strong typeof(weakSelf) strongSelf = weakSelf;
132
+ if (!strongSelf) return;
133
+
134
+ if (!strongSelf.starting) {
135
+ NSLog(@"palabra_start_cancelled");
136
+ return;
137
+ }
138
+
139
+ if (error) {
140
+ strongSelf.starting = NO;
141
+ [strongSelf notifyError:error];
142
+ strongSelf.remoteTrack.source.volume = strongSelf.originalVolume;
143
+ return;
144
+ }
145
+
146
+ NSDictionary *data = session[@"data"];
147
+ strongSelf.sessionId = data[@"id"];
148
+ strongSelf.wsUrl = data[@"ws_url"];
149
+ strongSelf.publisherToken = data[@"publisher"];
150
+
151
+ NSLog(@"palabra_ws_url: %@", strongSelf.wsUrl);
152
+
153
+ dispatch_async(dispatch_get_main_queue(), ^{
154
+ [strongSelf connectWebSocket];
155
+ });
156
+ }];
157
+ }
158
+
159
+ - (void)createSessionWithCompletion:(void (^)(NSDictionary *, NSError *))completion {
160
+ NSString *urlStr = [NSString stringWithFormat:@"%@/session-storage/session", self.apiUrl];
161
+ NSURL *url = [NSURL URLWithString:urlStr];
162
+
163
+ NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:url];
164
+ request.HTTPMethod = @"POST";
165
+ [request setValue:self.clientId forHTTPHeaderField:@"ClientId"];
166
+ [request setValue:self.clientSecret forHTTPHeaderField:@"ClientSecret"];
167
+ [request setValue:@"application/json" forHTTPHeaderField:@"Content-Type"];
168
+
169
+ NSDictionary *body = @{
170
+ @"data": @{
171
+ @"subscriber_count": @0,
172
+ @"publisher_can_subscribe": @YES
173
+ }
174
+ };
175
+
176
+ NSError *jsonError;
177
+ request.HTTPBody = [NSJSONSerialization dataWithJSONObject:body options:0 error:&jsonError];
178
+
179
+ if (jsonError) {
180
+ completion(nil, jsonError);
181
+ return;
182
+ }
183
+
184
+ NSURLSessionDataTask *task = [[NSURLSession sharedSession] dataTaskWithRequest:request
185
+ completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) {
186
+ if (error) {
187
+ dispatch_async(dispatch_get_main_queue(), ^{
188
+ completion(nil, error);
189
+ });
190
+ return;
191
+ }
192
+
193
+ NSHTTPURLResponse *httpResponse = (NSHTTPURLResponse *)response;
194
+ if (httpResponse.statusCode < 200 || httpResponse.statusCode >= 300) {
195
+ NSString *message = [self sessionErrorMessageForStatusCode:httpResponse.statusCode data:data];
196
+ NSError *httpError = [NSError errorWithDomain:@"PalabraClient"
197
+ code:httpResponse.statusCode
198
+ userInfo:@{NSLocalizedDescriptionKey: message}];
199
+ dispatch_async(dispatch_get_main_queue(), ^{
200
+ completion(nil, httpError);
201
+ });
202
+ return;
203
+ }
204
+
205
+ NSError *parseError;
206
+ NSDictionary *json = [NSJSONSerialization JSONObjectWithData:data options:0 error:&parseError];
207
+
208
+ dispatch_async(dispatch_get_main_queue(), ^{
209
+ completion(json, parseError);
210
+ });
211
+ }];
212
+
213
+ [task resume];
214
+ }
215
+
216
+ - (NSString *)sessionErrorMessageForStatusCode:(NSInteger)statusCode data:(NSData *)data {
217
+ NSString *fallback = [NSString stringWithFormat:@"session_http_error_%ld", (long)statusCode];
218
+ if (!data || data.length == 0) {
219
+ return fallback;
220
+ }
221
+
222
+ NSError *parseError;
223
+ id json = [NSJSONSerialization JSONObjectWithData:data options:0 error:&parseError];
224
+ if (parseError || ![json isKindOfClass:[NSDictionary class]]) {
225
+ return fallback;
226
+ }
227
+
228
+ NSArray *errors = ((NSDictionary *)json)[@"errors"];
229
+ if (![errors isKindOfClass:[NSArray class]] || errors.count == 0) {
230
+ return fallback;
231
+ }
232
+
233
+ NSDictionary *first = errors.firstObject;
234
+ if (![first isKindOfClass:[NSDictionary class]]) {
235
+ return fallback;
236
+ }
237
+
238
+ NSString *detail = first[@"detail"];
239
+ if (![detail isKindOfClass:[NSString class]] || detail.length == 0) {
240
+ return fallback;
241
+ }
242
+
243
+ return [NSString stringWithFormat:@"%@: %@", fallback, detail];
244
+ }
245
+
246
+ - (void)connectWebSocket {
247
+ if (!self.starting && !self.translating) {
248
+ NSLog(@"palabra_connect_skip");
249
+ return;
250
+ }
251
+
252
+ NSString *endpoint = [NSString stringWithFormat:@"%@?token=%@", self.wsUrl, self.publisherToken];
253
+ NSLog(@"palabra_connecting_ws: %@", endpoint);
254
+
255
+ NSURL *url = [NSURL URLWithString:endpoint];
256
+ self.webSocket = [self.urlSession webSocketTaskWithURL:url];
257
+ [self.webSocket resume];
258
+
259
+ [self receiveMessage];
260
+ }
261
+
262
+ - (void)handleWebSocketOpened {
263
+ if (self.connected || self.translating || !self.webSocket) {
264
+ return;
265
+ }
266
+
267
+ self.starting = NO;
268
+ self.connected = YES;
269
+ self.translating = YES;
270
+
271
+ [self.remoteTrack addRenderer:self];
272
+
273
+ NSError *audioError;
274
+ [self.audioEngine startAndReturnError:&audioError];
275
+ if (audioError) {
276
+ NSLog(@"palabra_audio_engine_error: %@", audioError);
277
+ [self.remoteTrack removeRenderer:self];
278
+ self.starting = NO;
279
+ self.translating = NO;
280
+ self.connected = NO;
281
+ [self.webSocket cancelWithCloseCode:NSURLSessionWebSocketCloseCodeInternalServerError reason:nil];
282
+ self.webSocket = nil;
283
+ [self notifyError:audioError];
284
+ return;
285
+ }
286
+
287
+ [self.playerNode play];
288
+ NSLog(@"palabra_audio_engine_started");
289
+
290
+ [self notifyConnectionState:@"connected"];
291
+
292
+ dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.5 * NSEC_PER_SEC)), dispatch_get_main_queue(), ^{
293
+ [self sendSetTask];
294
+ });
295
+ }
296
+
297
+ - (void)receiveMessage {
298
+ if (!self.webSocket) return;
299
+
300
+ __weak typeof(self) weakSelf = self;
301
+ [self.webSocket receiveMessageWithCompletionHandler:^(NSURLSessionWebSocketMessage *message, NSError *error) {
302
+ __strong typeof(weakSelf) strongSelf = weakSelf;
303
+ if (!strongSelf) return;
304
+
305
+ if (error) {
306
+ NSLog(@"palabra_ws_error: %@", error);
307
+ dispatch_async(dispatch_get_main_queue(), ^{
308
+ [strongSelf stop];
309
+ [strongSelf notifyError:error];
310
+ });
311
+ return;
312
+ }
313
+
314
+ if (message.type == NSURLSessionWebSocketMessageTypeString) {
315
+ [strongSelf handleMessage:message.string];
316
+ }
317
+
318
+ [strongSelf receiveMessage];
319
+ }];
320
+ }
321
+
322
+ - (void)sendSetTask {
323
+ if (!self.webSocket || !self.connected) return;
324
+
325
+ NSDictionary *msg = @{
326
+ @"message_type": @"set_task",
327
+ @"data": @{
328
+ @"input_stream": @{
329
+ @"content_type": @"audio",
330
+ @"source": @{
331
+ @"type": @"ws",
332
+ @"format": @"pcm_s16le",
333
+ @"sample_rate": @(kSampleRateIn),
334
+ @"channels": @(kChannels)
335
+ }
336
+ },
337
+ @"output_stream": @{
338
+ @"content_type": @"audio",
339
+ @"target": @{
340
+ @"type": @"ws",
341
+ @"format": @"pcm_s16le"
342
+ }
343
+ },
344
+ @"pipeline": @{
345
+ @"transcription": @{
346
+ @"source_language": self.sourceLang
347
+ },
348
+ @"translations": @[@{
349
+ @"target_language": self.targetLang,
350
+ @"speech_generation": @{
351
+ @"voice_cloning": @NO
352
+ }
353
+ }],
354
+ @"allowed_message_types": @[
355
+ @"partial_transcription",
356
+ @"validated_transcription",
357
+ @"translated_transcription"
358
+ ]
359
+ }
360
+ }
361
+ };
362
+
363
+ NSError *error;
364
+ NSData *jsonData = [NSJSONSerialization dataWithJSONObject:msg options:0 error:&error];
365
+ if (error) {
366
+ NSLog(@"palabra_set_task_error: %@", error);
367
+ return;
368
+ }
369
+
370
+ NSString *payload = [[NSString alloc] initWithData:jsonData encoding:NSUTF8StringEncoding];
371
+ NSLog(@"palabra_set_task: %@", payload);
372
+
373
+ NSURLSessionWebSocketMessage *wsMsg = [[NSURLSessionWebSocketMessage alloc] initWithString:payload];
374
+ [self.webSocket sendMessage:wsMsg completionHandler:^(NSError *error) {
375
+ if (error) {
376
+ NSLog(@"palabra_send_error: %@", error);
377
+ }
378
+ }];
379
+ }
380
+
381
+ - (void)handleMessage:(NSString *)text {
382
+ NSError *error;
383
+ NSDictionary *json = [NSJSONSerialization JSONObjectWithData:[text dataUsingEncoding:NSUTF8StringEncoding]
384
+ options:0
385
+ error:&error];
386
+ if (error) {
387
+ NSLog(@"palabra_parse_error: %@", error);
388
+ return;
389
+ }
390
+
391
+ NSString *type = json[@"message_type"] ?: @"";
392
+ NSLog(@"palabra_msg: %@", type);
393
+
394
+ if ([type isEqualToString:@"output_audio_data"]) {
395
+ NSDictionary *data = json[@"data"];
396
+ NSString *audioBase64 = data[@"data"] ?: @"";
397
+ if (audioBase64.length > 0) {
398
+ NSData *audioData = [[NSData alloc] initWithBase64EncodedString:audioBase64 options:0];
399
+ NSLog(@"palabra_audio_out: %lu bytes", (unsigned long)audioData.length);
400
+ [self playAudio:audioData];
401
+ }
402
+ } else if ([type containsString:@"transcription"]) {
403
+ NSDictionary *data = json[@"data"];
404
+ NSDictionary *transcription = data[@"transcription"];
405
+ if (transcription) {
406
+ NSString *text = transcription[@"text"] ?: @"";
407
+ NSString *lang = transcription[@"language"] ?: @"";
408
+ BOOL isFinal = ![type isEqualToString:@"partial_transcription"];
409
+ NSLog(@"palabra_transcription: %@ (%@)", text, lang);
410
+
411
+ dispatch_async(dispatch_get_main_queue(), ^{
412
+ [self notifyTranscription:text language:lang isFinal:isFinal];
413
+ });
414
+ }
415
+ } else if ([type isEqualToString:@"error"]) {
416
+ NSDictionary *data = json[@"data"];
417
+ NSString *desc = data[@"desc"] ?: @"unknown";
418
+ NSLog(@"palabra_error: %@", desc);
419
+ NSError *err = [NSError errorWithDomain:@"PalabraClient" code:500 userInfo:@{NSLocalizedDescriptionKey: desc}];
420
+ dispatch_async(dispatch_get_main_queue(), ^{
421
+ [self notifyError:err];
422
+ });
423
+ } else if ([type isEqualToString:@"task_ready"]) {
424
+ NSLog(@"palabra_task_ready");
425
+ }
426
+ }
427
+
428
+ - (void)playAudio:(NSData *)audioData {
429
+ if (!self.translating || !self.playerNode) return;
430
+
431
+ AVAudioFrameCount frameCount = (AVAudioFrameCount)(audioData.length / 2);
432
+ AVAudioPCMBuffer *buffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:self.audioFormat frameCapacity:frameCount];
433
+ buffer.frameLength = frameCount;
434
+
435
+ memcpy(buffer.int16ChannelData[0], audioData.bytes, audioData.length);
436
+
437
+ [self.playerNode scheduleBuffer:buffer completionHandler:nil];
438
+ }
439
+
440
+ - (void)stop {
441
+ if (!self.starting && !self.translating && !self.connected && !self.remoteTrack && !self.webSocket) return;
442
+
443
+ self.starting = NO;
444
+ self.translating = NO;
445
+ self.connected = NO;
446
+
447
+ if (self.remoteTrack) {
448
+ [self.remoteTrack removeRenderer:self];
449
+ self.remoteTrack.source.volume = self.originalVolume;
450
+ }
451
+
452
+ if (self.webSocket) {
453
+ NSDictionary *endMsg = @{
454
+ @"message_type": @"end_task",
455
+ @"data": @{@"force": @NO}
456
+ };
457
+ NSData *jsonData = [NSJSONSerialization dataWithJSONObject:endMsg options:0 error:nil];
458
+ NSString *payload = [[NSString alloc] initWithData:jsonData encoding:NSUTF8StringEncoding];
459
+ NSURLSessionWebSocketMessage *msg = [[NSURLSessionWebSocketMessage alloc] initWithString:payload];
460
+ [self.webSocket sendMessage:msg completionHandler:nil];
461
+
462
+ [self.webSocket cancelWithCloseCode:NSURLSessionWebSocketCloseCodeNormalClosure reason:nil];
463
+ self.webSocket = nil;
464
+ }
465
+
466
+ [self.playerNode stop];
467
+ [self.audioEngine stop];
468
+
469
+ [self.bufferLock lock];
470
+ [self.audioBuffer setLength:0];
471
+ [self.bufferLock unlock];
472
+
473
+ self.remoteTrack = nil;
474
+ self.remoteTrackId = nil;
475
+ [self notifyConnectionState:@"disconnected"];
476
+ }
477
+
478
+ - (void)setTranslatedVolume:(double)volume {
479
+ if (self.playerNode) {
480
+ self.playerNode.volume = volume;
481
+ }
482
+ }
483
+
484
+ - (void)sendAudioToPalabra:(AVAudioPCMBuffer *)buffer {
485
+ }
486
+
487
+ #pragma mark - RTCAudioRenderer
488
+
489
+ - (void)renderPCMBuffer:(AVAudioPCMBuffer *)buffer {
490
+ if (!self.translating || !self.webSocket) {
491
+ return;
492
+ }
493
+
494
+ static int logCounter = 0;
495
+ if (logCounter++ % 100 == 0) {
496
+ NSLog(@"palabra_render_pcm: frames=%u rate=%.0f channels=%u",
497
+ (unsigned int)buffer.frameLength,
498
+ buffer.format.sampleRate,
499
+ (unsigned int)buffer.format.channelCount);
500
+ }
501
+
502
+ NSData *resampled = [self resampleBuffer:buffer toRate:kSampleRateIn channels:kChannels];
503
+ if (resampled.length == 0) {
504
+ NSLog(@"palabra_resample_empty");
505
+ return;
506
+ }
507
+
508
+ [self.bufferLock lock];
509
+ [self.audioBuffer appendData:resampled];
510
+
511
+ while (self.audioBuffer.length >= kChunkBytes) {
512
+ NSData *chunk = [self.audioBuffer subdataWithRange:NSMakeRange(0, kChunkBytes)];
513
+ NSMutableData *remaining = [NSMutableData dataWithData:[self.audioBuffer subdataWithRange:NSMakeRange(kChunkBytes, self.audioBuffer.length - kChunkBytes)]];
514
+ self.audioBuffer = remaining;
515
+
516
+ [self sendAudioChunk:chunk];
517
+ }
518
+ [self.bufferLock unlock];
519
+ }
520
+
521
+ - (NSData *)resampleBuffer:(AVAudioPCMBuffer *)buffer toRate:(int)dstRate channels:(int)dstChannels {
522
+ int srcRate = (int)buffer.format.sampleRate;
523
+ int srcChannels = (int)buffer.format.channelCount;
524
+ int srcSamples = (int)buffer.frameLength;
525
+
526
+ if (srcSamples == 0) {
527
+ return [NSData data];
528
+ }
529
+
530
+ AVAudioCommonFormat format = buffer.format.commonFormat;
531
+
532
+ if (srcRate == dstRate && srcChannels == dstChannels && format == AVAudioPCMFormatInt16) {
533
+ return [NSData dataWithBytes:buffer.int16ChannelData[0] length:srcSamples * 2];
534
+ }
535
+
536
+ float *srcData = NULL;
537
+ float *tempFloat = NULL;
538
+
539
+ if (format == AVAudioPCMFormatFloat32) {
540
+ srcData = buffer.floatChannelData[0];
541
+ } else if (format == AVAudioPCMFormatInt16) {
542
+ tempFloat = malloc(srcSamples * sizeof(float));
543
+ int16_t *srcInt16 = buffer.int16ChannelData[0];
544
+ for (int i = 0; i < srcSamples; i++) {
545
+ tempFloat[i] = (float)srcInt16[i] / 32768.0f;
546
+ }
547
+ srcData = tempFloat;
548
+ } else {
549
+ NSLog(@"palabra_unsupported_format: %ld", (long)format);
550
+ return [NSData data];
551
+ }
552
+
553
+ float *monoSrc = srcData;
554
+ float *monoBuffer = NULL;
555
+ if (srcChannels == 2) {
556
+ monoBuffer = malloc(srcSamples * sizeof(float));
557
+ if (format == AVAudioPCMFormatFloat32 && buffer.floatChannelData[1]) {
558
+ for (int i = 0; i < srcSamples; i++) {
559
+ monoBuffer[i] = (buffer.floatChannelData[0][i] + buffer.floatChannelData[1][i]) / 2.0f;
560
+ }
561
+ } else if (format == AVAudioPCMFormatInt16 && buffer.int16ChannelData[1]) {
562
+ for (int i = 0; i < srcSamples; i++) {
563
+ float ch0 = (float)buffer.int16ChannelData[0][i] / 32768.0f;
564
+ float ch1 = (float)buffer.int16ChannelData[1][i] / 32768.0f;
565
+ monoBuffer[i] = (ch0 + ch1) / 2.0f;
566
+ }
567
+ } else {
568
+ for (int i = 0; i < srcSamples; i++) {
569
+ monoBuffer[i] = srcData[i];
570
+ }
571
+ }
572
+ monoSrc = monoBuffer;
573
+ }
574
+
575
+ int dstSamples = (int)((long)srcSamples * dstRate / srcRate);
576
+ if (dstSamples == 0) dstSamples = 1;
577
+
578
+ int16_t *dstData = malloc(dstSamples * sizeof(int16_t));
579
+
580
+ if (dstSamples == 1 || srcSamples == 1) {
581
+ float sample = monoSrc[0];
582
+ sample = fmaxf(-1.0f, fminf(1.0f, sample));
583
+ dstData[0] = (int16_t)(sample * 32767.0f);
584
+
585
+ if (monoBuffer) free(monoBuffer);
586
+ if (tempFloat) free(tempFloat);
587
+
588
+ NSData *result = [NSData dataWithBytes:dstData length:sizeof(int16_t)];
589
+ free(dstData);
590
+ return result;
591
+ }
592
+
593
+ for (int i = 0; i < dstSamples; i++) {
594
+ float srcIdx = (float)i * (srcSamples - 1) / (dstSamples - 1);
595
+ int idx0 = (int)srcIdx;
596
+ int idx1 = MIN(idx0 + 1, srcSamples - 1);
597
+ float frac = srcIdx - idx0;
598
+ float sample = monoSrc[idx0] * (1 - frac) + monoSrc[idx1] * frac;
599
+ sample = fmaxf(-1.0f, fminf(1.0f, sample));
600
+ dstData[i] = (int16_t)(sample * 32767.0f);
601
+ }
602
+
603
+ if (monoBuffer) free(monoBuffer);
604
+ if (tempFloat) free(tempFloat);
605
+
606
+ NSData *result = [NSData dataWithBytes:dstData length:dstSamples * 2];
607
+ free(dstData);
608
+ return result;
609
+ }
610
+
611
+ - (void)sendAudioChunk:(NSData *)chunk {
612
+ if (!self.webSocket || !self.connected) return;
613
+
614
+ static int sendCounter = 0;
615
+ if (sendCounter++ % 50 == 0) {
616
+ NSLog(@"palabra_send_chunk: %lu bytes (count=%d)", (unsigned long)chunk.length, sendCounter);
617
+ }
618
+
619
+ NSString *base64 = [chunk base64EncodedStringWithOptions:0];
620
+ NSDictionary *msg = @{
621
+ @"message_type": @"input_audio_data",
622
+ @"data": @{
623
+ @"data": base64
624
+ }
625
+ };
626
+
627
+ NSData *jsonData = [NSJSONSerialization dataWithJSONObject:msg options:0 error:nil];
628
+ NSString *payload = [[NSString alloc] initWithData:jsonData encoding:NSUTF8StringEncoding];
629
+
630
+ NSURLSessionWebSocketMessage *wsMsg = [[NSURLSessionWebSocketMessage alloc] initWithString:payload];
631
+ [self.webSocket sendMessage:wsMsg completionHandler:^(NSError *error) {
632
+ if (error) {
633
+ NSLog(@"palabra_chunk_send_error: %@", error);
634
+ }
635
+ }];
636
+ }
637
+
638
+ #pragma mark - Notifications
639
+
640
+ - (void)notifyConnectionState:(NSString *)state {
641
+ if ([self.delegate respondsToSelector:@selector(palabraDidChangeConnectionState:)]) {
642
+ [self.delegate palabraDidChangeConnectionState:state];
643
+ }
644
+ }
645
+
646
+ - (void)notifyError:(NSError *)error {
647
+ if ([self.delegate respondsToSelector:@selector(palabraDidFailWithError:)]) {
648
+ [self.delegate palabraDidFailWithError:error];
649
+ }
650
+ }
651
+
652
+ - (void)notifyTranscription:(NSString *)text language:(NSString *)lang isFinal:(BOOL)isFinal {
653
+ if ([self.delegate respondsToSelector:@selector(palabraDidReceiveTranscription:)]) {
654
+ [self.delegate palabraDidReceiveTranscription:@{
655
+ @"text": text,
656
+ @"lang": lang,
657
+ @"isFinal": @(isFinal)
658
+ }];
659
+ }
660
+ }
661
+
662
+ #pragma mark - NSURLSessionWebSocketDelegate
663
+
664
+ - (void)URLSession:(NSURLSession *)session webSocketTask:(NSURLSessionWebSocketTask *)webSocketTask didOpenWithProtocol:(NSString *)protocol {
665
+ NSLog(@"palabra_ws_opened");
666
+ dispatch_async(dispatch_get_main_queue(), ^{
667
+ [self handleWebSocketOpened];
668
+ });
669
+ }
670
+
671
+ - (void)URLSession:(NSURLSession *)session webSocketTask:(NSURLSessionWebSocketTask *)webSocketTask didCloseWithCode:(NSURLSessionWebSocketCloseCode)closeCode reason:(NSData *)reason {
672
+ NSLog(@"palabra_ws_closed: %ld", (long)closeCode);
673
+ dispatch_async(dispatch_get_main_queue(), ^{
674
+ if (self.translating) {
675
+ [self stop];
676
+ }
677
+ });
678
+ }
679
+
680
+ @end