@sbhjt-gr/react-native-webrtc 137.0.4 → 137.0.5-palabra.10

package/README.md

@@ -1,13 +1,34 @@
 [<img src="https://avatars.githubusercontent.com/u/42463376" alt="React Native WebRTC" style="height: 6em;" />](https://github.com/react-native-webrtc/react-native-webrtc)
 
-# React-Native-WebRTC
-[![npm version](https://img.shields.io/npm/v/@sbhjt-gr/react-native-webrtc)](https://www.npmjs.com/package/@sbhjt-gr/react-native-webrtc)
+# React-Native-WebRTC for LiveKit (WiLang Fork)
+[![npm version](https://img.shields.io/npm/v/@livekit/react-native-webrtc)](https://www.npmjs.com/package/@livekit/react-native-webrtc)
 [![Discourse topics](https://img.shields.io/discourse/topics?server=https%3A%2F%2Freact-native-webrtc.discourse.group%2F)](https://react-native-webrtc.discourse.group/)
 
-A WebRTC module for React Native.
+A WebRTC module for React Native with native Palabra AI translation support.
 
 > [!NOTE]
-> This is a fork of [livekit/react-native-webrtc](https://github.com/livekit/react-native-webrtc) with additional audio interception features for real-time translation.
+> This is a fork with integrated Palabra translation for real-time speech-to-speech translation during WebRTC calls.
+> Used by WiLang app for video/voice call translation.
+
+## Palabra Translation
+
+This fork adds native Palabra AI integration for translating remote peer audio in real-time:
+
+```javascript
+import { startPalabraTranslation, stopPalabraTranslation } from '@livekit/react-native-webrtc';
+
+await startPalabraTranslation(
+  peerConnectionId,
+  remoteAudioTrackId,
+  PALABRA_CLIENT_ID,
+  PALABRA_CLIENT_SECRET,
+  'en',
+  'es',
+  'https://api.palabra.ai'
+);
+
+await stopPalabraTranslation();
+```
 
 ## Feature Overview
 
@@ -52,9 +73,9 @@ Software encode/decode factories have been enabled by default.
 Use one of the following preferred package install methods to immediately get going.  
 Don't forget to follow platform guides below to cover any extra required steps.  
 
-**npm:** `npm install @sbhjt-gr/react-native-webrtc --save`  
-**yarn:** `yarn add @sbhjt-gr/react-native-webrtc`  
-**pnpm:** `pnpm install @sbhjt-gr/react-native-webrtc`  
+**npm:** `npm install @livekit/react-native-webrtc --save`  
+**yarn:** `yarn add @livekit/react-native-webrtc`  
+**pnpm:** `pnpm install @livekit/react-native-webrtc`  
 
 ## Guides
 

package/android/build.gradle

@@ -33,4 +33,5 @@ dependencies {
     implementation "com.facebook.react:react-android:+"
     api 'io.github.webrtc-sdk:android:137.7151.04'
     implementation "androidx.core:core:1.7.0"
+    implementation "com.squareup.okhttp3:okhttp:4.12.0"
 }

package/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java

@@ -23,11 +23,13 @@ import com.facebook.react.module.annotations.ReactModule;
 import com.facebook.react.modules.core.DeviceEventManagerModule;
 import com.oney.WebRTCModule.webrtcutils.H264AndSoftwareVideoDecoderFactory;
 import com.oney.WebRTCModule.webrtcutils.H264AndSoftwareVideoEncoderFactory;
+import com.oney.WebRTCModule.palabra.PalabraClient;
+import com.oney.WebRTCModule.palabra.PalabraConfig;
+import com.oney.WebRTCModule.palabra.PalabraListener;
 
 import org.webrtc.AddIceObserver;
 import org.webrtc.AudioProcessingFactory;
 import org.webrtc.AudioTrack;
-import org.webrtc.AudioTrackSink;
 import org.webrtc.CryptoOptions;
 import org.webrtc.EglBase;
 import org.webrtc.IceCandidate;
@@ -53,17 +55,12 @@ import org.webrtc.VideoTrack;
 import org.webrtc.audio.AudioDeviceModule;
 import org.webrtc.audio.JavaAudioDeviceModule;
 
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.nio.IntBuffer;
-import java.nio.ShortBuffer;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 
 @ReactModule(name = "WebRTCModule")
@@ -78,10 +75,11 @@ public class WebRTCModule extends ReactContextBaseJavaModule {
     // Need to expose the peer connection codec factories here to get capabilities
     private final SparseArray<PeerConnectionObserver> mPeerConnectionObservers;
     final Map<String, MediaStream> localStreams;
-    private final Map<String, AudioTrackSink> audioSinkMap = new ConcurrentHashMap<>();
 
     private final GetUserMediaImpl getUserMediaImpl;
 
+    private PalabraClient palabraClient;
+
     public WebRTCModule(ReactApplicationContext reactContext) {
         super(reactContext);
 
@@ -486,10 +484,6 @@ public class WebRTCModule extends ReactContextBaseJavaModule {
         return pco.remoteTracks.get(trackId);
     }
 
-    private String audioSinkKey(int pcId, String trackId) {
-        return pcId + ":" + trackId;
-    }
-
     MediaStreamTrack getLocalTrack(String trackId) {
         return getUserMediaImpl.getTrack(trackId);
     }
@@ -964,118 +958,6 @@ public class WebRTCModule extends ReactContextBaseJavaModule {
         });
     }
 
-    @ReactMethod
-    public void mediaStreamTrackSetPlaybackEnabled(int pcId, String id, boolean enabled) {
-        ThreadUtils.runOnExecutor(() -> {
-            MediaStreamTrack track = getTrack(pcId, id);
-            if (track == null) {
-                Log.d(TAG, "mediaStreamTrackSetPlaybackEnabled() could not find track " + id);
-                return;
-            }
-
-            if (!(track instanceof AudioTrack)) {
-                Log.d(TAG, "mediaStreamTrackSetPlaybackEnabled() track is not an AudioTrack!");
-                return;
-            }
-
-            ((AudioTrack) track).setVolume(enabled ? 1.0 : 0.0);
-        });
-    }
-
-    private class AudioSamplesInterceptor implements AudioTrackSink {
-        private final String trackId;
-
-        AudioSamplesInterceptor(String trackId) {
-            this.trackId = trackId;
-        }
-
-        @Override
-        public void onData(
-                ByteBuffer audioData,
-                int bitsPerSample,
-                int sampleRate,
-                int numberOfChannels,
-                int numberOfFrames,
-                long absoluteCaptureTimestampMs) {
-            WritableMap params = Arguments.createMap();
-            params.putString("trackId", trackId);
-            params.putArray("samples", convertSamples(audioData, bitsPerSample));
-            params.putInt("sampleRate", sampleRate);
-            params.putInt("channels", numberOfChannels);
-            params.putInt("bitsPerSample", bitsPerSample);
-            params.putInt("framesPerBuffer", numberOfFrames);
-            params.putDouble("timestamp", (double) absoluteCaptureTimestampMs);
-
-            sendEvent("audioSamples", params);
-        }
-
-        private WritableArray convertSamples(ByteBuffer audioData, int bitsPerSample) {
-            WritableArray samples = Arguments.createArray();
-            if (audioData == null) {
-                return samples;
-            }
-
-            ByteBuffer buffer = audioData.slice().order(ByteOrder.LITTLE_ENDIAN);
-
-            if (bitsPerSample == 16) {
-                ShortBuffer shortBuffer = buffer.asShortBuffer();
-                while (shortBuffer.hasRemaining()) {
-                    samples.pushDouble(shortBuffer.get());
-                }
-            } else if (bitsPerSample == 32) {
-                IntBuffer intBuffer = buffer.asIntBuffer();
-                while (intBuffer.hasRemaining()) {
-                    samples.pushDouble(intBuffer.get());
-                }
-            } else {
-                while (buffer.hasRemaining()) {
-                    samples.pushDouble(buffer.get());
-                }
-            }
-
-            return samples;
-        }
-    }
-
-    @ReactMethod
-    public void mediaStreamTrackEnableAudioSink(int pcId, String id, boolean enabled) {
-        ThreadUtils.runOnExecutor(() -> {
-            String sinkKey = audioSinkKey(pcId, id);
-
-            if (enabled) {
-                if (audioSinkMap.containsKey(sinkKey)) {
-                    return;
-                }
-
-                MediaStreamTrack track = getTrack(pcId, id);
-                if (track == null) {
-                    Log.d(TAG, "mediaStreamTrackEnableAudioSink() could not find track " + id);
-                    return;
-                }
-
-                if (!(track instanceof AudioTrack)) {
-                    Log.d(TAG, "mediaStreamTrackEnableAudioSink() track is not an AudioTrack!");
-                    return;
-                }
-
-                AudioSamplesInterceptor sink = new AudioSamplesInterceptor(id);
-                ((AudioTrack) track).addSink(sink);
-                audioSinkMap.put(sinkKey, sink);
-            } else {
-                AudioTrackSink sink = audioSinkMap.remove(sinkKey);
-
-                if (sink == null) {
-                    return;
-                }
-
-                MediaStreamTrack track = getTrack(pcId, id);
-                if (track instanceof AudioTrack) {
-                    ((AudioTrack) track).removeSink(sink);
-                }
-            }
-        });
-    }
-
     /**
      * This serializes the transceivers current direction and mid and returns them
      * for update when an sdp negotiation/renegotiation happens
@@ -1690,4 +1572,72 @@ public class WebRTCModule extends ReactContextBaseJavaModule {
     public void removeListeners(Integer count) {
         // Keep: Required for RN built in Event Emitter Calls.
     }
+
+    @ReactMethod
+    public void startPalabraTranslation(int peerConnectionId, String trackId, String clientId,
+                                        String clientSecret, String sourceLang, String targetLang,
+                                        String apiUrl, Promise promise) {
+        PeerConnectionObserver pco = mPeerConnectionObservers.get(peerConnectionId);
+        if (pco == null) {
+            promise.reject("E_INVALID", "pc_not_found");
+            return;
+        }
+
+        MediaStreamTrack track = pco.remoteTracks.get(trackId);
+        if (track == null) {
+            promise.reject("E_INVALID", "track_not_found");
+            return;
+        }
+
+        if (!(track instanceof AudioTrack)) {
+            promise.reject("E_INVALID", "not_audio_track");
+            return;
+        }
+
+        AudioTrack audioTrack = (AudioTrack) track;
+
+        if (palabraClient != null) {
+            palabraClient.stop();
+        }
+
+        PalabraConfig config = new PalabraConfig(clientId, clientSecret, sourceLang, targetLang, apiUrl);
+        palabraClient = new PalabraClient(getReactApplicationContext(), config);
+        palabraClient.setListener(new PalabraListener() {
+            @Override
+            public void onTranscription(String text, String lang, boolean isFinal) {
+                WritableMap params = Arguments.createMap();
+                params.putString("text", text);
+                params.putString("lang", lang);
+                params.putBoolean("isFinal", isFinal);
+                sendEvent("palabraTranscription", params);
+            }
+
+            @Override
+            public void onConnectionState(String state) {
+                WritableMap params = Arguments.createMap();
+                params.putString("state", state);
+                sendEvent("palabraConnectionState", params);
+            }
+
+            @Override
+            public void onError(int code, String message) {
+                WritableMap params = Arguments.createMap();
+                params.putInt("code", code);
+                params.putString("message", message);
+                sendEvent("palabraError", params);
+            }
+        });
+
+        palabraClient.start(audioTrack);
+        promise.resolve(null);
+    }
+
+    @ReactMethod
+    public void stopPalabraTranslation(Promise promise) {
+        if (palabraClient != null) {
+            palabraClient.stop();
+            palabraClient = null;
+        }
+        promise.resolve(null);
+    }
 }