@sbhjt-gr/react-native-webrtc 137.0.3 → 137.0.5-palabra.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/README.md +28 -7
  2. package/android/build.gradle +1 -0
  3. package/android/src/main/java/com/oney/WebRTCModule/WebRTCModule.java +73 -23
  4. package/android/src/main/java/com/oney/WebRTCModule/palabra/PalabraClient.java +464 -0
  5. package/android/src/main/java/com/oney/WebRTCModule/palabra/PalabraConfig.java +17 -0
  6. package/android/src/main/java/com/oney/WebRTCModule/palabra/PalabraListener.java +7 -0
  7. package/ios/RCTWebRTC/PalabraAudioSink.h +13 -0
  8. package/ios/RCTWebRTC/PalabraAudioSink.m +18 -0
  9. package/ios/RCTWebRTC/PalabraClient.h +36 -0
  10. package/ios/RCTWebRTC/PalabraClient.m +584 -0
  11. package/ios/RCTWebRTC/WebRTCModule+Palabra.h +4 -0
  12. package/ios/RCTWebRTC/WebRTCModule+Palabra.m +83 -0
  13. package/ios/RCTWebRTC/WebRTCModule+RTCMediaStream.m +0 -12
  14. package/ios/RCTWebRTC/WebRTCModule.h +7 -1
  15. package/ios/RCTWebRTC/WebRTCModule.m +3 -1
  16. package/lib/commonjs/EventEmitter.js +1 -19
  17. package/lib/commonjs/EventEmitter.js.map +1 -1
  18. package/lib/commonjs/MediaStreamTrack.js +15 -51
  19. package/lib/commonjs/MediaStreamTrack.js.map +1 -1
  20. package/lib/commonjs/index.js.map +1 -1
  21. package/lib/module/EventEmitter.js +1 -18
  22. package/lib/module/EventEmitter.js.map +1 -1
  23. package/lib/module/MediaStreamTrack.js +16 -52
  24. package/lib/module/MediaStreamTrack.js.map +1 -1
  25. package/lib/module/index.js.map +1 -1
  26. package/lib/typescript/EventEmitter.d.ts +0 -1
  27. package/lib/typescript/MediaStreamTrack.d.ts +13 -15
  28. package/lib/typescript/index.d.ts +2 -2
  29. package/livekit-react-native-webrtc.podspec +3 -3
  30. package/package.json +4 -4
  31. package/src/EventEmitter.ts +3 -21
  32. package/src/MediaStreamTrack.ts +40 -89
  33. package/src/index.ts +3 -1
package/README.md CHANGED
@@ -1,13 +1,34 @@
1
1
  [<img src="https://avatars.githubusercontent.com/u/42463376" alt="React Native WebRTC" style="height: 6em;" />](https://github.com/react-native-webrtc/react-native-webrtc)
2
2
 
3
- # React-Native-WebRTC
4
- [![npm version](https://img.shields.io/npm/v/@sbhjt-gr/react-native-webrtc)](https://www.npmjs.com/package/@sbhjt-gr/react-native-webrtc)
3
+ # React-Native-WebRTC for LiveKit (WiLang Fork)
4
+ [![npm version](https://img.shields.io/npm/v/@livekit/react-native-webrtc)](https://www.npmjs.com/package/@livekit/react-native-webrtc)
5
5
  [![Discourse topics](https://img.shields.io/discourse/topics?server=https%3A%2F%2Freact-native-webrtc.discourse.group%2F)](https://react-native-webrtc.discourse.group/)
6
6
 
7
- A WebRTC module for React Native.
7
+ A WebRTC module for React Native with native Palabra AI translation support.
8
8
 
9
9
  > [!NOTE]
10
- > This is a fork of [livekit/react-native-webrtc](https://github.com/livekit/react-native-webrtc) with additional audio interception features for real-time translation.
10
+ > This is a fork with integrated Palabra translation for real-time speech-to-speech translation during WebRTC calls.
11
+ > Used by WiLang app for video/voice call translation.
12
+
13
+ ## Palabra Translation
14
+
15
+ This fork adds native Palabra AI integration for translating remote peer audio in real-time:
16
+
17
+ ```javascript
18
+ import { startPalabraTranslation, stopPalabraTranslation } from '@livekit/react-native-webrtc';
19
+
20
+ await startPalabraTranslation(
21
+ peerConnectionId,
22
+ remoteAudioTrackId,
23
+ PALABRA_CLIENT_ID,
24
+ PALABRA_CLIENT_SECRET,
25
+ 'en',
26
+ 'es',
27
+ 'https://api.palabra.ai'
28
+ );
29
+
30
+ await stopPalabraTranslation();
31
+ ```
11
32
 
12
33
  ## Feature Overview
13
34
 
@@ -52,9 +73,9 @@ Software encode/decode factories have been enabled by default.
52
73
  Use one of the following preferred package install methods to immediately get going.
53
74
  Don't forget to follow platform guides below to cover any extra required steps.
54
75
 
55
- **npm:** `npm install @sbhjt-gr/react-native-webrtc --save`
56
- **yarn:** `yarn add @sbhjt-gr/react-native-webrtc`
57
- **pnpm:** `pnpm install @sbhjt-gr/react-native-webrtc`
76
+ **npm:** `npm install @livekit/react-native-webrtc --save`
77
+ **yarn:** `yarn add @livekit/react-native-webrtc`
78
+ **pnpm:** `pnpm install @livekit/react-native-webrtc`
58
79
 
59
80
  ## Guides
60
81
 
@@ -33,4 +33,5 @@ dependencies {
33
33
  implementation "com.facebook.react:react-android:+"
34
34
  api 'io.github.webrtc-sdk:android:137.7151.04'
35
35
  implementation "androidx.core:core:1.7.0"
36
+ implementation "com.squareup.okhttp3:okhttp:4.12.0"
36
37
  }
@@ -23,6 +23,9 @@ import com.facebook.react.module.annotations.ReactModule;
23
23
  import com.facebook.react.modules.core.DeviceEventManagerModule;
24
24
  import com.oney.WebRTCModule.webrtcutils.H264AndSoftwareVideoDecoderFactory;
25
25
  import com.oney.WebRTCModule.webrtcutils.H264AndSoftwareVideoEncoderFactory;
26
+ import com.oney.WebRTCModule.palabra.PalabraClient;
27
+ import com.oney.WebRTCModule.palabra.PalabraConfig;
28
+ import com.oney.WebRTCModule.palabra.PalabraListener;
26
29
 
27
30
  import org.webrtc.AddIceObserver;
28
31
  import org.webrtc.AudioProcessingFactory;
@@ -75,6 +78,8 @@ public class WebRTCModule extends ReactContextBaseJavaModule {
75
78
 
76
79
  private final GetUserMediaImpl getUserMediaImpl;
77
80
 
81
+ private PalabraClient palabraClient;
82
+
78
83
  public WebRTCModule(ReactApplicationContext reactContext) {
79
84
  super(reactContext);
80
85
 
@@ -953,29 +958,6 @@ public class WebRTCModule extends ReactContextBaseJavaModule {
953
958
  });
954
959
  }
955
960
 
956
- @ReactMethod
957
- public void mediaStreamTrackSetPlaybackEnabled(int pcId, String id, boolean enabled) {
958
- ThreadUtils.runOnExecutor(() -> {
959
- MediaStreamTrack track = getTrack(pcId, id);
960
- if (track == null) {
961
- Log.d(TAG, "mediaStreamTrackSetPlaybackEnabled() could not find track " + id);
962
- return;
963
- }
964
-
965
- if (!(track instanceof AudioTrack)) {
966
- Log.d(TAG, "mediaStreamTrackSetPlaybackEnabled() track is not an AudioTrack!");
967
- return;
968
- }
969
-
970
- ((AudioTrack) track).setVolume(enabled ? 1.0 : 0.0);
971
- });
972
- }
973
-
974
- @ReactMethod
975
- public void mediaStreamTrackEnableAudioSink(int pcId, String id, boolean enabled) {
976
- Log.d(TAG, "mediaStreamTrackEnableAudioSink() pcId=" + pcId + " trackId=" + id + " enabled=" + enabled + " - Not fully implemented on Android");
977
- }
978
-
979
961
  /**
980
962
  * This serializes the transceivers current direction and mid and returns them
981
963
  * for update when an sdp negotiation/renegotiation happens
@@ -1590,4 +1572,72 @@ public class WebRTCModule extends ReactContextBaseJavaModule {
1590
1572
  public void removeListeners(Integer count) {
1591
1573
  // Keep: Required for RN built in Event Emitter Calls.
1592
1574
  }
1575
+
1576
+ @ReactMethod
1577
+ public void startPalabraTranslation(int peerConnectionId, String trackId, String clientId,
1578
+ String clientSecret, String sourceLang, String targetLang,
1579
+ String apiUrl, Promise promise) {
1580
+ PeerConnectionObserver pco = mPeerConnectionObservers.get(peerConnectionId);
1581
+ if (pco == null) {
1582
+ promise.reject("E_INVALID", "pc_not_found");
1583
+ return;
1584
+ }
1585
+
1586
+ MediaStreamTrack track = pco.remoteTracks.get(trackId);
1587
+ if (track == null) {
1588
+ promise.reject("E_INVALID", "track_not_found");
1589
+ return;
1590
+ }
1591
+
1592
+ if (!(track instanceof AudioTrack)) {
1593
+ promise.reject("E_INVALID", "not_audio_track");
1594
+ return;
1595
+ }
1596
+
1597
+ AudioTrack audioTrack = (AudioTrack) track;
1598
+
1599
+ if (palabraClient != null) {
1600
+ palabraClient.stop();
1601
+ }
1602
+
1603
+ PalabraConfig config = new PalabraConfig(clientId, clientSecret, sourceLang, targetLang, apiUrl);
1604
+ palabraClient = new PalabraClient(getReactApplicationContext(), config);
1605
+ palabraClient.setListener(new PalabraListener() {
1606
+ @Override
1607
+ public void onTranscription(String text, String lang, boolean isFinal) {
1608
+ WritableMap params = Arguments.createMap();
1609
+ params.putString("text", text);
1610
+ params.putString("lang", lang);
1611
+ params.putBoolean("isFinal", isFinal);
1612
+ sendEvent("palabraTranscription", params);
1613
+ }
1614
+
1615
+ @Override
1616
+ public void onConnectionState(String state) {
1617
+ WritableMap params = Arguments.createMap();
1618
+ params.putString("state", state);
1619
+ sendEvent("palabraConnectionState", params);
1620
+ }
1621
+
1622
+ @Override
1623
+ public void onError(int code, String message) {
1624
+ WritableMap params = Arguments.createMap();
1625
+ params.putInt("code", code);
1626
+ params.putString("message", message);
1627
+ sendEvent("palabraError", params);
1628
+ }
1629
+ });
1630
+
1631
+ palabraClient.start(audioTrack);
1632
+ promise.resolve(null);
1633
+ }
1634
+
1635
+ @ReactMethod
1636
+ public void stopPalabraTranslation(Promise promise) {
1637
+ if (palabraClient != null) {
1638
+ palabraClient.stop();
1639
+ palabraClient = null;
1640
+ }
1641
+ promise.resolve(null);
1642
+ }
1593
1643
  }
@@ -0,0 +1,464 @@
1
+ package com.oney.WebRTCModule.palabra;
2
+
3
+ import android.content.Context;
4
+ import android.media.AudioFormat;
5
+ import android.media.AudioManager;
6
+ import android.media.AudioTrack;
7
+ import android.os.Handler;
8
+ import android.os.Looper;
9
+ import android.util.Base64;
10
+ import android.util.Log;
11
+
12
+ import org.json.JSONArray;
13
+ import org.json.JSONException;
14
+ import org.json.JSONObject;
15
+ import org.webrtc.AudioTrackSink;
16
+
17
+ import java.io.BufferedReader;
18
+ import java.io.ByteArrayOutputStream;
19
+ import java.io.IOException;
20
+ import java.io.InputStreamReader;
21
+ import java.io.OutputStream;
22
+ import java.net.HttpURLConnection;
23
+ import java.net.URL;
24
+ import java.nio.ByteBuffer;
25
+ import java.nio.ByteOrder;
26
+ import java.nio.charset.StandardCharsets;
27
+ import java.util.concurrent.ExecutorService;
28
+ import java.util.concurrent.Executors;
29
+ import java.util.concurrent.TimeUnit;
30
+ import java.util.concurrent.atomic.AtomicBoolean;
31
+
32
+ import okhttp3.OkHttpClient;
33
+ import okhttp3.Request;
34
+ import okhttp3.Response;
35
+ import okhttp3.WebSocket;
36
+ import okhttp3.WebSocketListener;
37
+
38
+ public class PalabraClient implements AudioTrackSink {
39
+ private static final String TAG = "PalabraClient";
40
+ private static final int SAMPLE_RATE_IN = 16000;
41
+ private static final int SAMPLE_RATE_OUT = 24000;
42
+ private static final int CHANNELS = 1;
43
+ private static final int CHUNK_MS = 320;
44
+ private static final int CHUNK_SAMPLES = SAMPLE_RATE_IN * CHUNK_MS / 1000;
45
+ private static final int CHUNK_BYTES = CHUNK_SAMPLES * 2;
46
+
47
+ private final Context context;
48
+ private final PalabraConfig config;
49
+ private PalabraListener listener;
50
+
51
+ private org.webrtc.AudioTrack remoteTrack;
52
+ private OkHttpClient httpClient;
53
+ private WebSocket webSocket;
54
+
55
+ private AudioTrack audioPlayer;
56
+ private final ExecutorService executor = Executors.newSingleThreadExecutor();
57
+ private final Handler mainHandler = new Handler(Looper.getMainLooper());
58
+
59
+ private String sessionId;
60
+ private String wsUrl;
61
+ private String publisherToken;
62
+
63
+ private AtomicBoolean connected = new AtomicBoolean(false);
64
+ private AtomicBoolean translating = new AtomicBoolean(false);
65
+
66
+ private ByteArrayOutputStream audioBuffer = new ByteArrayOutputStream();
67
+ private final Object bufferLock = new Object();
68
+
69
+ public PalabraClient(Context context, PalabraConfig config) {
70
+ this.context = context;
71
+ this.config = config;
72
+ this.httpClient = new OkHttpClient.Builder()
73
+ .connectTimeout(30, TimeUnit.SECONDS)
74
+ .readTimeout(30, TimeUnit.SECONDS)
75
+ .writeTimeout(30, TimeUnit.SECONDS)
76
+ .build();
77
+ setupAudioPlayer();
78
+ }
79
+
80
+ public void setListener(PalabraListener listener) {
81
+ this.listener = listener;
82
+ }
83
+
84
+ public boolean isConnected() {
85
+ return connected.get();
86
+ }
87
+
88
+ public boolean isTranslating() {
89
+ return translating.get();
90
+ }
91
+
92
+ private void setupAudioPlayer() {
93
+ int channelConfig = AudioFormat.CHANNEL_OUT_MONO;
94
+ int audioFormat = AudioFormat.ENCODING_PCM_16BIT;
95
+ int bufferSize = AudioTrack.getMinBufferSize(SAMPLE_RATE_OUT, channelConfig, audioFormat) * 2;
96
+
97
+ audioPlayer = new AudioTrack(
98
+ AudioManager.STREAM_VOICE_CALL,
99
+ SAMPLE_RATE_OUT,
100
+ channelConfig,
101
+ audioFormat,
102
+ bufferSize,
103
+ AudioTrack.MODE_STREAM
104
+ );
105
+ }
106
+
107
+ public void start(org.webrtc.AudioTrack remoteAudioTrack) {
108
+ if (translating.get()) {
109
+ return;
110
+ }
111
+
112
+ this.remoteTrack = remoteAudioTrack;
113
+ remoteAudioTrack.setVolume(0);
114
+
115
+ notifyConnectionState("connecting");
116
+
117
+ executor.execute(() -> {
118
+ try {
119
+ JSONObject session = createSession();
120
+ Log.d(TAG, "session_response: " + session.toString());
121
+ JSONObject data = session.getJSONObject("data");
122
+ sessionId = data.getString("id");
123
+ wsUrl = data.getString("ws_url");
124
+ publisherToken = data.getString("publisher");
125
+ Log.d(TAG, "ws_url: " + wsUrl);
126
+
127
+ mainHandler.post(this::connectWebSocket);
128
+ } catch (Exception e) {
129
+ Log.e(TAG, "session_create_failed", e);
130
+ mainHandler.post(() -> {
131
+ if (remoteAudioTrack != null) {
132
+ remoteAudioTrack.setVolume(1.0);
133
+ }
134
+ notifyError(500, e.getMessage());
135
+ });
136
+ }
137
+ });
138
+ }
139
+
140
+ private JSONObject createSession() throws IOException, JSONException {
141
+ URL url = new URL(config.apiUrl + "/session-storage/session");
142
+ HttpURLConnection conn = (HttpURLConnection) url.openConnection();
143
+ conn.setRequestMethod("POST");
144
+ conn.setRequestProperty("ClientId", config.clientId);
145
+ conn.setRequestProperty("ClientSecret", config.clientSecret);
146
+ conn.setRequestProperty("Content-Type", "application/json");
147
+ conn.setDoOutput(true);
148
+
149
+ JSONObject body = new JSONObject();
150
+ JSONObject bodyData = new JSONObject();
151
+ bodyData.put("subscriber_count", 0);
152
+ bodyData.put("publisher_can_subscribe", true);
153
+ body.put("data", bodyData);
154
+
155
+ try (OutputStream os = conn.getOutputStream()) {
156
+ os.write(body.toString().getBytes(StandardCharsets.UTF_8));
157
+ }
158
+
159
+ int responseCode = conn.getResponseCode();
160
+ if (responseCode < 200 || responseCode >= 300) {
161
+ throw new IOException("session_http_error_" + responseCode);
162
+ }
163
+
164
+ StringBuilder response = new StringBuilder();
165
+ try (BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream()))) {
166
+ String line;
167
+ while ((line = br.readLine()) != null) {
168
+ response.append(line);
169
+ }
170
+ }
171
+
172
+ return new JSONObject(response.toString());
173
+ }
174
+
175
+ private void connectWebSocket() {
176
+ String endpoint = wsUrl + "?token=" + publisherToken;
177
+ Log.d(TAG, "connecting_ws: " + endpoint);
178
+
179
+ Request request = new Request.Builder()
180
+ .url(endpoint)
181
+ .build();
182
+
183
+ webSocket = httpClient.newWebSocket(request, new WebSocketListener() {
184
+ @Override
185
+ public void onOpen(WebSocket ws, Response response) {
186
+ Log.d(TAG, "ws_open");
187
+ connected.set(true);
188
+ translating.set(true);
189
+
190
+ remoteTrack.addSink(PalabraClient.this);
191
+ audioPlayer.play();
192
+
193
+ mainHandler.post(() -> notifyConnectionState("connected"));
194
+
195
+ mainHandler.postDelayed(() -> sendSetTask(), 500);
196
+ }
197
+
198
+ @Override
199
+ public void onMessage(WebSocket ws, String text) {
200
+ handleMessage(text);
201
+ }
202
+
203
+ @Override
204
+ public void onFailure(WebSocket ws, Throwable t, Response response) {
205
+ Log.e(TAG, "ws_error", t);
206
+ mainHandler.post(() -> {
207
+ stop();
208
+ notifyError(500, t.getMessage());
209
+ });
210
+ }
211
+
212
+ @Override
213
+ public void onClosed(WebSocket ws, int code, String reason) {
214
+ Log.d(TAG, "ws_closed: " + code);
215
+ mainHandler.post(() -> stop());
216
+ }
217
+ });
218
+ }
219
+
220
+ private void sendSetTask() {
221
+ if (webSocket == null || !connected.get()) {
222
+ return;
223
+ }
224
+
225
+ try {
226
+ JSONObject msg = new JSONObject();
227
+ msg.put("message_type", "set_task");
228
+
229
+ JSONObject data = new JSONObject();
230
+
231
+ JSONObject inputStream = new JSONObject();
232
+ inputStream.put("content_type", "audio");
233
+ JSONObject source = new JSONObject();
234
+ source.put("type", "ws");
235
+ source.put("format", "pcm_s16le");
236
+ source.put("sample_rate", SAMPLE_RATE_IN);
237
+ source.put("channels", CHANNELS);
238
+ inputStream.put("source", source);
239
+ data.put("input_stream", inputStream);
240
+
241
+ JSONObject outputStream = new JSONObject();
242
+ outputStream.put("content_type", "audio");
243
+ JSONObject target = new JSONObject();
244
+ target.put("type", "ws");
245
+ target.put("format", "pcm_s16le");
246
+ outputStream.put("target", target);
247
+ data.put("output_stream", outputStream);
248
+
249
+ JSONObject pipeline = new JSONObject();
250
+
251
+ JSONObject transcription = new JSONObject();
252
+ transcription.put("source_language", config.sourceLang);
253
+ pipeline.put("transcription", transcription);
254
+
255
+ JSONArray translations = new JSONArray();
256
+ JSONObject translation = new JSONObject();
257
+ translation.put("target_language", config.targetLang);
258
+ JSONObject speechGen = new JSONObject();
259
+ speechGen.put("voice_cloning", false);
260
+ translation.put("speech_generation", speechGen);
261
+ translations.put(translation);
262
+ pipeline.put("translations", translations);
263
+
264
+ JSONArray allowedTypes = new JSONArray();
265
+ allowedTypes.put("partial_transcription");
266
+ allowedTypes.put("validated_transcription");
267
+ allowedTypes.put("translated_transcription");
268
+ pipeline.put("allowed_message_types", allowedTypes);
269
+
270
+ data.put("pipeline", pipeline);
271
+ msg.put("data", data);
272
+
273
+ String payload = msg.toString();
274
+ Log.d(TAG, "set_task: " + payload);
275
+ webSocket.send(payload);
276
+ } catch (JSONException e) {
277
+ Log.e(TAG, "set_task_error", e);
278
+ }
279
+ }
280
+
281
+ private void handleMessage(String text) {
282
+ try {
283
+ JSONObject json = new JSONObject(text);
284
+ String type = json.optString("message_type", "");
285
+
286
+ if ("output_audio_data".equals(type)) {
287
+ JSONObject data = json.getJSONObject("data");
288
+ String audioBase64 = data.optString("data", "");
289
+ if (!audioBase64.isEmpty()) {
290
+ byte[] audioBytes = Base64.decode(audioBase64, Base64.DEFAULT);
291
+ if (audioPlayer != null && translating.get()) {
292
+ audioPlayer.write(audioBytes, 0, audioBytes.length);
293
+ }
294
+ }
295
+ } else if (type.contains("transcription")) {
296
+ JSONObject data = json.getJSONObject("data");
297
+ JSONObject transcription = data.optJSONObject("transcription");
298
+ if (transcription != null) {
299
+ String txt = transcription.optString("text", "");
300
+ String lang = transcription.optString("language", "");
301
+ boolean isFinal = !"partial_transcription".equals(type);
302
+ mainHandler.post(() -> notifyTranscription(txt, lang, isFinal));
303
+ }
304
+ } else if ("error".equals(type)) {
305
+ JSONObject data = json.optJSONObject("data");
306
+ String desc = data != null ? data.optString("desc", "unknown") : "unknown";
307
+ Log.e(TAG, "palabra_error: " + desc);
308
+ mainHandler.post(() -> notifyError(500, desc));
309
+ }
310
+ } catch (JSONException e) {
311
+ Log.e(TAG, "msg_parse_error", e);
312
+ }
313
+ }
314
+
315
+ public void stop() {
316
+ if (!translating.getAndSet(false)) {
317
+ return;
318
+ }
319
+
320
+ connected.set(false);
321
+
322
+ if (remoteTrack != null) {
323
+ try {
324
+ remoteTrack.removeSink(this);
325
+ remoteTrack.setVolume(1.0);
326
+ } catch (Exception e) {
327
+ Log.w(TAG, "stop_track_cleanup_error: " + e.getMessage());
328
+ }
329
+ }
330
+
331
+ if (webSocket != null) {
332
+ try {
333
+ JSONObject endMsg = new JSONObject();
334
+ endMsg.put("message_type", "end_task");
335
+ endMsg.put("data", new JSONObject().put("force", false));
336
+ webSocket.send(endMsg.toString());
337
+ } catch (JSONException e) {
338
+ Log.e(TAG, "end_task_error", e);
339
+ }
340
+ try {
341
+ webSocket.close(1000, "stop");
342
+ } catch (Exception e) {
343
+ Log.w(TAG, "websocket_close_error: " + e.getMessage());
344
+ }
345
+ webSocket = null;
346
+ }
347
+
348
+ if (audioPlayer != null) {
349
+ try {
350
+ audioPlayer.stop();
351
+ } catch (Exception e) {
352
+ Log.w(TAG, "audio_player_stop_error: " + e.getMessage());
353
+ }
354
+ }
355
+
356
+ synchronized (bufferLock) {
357
+ audioBuffer.reset();
358
+ }
359
+
360
+ remoteTrack = null;
361
+ notifyConnectionState("disconnected");
362
+ }
363
+
364
+ @Override
365
+ public void onData(ByteBuffer audioData, int bitsPerSample, int sampleRate, int channels, int frames, long timestamp) {
366
+ if (!translating.get() || webSocket == null) {
367
+ return;
368
+ }
369
+
370
+ byte[] samples = new byte[audioData.remaining()];
371
+ audioData.get(samples);
372
+
373
+ byte[] resampled = resample(samples, sampleRate, channels, SAMPLE_RATE_IN, CHANNELS);
374
+
375
+ synchronized (bufferLock) {
376
+ try {
377
+ audioBuffer.write(resampled);
378
+
379
+ while (audioBuffer.size() >= CHUNK_BYTES) {
380
+ byte[] chunk = new byte[CHUNK_BYTES];
381
+ byte[] all = audioBuffer.toByteArray();
382
+ System.arraycopy(all, 0, chunk, 0, CHUNK_BYTES);
383
+
384
+ audioBuffer.reset();
385
+ if (all.length > CHUNK_BYTES) {
386
+ audioBuffer.write(all, CHUNK_BYTES, all.length - CHUNK_BYTES);
387
+ }
388
+
389
+ sendAudioChunk(chunk);
390
+ }
391
+ } catch (IOException e) {
392
+ Log.e(TAG, "buffer_error", e);
393
+ }
394
+ }
395
+ }
396
+
397
+ private byte[] resample(byte[] input, int srcRate, int srcChannels, int dstRate, int dstChannels) {
398
+ if (srcRate == dstRate && srcChannels == dstChannels) {
399
+ return input;
400
+ }
401
+
402
+ int srcSamples = input.length / (2 * srcChannels);
403
+ int dstSamples = (int) ((long) srcSamples * dstRate / srcRate);
404
+
405
+ short[] srcData = new short[srcSamples * srcChannels];
406
+ ByteBuffer.wrap(input).order(ByteOrder.LITTLE_ENDIAN).asShortBuffer().get(srcData);
407
+
408
+ short[] monoSrc = srcData;
409
+ if (srcChannels == 2) {
410
+ monoSrc = new short[srcSamples];
411
+ for (int i = 0; i < srcSamples; i++) {
412
+ monoSrc[i] = (short) ((srcData[i * 2] + srcData[i * 2 + 1]) / 2);
413
+ }
414
+ }
415
+
416
+ short[] dstData = new short[dstSamples];
417
+ for (int i = 0; i < dstSamples; i++) {
418
+ float srcIdx = (float) i * (monoSrc.length - 1) / (dstSamples - 1);
419
+ int idx0 = (int) srcIdx;
420
+ int idx1 = Math.min(idx0 + 1, monoSrc.length - 1);
421
+ float frac = srcIdx - idx0;
422
+ dstData[i] = (short) (monoSrc[idx0] * (1 - frac) + monoSrc[idx1] * frac);
423
+ }
424
+
425
+ byte[] output = new byte[dstSamples * 2];
426
+ ByteBuffer.wrap(output).order(ByteOrder.LITTLE_ENDIAN).asShortBuffer().put(dstData);
427
+ return output;
428
+ }
429
+
430
+ private void sendAudioChunk(byte[] chunk) {
431
+ if (webSocket == null || !connected.get()) {
432
+ return;
433
+ }
434
+
435
+ try {
436
+ JSONObject msg = new JSONObject();
437
+ msg.put("message_type", "input_audio_data");
438
+ JSONObject data = new JSONObject();
439
+ data.put("data", Base64.encodeToString(chunk, Base64.NO_WRAP));
440
+ msg.put("data", data);
441
+ webSocket.send(msg.toString());
442
+ } catch (JSONException e) {
443
+ Log.e(TAG, "send_audio_error", e);
444
+ }
445
+ }
446
+
447
+ private void notifyConnectionState(String state) {
448
+ if (listener != null) {
449
+ listener.onConnectionState(state);
450
+ }
451
+ }
452
+
453
+ private void notifyError(int code, String message) {
454
+ if (listener != null) {
455
+ listener.onError(code, message);
456
+ }
457
+ }
458
+
459
+ private void notifyTranscription(String text, String lang, boolean isFinal) {
460
+ if (listener != null) {
461
+ listener.onTranscription(text, lang, isFinal);
462
+ }
463
+ }
464
+ }
@@ -0,0 +1,17 @@
1
+ package com.oney.WebRTCModule.palabra;
2
+
3
+ public class PalabraConfig {
4
+ public final String clientId;
5
+ public final String clientSecret;
6
+ public final String sourceLang;
7
+ public final String targetLang;
8
+ public final String apiUrl;
9
+
10
+ public PalabraConfig(String clientId, String clientSecret, String sourceLang, String targetLang, String apiUrl) {
11
+ this.clientId = clientId;
12
+ this.clientSecret = clientSecret;
13
+ this.sourceLang = sourceLang;
14
+ this.targetLang = targetLang;
15
+ this.apiUrl = apiUrl;
16
+ }
17
+ }
@@ -0,0 +1,7 @@
1
+ package com.oney.WebRTCModule.palabra;
2
+
3
+ public interface PalabraListener {
4
+ void onTranscription(String text, String lang, boolean isFinal);
5
+ void onConnectionState(String state);
6
+ void onError(int code, String message);
7
+ }
@@ -0,0 +1,13 @@
1
+ #import <Foundation/Foundation.h>
2
+ #import <WebRTC/WebRTC.h>
3
+ #import <AVFoundation/AVFoundation.h>
4
+
5
+ @class PalabraClient;
6
+
7
+ @interface PalabraAudioSink : NSObject <RTCAudioRenderer>
8
+
9
+ @property (nonatomic, weak) PalabraClient *client;
10
+
11
+ - (instancetype)initWithClient:(PalabraClient *)client;
12
+
13
+ @end