@iternio/react-native-auto-play 0.4.11 → 0.4.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -0
- package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridAutoPlay.kt +0 -89
- package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridVoice.kt +97 -0
- package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/VoiceInputManager.kt +286 -20
- package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/utils/ThreadUtil.kt +6 -13
- package/ios/hybrid/HybridAutoPlay.swift +2 -47
- package/ios/hybrid/HybridVoice.swift +65 -0
- package/ios/utils/VoiceInputManager.swift +144 -40
- package/lib/HybridAutoPlay.d.ts +2 -0
- package/lib/HybridAutoPlay.js +2 -0
- package/lib/components/OnAppearedChildRenderer.d.ts +10 -0
- package/lib/components/OnAppearedChildRenderer.js +26 -0
- package/lib/hooks/useIsAutoPlayFocused.d.ts +7 -0
- package/lib/hooks/useIsAutoPlayFocused.js +20 -0
- package/lib/hybrid/HybridVoice.d.ts +52 -0
- package/lib/hybrid/HybridVoice.js +52 -0
- package/lib/hybrid.d.ts +2 -0
- package/lib/hybrid.js +2 -0
- package/lib/index.d.ts +3 -1
- package/lib/index.js +2 -1
- package/lib/specs/AutoPlay.nitro.d.ts +0 -29
- package/lib/specs/AutomotivePermissionRequestTemplate.d.ts +11 -0
- package/lib/specs/AutomotivePermissionRequestTemplate.js +1 -0
- package/lib/specs/AutomotivePermissionRequestTemplate.nitro.d.ts +11 -0
- package/lib/specs/AutomotivePermissionRequestTemplate.nitro.js +1 -0
- package/lib/specs/Voice.nitro.d.ts +11 -0
- package/lib/specs/Voice.nitro.js +1 -0
- package/lib/templates/AutomotivePermissionRequestTemplate.d.ts +23 -0
- package/lib/templates/AutomotivePermissionRequestTemplate.js +18 -0
- package/lib/types/Glyphmap.d.ts +4105 -0
- package/lib/types/Glyphmap.js +4105 -0
- package/lib/types/Voice.d.ts +16 -0
- package/lib/types/Voice.js +1 -0
- package/nitro.json +10 -0
- package/nitrogen/generated/android/ReactNativeAutoPlay+autolinking.cmake +2 -0
- package/nitrogen/generated/android/ReactNativeAutoPlayOnLoad.cpp +18 -0
- package/nitrogen/generated/android/c++/JFunc_void_VoiceInputChunk.hpp +81 -0
- package/nitrogen/generated/android/c++/JHybridAutoPlaySpec.cpp +0 -43
- package/nitrogen/generated/android/c++/JHybridAutoPlaySpec.hpp +0 -4
- package/nitrogen/generated/android/c++/JHybridVoiceSpec.cpp +104 -0
- package/nitrogen/generated/android/c++/JHybridVoiceSpec.hpp +66 -0
- package/nitrogen/generated/android/c++/JVoiceInputChunk.hpp +64 -0
- package/nitrogen/generated/android/c++/JVoiceInputResult.hpp +64 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/swe/iternio/reactnativeautoplay/Func_void_VoiceInputChunk.kt +80 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridAutoPlaySpec.kt +0 -17
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridVoiceSpec.kt +72 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/swe/iternio/reactnativeautoplay/VoiceInputChunk.kt +56 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/swe/iternio/reactnativeautoplay/VoiceInputResult.kt +56 -0
- package/nitrogen/generated/ios/ReactNativeAutoPlay-Swift-Cxx-Bridge.cpp +41 -16
- package/nitrogen/generated/ios/ReactNativeAutoPlay-Swift-Cxx-Bridge.hpp +201 -126
- package/nitrogen/generated/ios/ReactNativeAutoPlay-Swift-Cxx-Umbrella.hpp +11 -0
- package/nitrogen/generated/ios/ReactNativeAutoPlayAutolinking.mm +8 -0
- package/nitrogen/generated/ios/ReactNativeAutoPlayAutolinking.swift +12 -0
- package/nitrogen/generated/ios/c++/HybridAutoPlaySpecSwift.hpp +0 -34
- package/nitrogen/generated/ios/c++/HybridVoiceSpecSwift.cpp +11 -0
- package/nitrogen/generated/ios/c++/HybridVoiceSpecSwift.hpp +116 -0
- package/nitrogen/generated/ios/swift/Func_void_VoiceInputChunk.swift +46 -0
- package/nitrogen/generated/ios/swift/{Func_void_std__shared_ptr_ArrayBuffer_.swift → Func_void_VoiceInputResult.swift} +10 -10
- package/nitrogen/generated/ios/swift/Func_void_bool.swift +5 -5
- package/nitrogen/generated/ios/swift/HybridAutoPlaySpec.swift +0 -4
- package/nitrogen/generated/ios/swift/HybridAutoPlaySpec_cxx.swift +0 -82
- package/nitrogen/generated/ios/swift/HybridVoiceSpec.swift +58 -0
- package/nitrogen/generated/ios/swift/HybridVoiceSpec_cxx.swift +234 -0
- package/nitrogen/generated/ios/swift/VoiceInputChunk.swift +60 -0
- package/nitrogen/generated/ios/swift/VoiceInputResult.swift +60 -0
- package/nitrogen/generated/shared/c++/HybridAutoPlaySpec.cpp +0 -4
- package/nitrogen/generated/shared/c++/HybridAutoPlaySpec.hpp +0 -5
- package/nitrogen/generated/shared/c++/HybridVoiceSpec.cpp +24 -0
- package/nitrogen/generated/shared/c++/HybridVoiceSpec.hpp +73 -0
- package/nitrogen/generated/shared/c++/VoiceInputChunk.hpp +89 -0
- package/nitrogen/generated/shared/c++/VoiceInputResult.hpp +89 -0
- package/package.json +1 -1
- package/src/hybrid/HybridVoice.ts +79 -0
- package/src/index.ts +3 -1
- package/src/specs/AutoPlay.nitro.ts +0 -37
- package/src/specs/Voice.nitro.ts +16 -0
- package/src/types/Voice.ts +18 -0
package/README.md
CHANGED
|
@@ -314,6 +314,32 @@ The library does **not** bundle any icon font — the consuming app must provide
|
|
|
314
314
|
|
|
315
315
|
For cross-platform compatibility use **lowercase names with underscores only** (e.g. `material_symbols`).
|
|
316
316
|
|
|
317
|
+
**or**
|
|
318
|
+
|
|
319
|
+
1. use expo-font
|
|
320
|
+
```js
|
|
321
|
+
[
|
|
322
|
+
'expo-font',
|
|
323
|
+
{
|
|
324
|
+
android: {
|
|
325
|
+
fonts: [
|
|
326
|
+
{
|
|
327
|
+
fontFamily: 'MaterialSymbols',
|
|
328
|
+
fontDefinitions: [
|
|
329
|
+
{
|
|
330
|
+
path: './assets/fonts/material_symbols.ttf',
|
|
331
|
+
weight: 800,
|
|
332
|
+
},
|
|
333
|
+
],
|
|
334
|
+
},
|
|
335
|
+
],
|
|
336
|
+
},
|
|
337
|
+
ios: ['./assets/fonts/material_symbols.ttf'],
|
|
338
|
+
},
|
|
339
|
+
],
|
|
340
|
+
```
|
|
341
|
+
For cross-platform compatibility use **lowercase names with underscores only** (e.g. `material_symbols`).
|
|
342
|
+
|
|
317
343
|
2. Register the font and an optional glyph map at startup:
|
|
318
344
|
|
|
319
345
|
```ts
|
package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridAutoPlay.kt
CHANGED
|
@@ -1,19 +1,11 @@
|
|
|
1
1
|
package com.margelo.nitro.swe.iternio.reactnativeautoplay
|
|
2
2
|
|
|
3
|
-
import android.content.pm.PackageManager
|
|
4
3
|
import android.os.Build
|
|
5
|
-
import androidx.core.content.ContextCompat
|
|
6
4
|
import com.facebook.react.bridge.UiThreadUtil
|
|
7
|
-
import com.facebook.react.modules.core.PermissionAwareActivity
|
|
8
|
-
import com.facebook.react.modules.core.PermissionListener
|
|
9
|
-
import com.margelo.nitro.NitroModules
|
|
10
|
-
import com.margelo.nitro.core.ArrayBuffer
|
|
11
5
|
import com.margelo.nitro.core.Promise
|
|
12
6
|
import com.margelo.nitro.swe.iternio.reactnativeautoplay.template.AndroidAutoTemplate
|
|
13
7
|
import com.margelo.nitro.swe.iternio.reactnativeautoplay.template.MessageTemplate
|
|
14
8
|
import com.margelo.nitro.swe.iternio.reactnativeautoplay.utils.ThreadUtil
|
|
15
|
-
import kotlinx.coroutines.suspendCancellableCoroutine
|
|
16
|
-
import java.nio.ByteBuffer
|
|
17
9
|
import java.util.concurrent.ConcurrentHashMap
|
|
18
10
|
import java.util.concurrent.CopyOnWriteArrayList
|
|
19
11
|
import kotlin.coroutines.resume
|
|
@@ -255,84 +247,6 @@ class HybridAutoPlay : HybridAutoPlaySpec() {
|
|
|
255
247
|
}
|
|
256
248
|
}
|
|
257
249
|
|
|
258
|
-
override fun hasVoiceInputPermission(): Boolean {
|
|
259
|
-
val context = NitroModules.applicationContext ?: return false
|
|
260
|
-
return ContextCompat.checkSelfPermission(
|
|
261
|
-
context, android.Manifest.permission.RECORD_AUDIO
|
|
262
|
-
) == PackageManager.PERMISSION_GRANTED
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
override fun requestVoiceInputPermission(): Promise<Boolean> {
|
|
266
|
-
return Promise.async {
|
|
267
|
-
if (hasVoiceInputPermission()) {
|
|
268
|
-
return@async true
|
|
269
|
-
}
|
|
270
|
-
|
|
271
|
-
val carContext = AndroidAutoSession.getRootContext()
|
|
272
|
-
|
|
273
|
-
if (carContext != null) {
|
|
274
|
-
suspendCancellableCoroutine {
|
|
275
|
-
carContext.requestPermissions(listOf(android.Manifest.permission.RECORD_AUDIO)) { approved, _ ->
|
|
276
|
-
it.resume(approved.contains(android.Manifest.permission.RECORD_AUDIO))
|
|
277
|
-
}
|
|
278
|
-
}
|
|
279
|
-
} else {
|
|
280
|
-
val context = NitroModules.applicationContext ?: return@async false
|
|
281
|
-
val activity =
|
|
282
|
-
context.currentActivity as? PermissionAwareActivity ?: return@async false
|
|
283
|
-
val code = (Math.random() * 10000).toInt()
|
|
284
|
-
|
|
285
|
-
suspendCancellableCoroutine {
|
|
286
|
-
activity.requestPermissions(
|
|
287
|
-
arrayOf(android.Manifest.permission.RECORD_AUDIO),
|
|
288
|
-
code,
|
|
289
|
-
PermissionListener { requestCode, _, grantResults ->
|
|
290
|
-
if (requestCode != code) {
|
|
291
|
-
return@PermissionListener false
|
|
292
|
-
}
|
|
293
|
-
|
|
294
|
-
val granted =
|
|
295
|
-
grantResults.isNotEmpty() && grantResults.first() == PackageManager.PERMISSION_GRANTED
|
|
296
|
-
|
|
297
|
-
it.resume(granted)
|
|
298
|
-
|
|
299
|
-
return@PermissionListener true
|
|
300
|
-
})
|
|
301
|
-
}
|
|
302
|
-
}
|
|
303
|
-
}
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
override fun startVoiceInput(
|
|
307
|
-
silenceThresholdMs: Double?, maxDurationMs: Double?, listeningText: String?
|
|
308
|
-
): Promise<ArrayBuffer> {
|
|
309
|
-
return Promise.async {
|
|
310
|
-
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) {
|
|
311
|
-
throw UnsupportedOperationException("startVoiceInput requires at least API level ${Build.VERSION_CODES.O}")
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
val manager = VoiceInputManager(AndroidAutoSession.getRootContext())
|
|
315
|
-
voiceInputManager = manager
|
|
316
|
-
|
|
317
|
-
try {
|
|
318
|
-
val pcmBytes = manager.start(
|
|
319
|
-
silenceThresholdMs = silenceThresholdMs?.toLong() ?: 1_500L,
|
|
320
|
-
maxDurationMs = maxDurationMs?.toLong() ?: 10_000L,
|
|
321
|
-
)
|
|
322
|
-
val directBuffer =
|
|
323
|
-
ByteBuffer.allocateDirect(pcmBytes.size).put(pcmBytes).rewind() as ByteBuffer
|
|
324
|
-
ArrayBuffer.wrap(directBuffer)
|
|
325
|
-
} finally {
|
|
326
|
-
voiceInputManager = null
|
|
327
|
-
manager.dispose()
|
|
328
|
-
}
|
|
329
|
-
}
|
|
330
|
-
}
|
|
331
|
-
|
|
332
|
-
override fun stopVoiceInput() {
|
|
333
|
-
voiceInputManager?.stop()
|
|
334
|
-
}
|
|
335
|
-
|
|
336
250
|
companion object {
|
|
337
251
|
const val TAG = "HybridAutoPlay"
|
|
338
252
|
|
|
@@ -343,9 +257,6 @@ class HybridAutoPlay : HybridAutoPlaySpec() {
|
|
|
343
257
|
|
|
344
258
|
private val voiceInputListeners = CopyOnWriteArrayList<(Location?, String?) -> Unit>()
|
|
345
259
|
|
|
346
|
-
@Volatile
|
|
347
|
-
private var voiceInputManager: VoiceInputManager? = null
|
|
348
|
-
|
|
349
260
|
private val safeAreaInsetsListeners =
|
|
350
261
|
ConcurrentHashMap<String, CopyOnWriteArrayList<(SafeAreaInsets) -> Unit>>()
|
|
351
262
|
|
package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridVoice.kt
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
package com.margelo.nitro.swe.iternio.reactnativeautoplay
|
|
2
|
+
|
|
3
|
+
import android.content.pm.PackageManager
|
|
4
|
+
import android.os.Build
|
|
5
|
+
import androidx.core.content.ContextCompat
|
|
6
|
+
import com.facebook.react.modules.core.PermissionAwareActivity
|
|
7
|
+
import com.facebook.react.modules.core.PermissionListener
|
|
8
|
+
import com.margelo.nitro.NitroModules
|
|
9
|
+
import com.margelo.nitro.core.Promise
|
|
10
|
+
import kotlinx.coroutines.suspendCancellableCoroutine
|
|
11
|
+
import kotlin.coroutines.resume
|
|
12
|
+
|
|
13
|
+
class HybridVoice : HybridVoiceSpec() {
|
|
14
|
+
@Volatile
|
|
15
|
+
private var voiceInputManager: VoiceInputManager? = null
|
|
16
|
+
|
|
17
|
+
override fun hasVoiceInputPermission(): Boolean {
|
|
18
|
+
return VoiceInputManager.hasVoiceInputPermission()
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
override fun requestVoiceInputPermission(): Promise<Boolean> {
|
|
22
|
+
return Promise.async {
|
|
23
|
+
if (hasVoiceInputPermission()) {
|
|
24
|
+
return@async true
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
val carContext = AndroidAutoSession.getRootContext()
|
|
28
|
+
|
|
29
|
+
if (carContext != null) {
|
|
30
|
+
suspendCancellableCoroutine { cont ->
|
|
31
|
+
carContext.requestPermissions(
|
|
32
|
+
listOf(android.Manifest.permission.RECORD_AUDIO)
|
|
33
|
+
) { approved, _ ->
|
|
34
|
+
cont.resume(approved.contains(android.Manifest.permission.RECORD_AUDIO))
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
} else {
|
|
38
|
+
val context = NitroModules.applicationContext ?: return@async false
|
|
39
|
+
val activity =
|
|
40
|
+
context.currentActivity as? PermissionAwareActivity ?: return@async false
|
|
41
|
+
val code = (Math.random() * 10000).toInt()
|
|
42
|
+
|
|
43
|
+
suspendCancellableCoroutine { cont ->
|
|
44
|
+
activity.requestPermissions(
|
|
45
|
+
arrayOf(android.Manifest.permission.RECORD_AUDIO),
|
|
46
|
+
code,
|
|
47
|
+
PermissionListener { requestCode, _, grantResults ->
|
|
48
|
+
if (requestCode != code) {
|
|
49
|
+
return@PermissionListener false
|
|
50
|
+
}
|
|
51
|
+
cont.resume(
|
|
52
|
+
grantResults.isNotEmpty() &&
|
|
53
|
+
grantResults.first() == PackageManager.PERMISSION_GRANTED
|
|
54
|
+
)
|
|
55
|
+
true
|
|
56
|
+
}
|
|
57
|
+
)
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
override fun startVoiceInput(
|
|
64
|
+
silenceThresholdMs: Double?,
|
|
65
|
+
maxDurationMs: Double?,
|
|
66
|
+
listeningText: String?,
|
|
67
|
+
preferSpeechToText: Boolean?,
|
|
68
|
+
onChunk: ((chunk: VoiceInputChunk) -> Unit)?,
|
|
69
|
+
language: String?
|
|
70
|
+
): Promise<VoiceInputResult> {
|
|
71
|
+
return Promise.async {
|
|
72
|
+
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) {
|
|
73
|
+
throw UnsupportedOperationException("startVoiceInput requires at least API level ${Build.VERSION_CODES.O}")
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
val manager = VoiceInputManager(AndroidAutoSession.getRootContext())
|
|
77
|
+
voiceInputManager = manager
|
|
78
|
+
|
|
79
|
+
try {
|
|
80
|
+
manager.start(
|
|
81
|
+
silenceThresholdMs = silenceThresholdMs?.toLong() ?: 1_500L,
|
|
82
|
+
maxDurationMs = maxDurationMs?.toLong() ?: 10_000L,
|
|
83
|
+
preferSpeechToText = preferSpeechToText ?: false,
|
|
84
|
+
onChunk = onChunk,
|
|
85
|
+
language = language
|
|
86
|
+
)
|
|
87
|
+
} finally {
|
|
88
|
+
voiceInputManager = null
|
|
89
|
+
manager.dispose()
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
override fun stopVoiceInput() {
|
|
95
|
+
voiceInputManager?.stop()
|
|
96
|
+
}
|
|
97
|
+
}
|
package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/VoiceInputManager.kt
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
package com.margelo.nitro.swe.iternio.reactnativeautoplay
|
|
2
2
|
|
|
3
|
+
import android.annotation.SuppressLint
|
|
4
|
+
import android.content.Intent
|
|
3
5
|
import android.content.pm.PackageManager
|
|
4
6
|
import android.media.AudioAttributes
|
|
5
7
|
import android.media.AudioFocusRequest
|
|
@@ -8,18 +10,28 @@ import android.media.AudioManager
|
|
|
8
10
|
import android.media.AudioRecord
|
|
9
11
|
import android.media.MediaRecorder
|
|
10
12
|
import android.os.Build
|
|
13
|
+
import android.os.Bundle
|
|
14
|
+
import android.os.ParcelFileDescriptor
|
|
15
|
+
import android.speech.RecognitionListener
|
|
16
|
+
import android.speech.RecognizerIntent
|
|
17
|
+
import android.speech.SpeechRecognizer
|
|
11
18
|
import androidx.annotation.RequiresApi
|
|
12
19
|
import androidx.car.app.CarContext
|
|
13
20
|
import androidx.car.app.media.CarAudioRecord
|
|
14
21
|
import androidx.core.content.ContextCompat
|
|
22
|
+
import com.facebook.react.bridge.UiThreadUtil
|
|
15
23
|
import com.margelo.nitro.NitroModules
|
|
24
|
+
import com.margelo.nitro.core.ArrayBuffer
|
|
25
|
+
import com.margelo.nitro.swe.iternio.reactnativeautoplay.utils.ThreadUtil
|
|
16
26
|
import kotlinx.coroutines.CoroutineScope
|
|
17
27
|
import kotlinx.coroutines.Dispatchers
|
|
18
28
|
import kotlinx.coroutines.Job
|
|
29
|
+
import kotlinx.coroutines.async
|
|
19
30
|
import kotlinx.coroutines.cancel
|
|
20
31
|
import kotlinx.coroutines.launch
|
|
21
32
|
import kotlinx.coroutines.suspendCancellableCoroutine
|
|
22
33
|
import java.io.ByteArrayOutputStream
|
|
34
|
+
import java.nio.ByteBuffer
|
|
23
35
|
import kotlin.coroutines.Continuation
|
|
24
36
|
import kotlin.coroutines.resume
|
|
25
37
|
import kotlin.coroutines.resumeWithException
|
|
@@ -27,43 +39,274 @@ import kotlin.math.abs
|
|
|
27
39
|
|
|
28
40
|
/**
|
|
29
41
|
* Captures 16-bit PCM audio (16 kHz, mono).
|
|
30
|
-
* When [carContext] is provided uses CarAudioRecord (Android Auto/Automotive)
|
|
31
|
-
*
|
|
42
|
+
* When [carContext] is provided uses CarAudioRecord (Android Auto/Automotive),
|
|
43
|
+
* otherwise falls back to standard AudioRecord.
|
|
44
|
+
*
|
|
45
|
+
* When preferSpeechToText is true and SpeechRecognizer is available, it owns
|
|
46
|
+
* the microphone and streams partial results; the PCM path is not used.
|
|
47
|
+
* When SpeechRecognizer is unavailable the manager falls back to PCM recording.
|
|
32
48
|
*/
|
|
33
49
|
class VoiceInputManager(
|
|
34
50
|
private val carContext: CarContext?,
|
|
35
51
|
) {
|
|
52
|
+
// PCM recording state
|
|
36
53
|
private var carAudioRecord: CarAudioRecord? = null
|
|
37
54
|
private var audioRecord: AudioRecord? = null
|
|
38
55
|
private var audioFocusRequest: AudioFocusRequest? = null
|
|
39
56
|
private var recordingJob: Job? = null
|
|
40
|
-
private var
|
|
57
|
+
private var pcmContinuation: Continuation<ByteArray>? = null
|
|
41
58
|
private val scope = CoroutineScope(Dispatchers.IO)
|
|
42
59
|
|
|
43
60
|
@Volatile
|
|
44
61
|
private var isRecording = false
|
|
45
62
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
*/
|
|
63
|
+
// STT state — only set when SpeechRecognizer owns the mic
|
|
64
|
+
@Volatile
|
|
65
|
+
private var activeSpeechRecognizer: SpeechRecognizer? = null
|
|
66
|
+
|
|
51
67
|
@RequiresApi(Build.VERSION_CODES.O)
|
|
52
68
|
suspend fun start(
|
|
53
69
|
silenceThresholdMs: Long = 1_500,
|
|
54
70
|
maxDurationMs: Long = 10_000,
|
|
71
|
+
preferSpeechToText: Boolean = false,
|
|
72
|
+
onChunk: ((chunk: VoiceInputChunk) -> Unit)? = null,
|
|
73
|
+
language: String? = null
|
|
74
|
+
): VoiceInputResult {
|
|
75
|
+
if (preferSpeechToText) {
|
|
76
|
+
val context = NitroModules.applicationContext ?: throw IllegalArgumentException()
|
|
77
|
+
if (SpeechRecognizer.isRecognitionAvailable(context)) {
|
|
78
|
+
if (carContext != null) {
|
|
79
|
+
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
|
|
80
|
+
return startSTTFromCarAudio(silenceThresholdMs, maxDurationMs, onChunk, language)
|
|
81
|
+
}
|
|
82
|
+
// Car connected but API < 33: EXTRA_AUDIO_SOURCE unavailable, fall back to PCM
|
|
83
|
+
return startPCM(silenceThresholdMs, maxDurationMs, onChunk)
|
|
84
|
+
}
|
|
85
|
+
return ThreadUtil.postOnUiAndAwait { startSTT(context, onChunk, language) }.getOrThrow()
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return startPCM(silenceThresholdMs, maxDurationMs, onChunk)
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// MARK: - STT path (SpeechRecognizer owns the mic)
|
|
92
|
+
|
|
93
|
+
private suspend fun startSTT(
|
|
94
|
+
context: android.content.Context,
|
|
95
|
+
onChunk: ((chunk: VoiceInputChunk) -> Unit)?,
|
|
96
|
+
language: String?
|
|
97
|
+
): VoiceInputResult = suspendCancellableCoroutine { cont ->
|
|
98
|
+
val recognizer = SpeechRecognizer.createSpeechRecognizer(context)
|
|
99
|
+
activeSpeechRecognizer = recognizer
|
|
100
|
+
|
|
101
|
+
recognizer.setRecognitionListener(object : RecognitionListener {
|
|
102
|
+
override fun onResults(results: Bundle?) {
|
|
103
|
+
activeSpeechRecognizer = null
|
|
104
|
+
recognizer.destroy()
|
|
105
|
+
val text =
|
|
106
|
+
results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)?.firstOrNull()
|
|
107
|
+
cont.resume(VoiceInputResult(transcription = text, audio = null))
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
override fun onError(error: Int) {
|
|
111
|
+
activeSpeechRecognizer = null
|
|
112
|
+
recognizer.destroy()
|
|
113
|
+
// Return empty transcription — caller sees null audio and null transcription
|
|
114
|
+
cont.resume(VoiceInputResult(transcription = null, audio = null))
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
override fun onPartialResults(partialResults: Bundle?) {
|
|
118
|
+
val text = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)
|
|
119
|
+
?.firstOrNull()
|
|
120
|
+
if (!text.isNullOrEmpty()) {
|
|
121
|
+
onChunk?.invoke(VoiceInputChunk(partial = text, audio = null))
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
override fun onReadyForSpeech(params: Bundle?) {}
|
|
126
|
+
override fun onBeginningOfSpeech() {}
|
|
127
|
+
override fun onRmsChanged(rmsdB: Float) {}
|
|
128
|
+
override fun onBufferReceived(buffer: ByteArray?) {}
|
|
129
|
+
override fun onEndOfSpeech() {}
|
|
130
|
+
override fun onEvent(eventType: Int, params: Bundle?) {}
|
|
131
|
+
})
|
|
132
|
+
|
|
133
|
+
val intent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply {
|
|
134
|
+
putExtra(
|
|
135
|
+
RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM
|
|
136
|
+
)
|
|
137
|
+
putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true)
|
|
138
|
+
putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 1)
|
|
139
|
+
language?.let {
|
|
140
|
+
putExtra(RecognizerIntent.EXTRA_LANGUAGE, it)
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
recognizer.startListening(intent)
|
|
145
|
+
|
|
146
|
+
cont.invokeOnCancellation {
|
|
147
|
+
activeSpeechRecognizer = null
|
|
148
|
+
recognizer.destroy()
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// MARK: - STT path fed from CarAudioRecord via a pipe (API 33+)
|
|
153
|
+
@SuppressLint("MissingPermission")
|
|
154
|
+
@RequiresApi(Build.VERSION_CODES.TIRAMISU)
|
|
155
|
+
private suspend fun startSTTFromCarAudio(
|
|
156
|
+
silenceThresholdMs: Long,
|
|
157
|
+
maxDurationMs: Long,
|
|
158
|
+
onChunk: ((chunk: VoiceInputChunk) -> Unit)?,
|
|
159
|
+
language: String?
|
|
160
|
+
): VoiceInputResult {
|
|
161
|
+
if (!hasVoiceInputPermission()) {
|
|
162
|
+
throw SecurityException("RECORD_AUDIO permission not granted")
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
val appContext = NitroModules.applicationContext ?: throw IllegalArgumentException()
|
|
166
|
+
val pipes = ParcelFileDescriptor.createPipe()
|
|
167
|
+
val readFd = pipes[0]
|
|
168
|
+
val pipeOut = ParcelFileDescriptor.AutoCloseOutputStream(pipes[1])
|
|
169
|
+
|
|
170
|
+
val sttDeferred = scope.async {
|
|
171
|
+
ThreadUtil.postOnUiAndAwait {
|
|
172
|
+
startSTTWithSource(appContext, readFd, silenceThresholdMs, onChunk, language)
|
|
173
|
+
}.getOrThrow()
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
try {
|
|
177
|
+
recordPCM(silenceThresholdMs, maxDurationMs) { chunk ->
|
|
178
|
+
chunk.audio?.let { ab ->
|
|
179
|
+
try {
|
|
180
|
+
pipeOut.write(ab.toByteArray())
|
|
181
|
+
} catch (_: Exception) {
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
} finally {
|
|
186
|
+
try {
|
|
187
|
+
pipeOut.close()
|
|
188
|
+
} catch (_: Exception) {
|
|
189
|
+
}
|
|
190
|
+
try {
|
|
191
|
+
readFd.close()
|
|
192
|
+
} catch (_: Exception) {
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
return sttDeferred.await()
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
@RequiresApi(Build.VERSION_CODES.TIRAMISU)
|
|
200
|
+
private suspend fun startSTTWithSource(
|
|
201
|
+
context: android.content.Context,
|
|
202
|
+
audioSource: ParcelFileDescriptor,
|
|
203
|
+
silenceThresholdMs: Long,
|
|
204
|
+
onChunk: ((chunk: VoiceInputChunk) -> Unit)?,
|
|
205
|
+
language: String?
|
|
206
|
+
): VoiceInputResult = suspendCancellableCoroutine { cont ->
|
|
207
|
+
val recognizer = SpeechRecognizer.createSpeechRecognizer(context)
|
|
208
|
+
activeSpeechRecognizer = recognizer
|
|
209
|
+
// When EXTRA_AUDIO_SOURCE is used, onResults always returns an empty list — the actual
|
|
210
|
+
// transcription only arrives via onPartialResults. Track the last partial here.
|
|
211
|
+
var lastPartial: String? = null
|
|
212
|
+
|
|
213
|
+
recognizer.setRecognitionListener(object : RecognitionListener {
|
|
214
|
+
override fun onResults(results: Bundle?) {
|
|
215
|
+
activeSpeechRecognizer = null
|
|
216
|
+
recognizer.destroy()
|
|
217
|
+
val text =
|
|
218
|
+
results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)?.firstOrNull()
|
|
219
|
+
?: lastPartial
|
|
220
|
+
cont.resume(VoiceInputResult(transcription = text, audio = null))
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
override fun onError(error: Int) {
|
|
224
|
+
activeSpeechRecognizer = null
|
|
225
|
+
recognizer.destroy()
|
|
226
|
+
cont.resume(VoiceInputResult(transcription = lastPartial, audio = null))
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
override fun onPartialResults(partialResults: Bundle?) {
|
|
230
|
+
val text = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)
|
|
231
|
+
?.firstOrNull()
|
|
232
|
+
if (!text.isNullOrEmpty()) {
|
|
233
|
+
lastPartial = text
|
|
234
|
+
onChunk?.invoke(VoiceInputChunk(partial = text, audio = null))
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
override fun onReadyForSpeech(params: Bundle?) {}
|
|
239
|
+
override fun onBeginningOfSpeech() {}
|
|
240
|
+
override fun onRmsChanged(rmsdB: Float) {}
|
|
241
|
+
override fun onBufferReceived(buffer: ByteArray?) {}
|
|
242
|
+
override fun onEndOfSpeech() {}
|
|
243
|
+
override fun onEvent(eventType: Int, params: Bundle?) {}
|
|
244
|
+
})
|
|
245
|
+
|
|
246
|
+
val intent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply {
|
|
247
|
+
putExtra(
|
|
248
|
+
RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM
|
|
249
|
+
)
|
|
250
|
+
putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true)
|
|
251
|
+
putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 1)
|
|
252
|
+
language?.let {
|
|
253
|
+
putExtra(RecognizerIntent.EXTRA_LANGUAGE, it)
|
|
254
|
+
}
|
|
255
|
+
putExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE, audioSource)
|
|
256
|
+
putExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE_CHANNEL_COUNT, 1)
|
|
257
|
+
putExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE_ENCODING, AudioFormat.ENCODING_PCM_16BIT)
|
|
258
|
+
putExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE_SAMPLING_RATE, SAMPLE_RATE)
|
|
259
|
+
putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_MINIMUM_LENGTH_MILLIS, WARMUP_MS)
|
|
260
|
+
putExtra(
|
|
261
|
+
RecognizerIntent.EXTRA_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS,
|
|
262
|
+
silenceThresholdMs
|
|
263
|
+
)
|
|
264
|
+
putExtra(
|
|
265
|
+
RecognizerIntent.EXTRA_SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS,
|
|
266
|
+
silenceThresholdMs / 2,
|
|
267
|
+
)
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
recognizer.startListening(intent)
|
|
271
|
+
|
|
272
|
+
cont.invokeOnCancellation {
|
|
273
|
+
activeSpeechRecognizer = null
|
|
274
|
+
recognizer.destroy()
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
// MARK: - PCM path
|
|
279
|
+
|
|
280
|
+
@RequiresApi(Build.VERSION_CODES.O)
|
|
281
|
+
private suspend fun startPCM(
|
|
282
|
+
silenceThresholdMs: Long,
|
|
283
|
+
maxDurationMs: Long,
|
|
284
|
+
onChunk: ((chunk: VoiceInputChunk) -> Unit)?,
|
|
285
|
+
): VoiceInputResult {
|
|
286
|
+
val pcmBytes = recordPCM(silenceThresholdMs, maxDurationMs, onChunk)
|
|
287
|
+
val directBuffer =
|
|
288
|
+
ByteBuffer.allocateDirect(pcmBytes.size).put(pcmBytes).rewind() as ByteBuffer
|
|
289
|
+
return VoiceInputResult(transcription = null, audio = ArrayBuffer.wrap(directBuffer))
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
@SuppressLint("MissingPermission")
|
|
293
|
+
@RequiresApi(Build.VERSION_CODES.O)
|
|
294
|
+
private suspend fun recordPCM(
|
|
295
|
+
silenceThresholdMs: Long,
|
|
296
|
+
maxDurationMs: Long,
|
|
297
|
+
onChunk: ((chunk: VoiceInputChunk) -> Unit)?,
|
|
55
298
|
): ByteArray = suspendCancellableCoroutine { cont ->
|
|
56
|
-
|
|
57
|
-
if (appContext == null || ContextCompat.checkSelfPermission(
|
|
58
|
-
appContext,
|
|
59
|
-
android.Manifest.permission.RECORD_AUDIO,
|
|
60
|
-
) != PackageManager.PERMISSION_GRANTED
|
|
61
|
-
) {
|
|
299
|
+
if (!hasVoiceInputPermission()) {
|
|
62
300
|
cont.resumeWithException(SecurityException("RECORD_AUDIO permission not granted"))
|
|
63
301
|
return@suspendCancellableCoroutine
|
|
64
302
|
}
|
|
65
303
|
|
|
66
|
-
|
|
304
|
+
val appContext = NitroModules.applicationContext ?: run {
|
|
305
|
+
cont.resumeWithException(SecurityException("Missing application context"))
|
|
306
|
+
return@suspendCancellableCoroutine
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
pcmContinuation = cont
|
|
67
310
|
|
|
68
311
|
val audioManager = appContext.getSystemService(AudioManager::class.java)
|
|
69
312
|
|
|
@@ -80,7 +323,7 @@ class VoiceInputManager(
|
|
|
80
323
|
}.build()
|
|
81
324
|
|
|
82
325
|
if (audioManager.requestAudioFocus(focusRequest) != AudioManager.AUDIOFOCUS_REQUEST_GRANTED) {
|
|
83
|
-
|
|
326
|
+
pcmContinuation = null
|
|
84
327
|
cont.resumeWithException(IllegalStateException("Audio focus request denied"))
|
|
85
328
|
return@suspendCancellableCoroutine
|
|
86
329
|
}
|
|
@@ -136,6 +379,13 @@ class VoiceInputManager(
|
|
|
136
379
|
if (read > 0) {
|
|
137
380
|
outputStream.write(buffer, 0, read)
|
|
138
381
|
|
|
382
|
+
onChunk?.let { cb ->
|
|
383
|
+
val chunk = ByteArray(read) { buffer[it] }
|
|
384
|
+
val direct =
|
|
385
|
+
ByteBuffer.allocateDirect(read).put(chunk).rewind() as ByteBuffer
|
|
386
|
+
cb(VoiceInputChunk(partial = null, audio = ArrayBuffer.wrap(direct)))
|
|
387
|
+
}
|
|
388
|
+
|
|
139
389
|
val now = System.currentTimeMillis()
|
|
140
390
|
val elapsedMs = now - recordingStart
|
|
141
391
|
|
|
@@ -151,7 +401,9 @@ class VoiceInputManager(
|
|
|
151
401
|
val sample =
|
|
152
402
|
(buffer[i].toInt() and 0xFF) or (buffer[i + 1].toInt() shl 8)
|
|
153
403
|
val absSample = abs(sample.toShort().toInt())
|
|
154
|
-
if (absSample > peak)
|
|
404
|
+
if (absSample > peak) {
|
|
405
|
+
peak = absSample
|
|
406
|
+
}
|
|
155
407
|
i += 2
|
|
156
408
|
}
|
|
157
409
|
|
|
@@ -170,14 +422,21 @@ class VoiceInputManager(
|
|
|
170
422
|
}
|
|
171
423
|
} finally {
|
|
172
424
|
releaseResources()
|
|
173
|
-
val
|
|
174
|
-
|
|
175
|
-
|
|
425
|
+
val captured = pcmContinuation
|
|
426
|
+
pcmContinuation = null
|
|
427
|
+
captured?.resume(outputStream.toByteArray())
|
|
176
428
|
}
|
|
177
429
|
}
|
|
178
430
|
}
|
|
179
431
|
|
|
180
432
|
fun stop() {
|
|
433
|
+
// STT path: stopListening() triggers onResults/onError which resolves the continuation
|
|
434
|
+
activeSpeechRecognizer?.let { recognizer ->
|
|
435
|
+
UiThreadUtil.runOnUiThread {
|
|
436
|
+
recognizer.stopListening()
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
// PCM path and car-audio STT pump
|
|
181
440
|
isRecording = false
|
|
182
441
|
carAudioRecord?.stopRecording()
|
|
183
442
|
audioRecord?.stop()
|
|
@@ -210,5 +469,12 @@ class VoiceInputManager(
|
|
|
210
469
|
private const val WARMUP_MS = 500L
|
|
211
470
|
private const val SAMPLE_RATE = 16_000
|
|
212
471
|
private const val PHONE_BUFFER_SIZE = 3_200 // ~100ms at 16kHz/16-bit/mono
|
|
472
|
+
|
|
473
|
+
fun hasVoiceInputPermission(): Boolean {
|
|
474
|
+
val context = NitroModules.applicationContext ?: return false
|
|
475
|
+
return ContextCompat.checkSelfPermission(
|
|
476
|
+
context, android.Manifest.permission.RECORD_AUDIO
|
|
477
|
+
) == PackageManager.PERMISSION_GRANTED
|
|
478
|
+
}
|
|
213
479
|
}
|
|
214
480
|
}
|
package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/utils/ThreadUtil.kt
CHANGED
|
@@ -1,19 +1,12 @@
|
|
|
1
1
|
package com.margelo.nitro.swe.iternio.reactnativeautoplay.utils
|
|
2
2
|
|
|
3
|
-
import
|
|
4
|
-
import kotlinx.coroutines.
|
|
5
|
-
import kotlin.coroutines.resume
|
|
3
|
+
import kotlinx.coroutines.Dispatchers
|
|
4
|
+
import kotlinx.coroutines.withContext
|
|
6
5
|
|
|
7
6
|
object ThreadUtil {
|
|
8
|
-
suspend fun <T> postOnUiAndAwait(block: () -> T): Result<T> =
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
try {
|
|
12
|
-
val result = block()
|
|
13
|
-
cont.resume(Result.success(result))
|
|
14
|
-
} catch (e: Exception) {
|
|
15
|
-
cont.resume(Result.failure(e))
|
|
16
|
-
}
|
|
17
|
-
}
|
|
7
|
+
suspend fun <T> postOnUiAndAwait(block: suspend () -> T): Result<T> = runCatching {
|
|
8
|
+
withContext(Dispatchers.Main) {
|
|
9
|
+
block()
|
|
18
10
|
}
|
|
11
|
+
}
|
|
19
12
|
}
|