@iternio/react-native-auto-play 0.4.4 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridAutoPlay.kt +0 -89
- package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridVoice.kt +95 -0
- package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/VoiceInputManager.kt +276 -20
- package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/utils/ThreadUtil.kt +6 -13
- package/ios/hybrid/HybridAutoPlay.swift +2 -47
- package/ios/hybrid/HybridVoice.swift +63 -0
- package/ios/utils/VoiceInputManager.swift +141 -40
- package/lib/HybridAutoPlay.d.ts +2 -0
- package/lib/HybridAutoPlay.js +2 -0
- package/lib/hooks/useIsAutoPlayFocused.d.ts +7 -0
- package/lib/hooks/useIsAutoPlayFocused.js +20 -0
- package/lib/hybrid/HybridVoice.d.ts +12 -0
- package/lib/hybrid/HybridVoice.js +13 -0
- package/lib/hybrid.d.ts +2 -0
- package/lib/hybrid.js +2 -0
- package/lib/index.d.ts +3 -1
- package/lib/index.js +2 -1
- package/lib/specs/AutoPlay.nitro.d.ts +0 -29
- package/lib/specs/AutomotivePermissionRequestTemplate.d.ts +11 -0
- package/lib/specs/AutomotivePermissionRequestTemplate.js +1 -0
- package/lib/specs/AutomotivePermissionRequestTemplate.nitro.d.ts +11 -0
- package/lib/specs/AutomotivePermissionRequestTemplate.nitro.js +1 -0
- package/lib/specs/Voice.nitro.d.ts +51 -0
- package/lib/specs/Voice.nitro.js +1 -0
- package/lib/templates/AutomotivePermissionRequestTemplate.d.ts +23 -0
- package/lib/templates/AutomotivePermissionRequestTemplate.js +18 -0
- package/lib/types/Glyphmap.d.ts +4105 -0
- package/lib/types/Glyphmap.js +4105 -0
- package/lib/types/Voice.d.ts +15 -0
- package/lib/types/Voice.js +1 -0
- package/nitro.json +10 -0
- package/nitrogen/generated/android/ReactNativeAutoPlay+autolinking.cmake +2 -0
- package/nitrogen/generated/android/ReactNativeAutoPlayOnLoad.cpp +18 -0
- package/nitrogen/generated/android/c++/JFunc_void_VoiceInputChunk.hpp +81 -0
- package/nitrogen/generated/android/c++/JHybridAutoPlaySpec.cpp +0 -43
- package/nitrogen/generated/android/c++/JHybridAutoPlaySpec.hpp +0 -4
- package/nitrogen/generated/android/c++/JHybridVoiceSpec.cpp +104 -0
- package/nitrogen/generated/android/c++/JHybridVoiceSpec.hpp +66 -0
- package/nitrogen/generated/android/c++/JVoiceInputChunk.hpp +64 -0
- package/nitrogen/generated/android/c++/JVoiceInputResult.hpp +64 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/swe/iternio/reactnativeautoplay/Func_void_VoiceInputChunk.kt +80 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridAutoPlaySpec.kt +0 -17
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridVoiceSpec.kt +72 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/swe/iternio/reactnativeautoplay/VoiceInputChunk.kt +41 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/swe/iternio/reactnativeautoplay/VoiceInputResult.kt +41 -0
- package/nitrogen/generated/ios/ReactNativeAutoPlay-Swift-Cxx-Bridge.cpp +41 -16
- package/nitrogen/generated/ios/ReactNativeAutoPlay-Swift-Cxx-Bridge.hpp +201 -126
- package/nitrogen/generated/ios/ReactNativeAutoPlay-Swift-Cxx-Umbrella.hpp +11 -0
- package/nitrogen/generated/ios/ReactNativeAutoPlayAutolinking.mm +8 -0
- package/nitrogen/generated/ios/ReactNativeAutoPlayAutolinking.swift +12 -0
- package/nitrogen/generated/ios/c++/HybridAutoPlaySpecSwift.hpp +0 -34
- package/nitrogen/generated/ios/c++/HybridVoiceSpecSwift.cpp +11 -0
- package/nitrogen/generated/ios/c++/HybridVoiceSpecSwift.hpp +116 -0
- package/nitrogen/generated/ios/swift/Func_void_VoiceInputChunk.swift +46 -0
- package/nitrogen/generated/ios/swift/{Func_void_std__shared_ptr_ArrayBuffer_.swift → Func_void_VoiceInputResult.swift} +10 -10
- package/nitrogen/generated/ios/swift/Func_void_bool.swift +5 -5
- package/nitrogen/generated/ios/swift/HybridAutoPlaySpec.swift +0 -4
- package/nitrogen/generated/ios/swift/HybridAutoPlaySpec_cxx.swift +0 -82
- package/nitrogen/generated/ios/swift/HybridVoiceSpec.swift +58 -0
- package/nitrogen/generated/ios/swift/HybridVoiceSpec_cxx.swift +227 -0
- package/nitrogen/generated/ios/swift/VoiceInputChunk.swift +60 -0
- package/nitrogen/generated/ios/swift/VoiceInputResult.swift +60 -0
- package/nitrogen/generated/shared/c++/HybridAutoPlaySpec.cpp +0 -4
- package/nitrogen/generated/shared/c++/HybridAutoPlaySpec.hpp +0 -5
- package/nitrogen/generated/shared/c++/HybridVoiceSpec.cpp +24 -0
- package/nitrogen/generated/shared/c++/HybridVoiceSpec.hpp +73 -0
- package/nitrogen/generated/shared/c++/VoiceInputChunk.hpp +89 -0
- package/nitrogen/generated/shared/c++/VoiceInputResult.hpp +89 -0
- package/package.json +1 -1
- package/src/hybrid/HybridVoice.ts +30 -0
- package/src/index.ts +3 -1
- package/src/specs/AutoPlay.nitro.ts +0 -37
- package/src/specs/Voice.nitro.ts +58 -0
- package/src/types/Voice.ts +17 -0
package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridAutoPlay.kt
CHANGED
|
@@ -1,19 +1,11 @@
|
|
|
1
1
|
package com.margelo.nitro.swe.iternio.reactnativeautoplay
|
|
2
2
|
|
|
3
|
-
import android.content.pm.PackageManager
|
|
4
3
|
import android.os.Build
|
|
5
|
-
import androidx.core.content.ContextCompat
|
|
6
4
|
import com.facebook.react.bridge.UiThreadUtil
|
|
7
|
-
import com.facebook.react.modules.core.PermissionAwareActivity
|
|
8
|
-
import com.facebook.react.modules.core.PermissionListener
|
|
9
|
-
import com.margelo.nitro.NitroModules
|
|
10
|
-
import com.margelo.nitro.core.ArrayBuffer
|
|
11
5
|
import com.margelo.nitro.core.Promise
|
|
12
6
|
import com.margelo.nitro.swe.iternio.reactnativeautoplay.template.AndroidAutoTemplate
|
|
13
7
|
import com.margelo.nitro.swe.iternio.reactnativeautoplay.template.MessageTemplate
|
|
14
8
|
import com.margelo.nitro.swe.iternio.reactnativeautoplay.utils.ThreadUtil
|
|
15
|
-
import kotlinx.coroutines.suspendCancellableCoroutine
|
|
16
|
-
import java.nio.ByteBuffer
|
|
17
9
|
import java.util.concurrent.ConcurrentHashMap
|
|
18
10
|
import java.util.concurrent.CopyOnWriteArrayList
|
|
19
11
|
import kotlin.coroutines.resume
|
|
@@ -251,84 +243,6 @@ class HybridAutoPlay : HybridAutoPlaySpec() {
|
|
|
251
243
|
}
|
|
252
244
|
}
|
|
253
245
|
|
|
254
|
-
override fun hasVoiceInputPermission(): Boolean {
|
|
255
|
-
val context = NitroModules.applicationContext ?: return false
|
|
256
|
-
return ContextCompat.checkSelfPermission(
|
|
257
|
-
context, android.Manifest.permission.RECORD_AUDIO
|
|
258
|
-
) == PackageManager.PERMISSION_GRANTED
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
override fun requestVoiceInputPermission(): Promise<Boolean> {
|
|
262
|
-
return Promise.async {
|
|
263
|
-
if (hasVoiceInputPermission()) {
|
|
264
|
-
return@async true
|
|
265
|
-
}
|
|
266
|
-
|
|
267
|
-
val carContext = AndroidAutoSession.getRootContext()
|
|
268
|
-
|
|
269
|
-
if (carContext != null) {
|
|
270
|
-
suspendCancellableCoroutine {
|
|
271
|
-
carContext.requestPermissions(listOf(android.Manifest.permission.RECORD_AUDIO)) { approved, _ ->
|
|
272
|
-
it.resume(approved.contains(android.Manifest.permission.RECORD_AUDIO))
|
|
273
|
-
}
|
|
274
|
-
}
|
|
275
|
-
} else {
|
|
276
|
-
val context = NitroModules.applicationContext ?: return@async false
|
|
277
|
-
val activity =
|
|
278
|
-
context.currentActivity as? PermissionAwareActivity ?: return@async false
|
|
279
|
-
val code = (Math.random() * 10000).toInt()
|
|
280
|
-
|
|
281
|
-
suspendCancellableCoroutine {
|
|
282
|
-
activity.requestPermissions(
|
|
283
|
-
arrayOf(android.Manifest.permission.RECORD_AUDIO),
|
|
284
|
-
code,
|
|
285
|
-
PermissionListener { requestCode, _, grantResults ->
|
|
286
|
-
if (requestCode != code) {
|
|
287
|
-
return@PermissionListener false
|
|
288
|
-
}
|
|
289
|
-
|
|
290
|
-
val granted =
|
|
291
|
-
grantResults.isNotEmpty() && grantResults.first() == PackageManager.PERMISSION_GRANTED
|
|
292
|
-
|
|
293
|
-
it.resume(granted)
|
|
294
|
-
|
|
295
|
-
return@PermissionListener true
|
|
296
|
-
})
|
|
297
|
-
}
|
|
298
|
-
}
|
|
299
|
-
}
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
override fun startVoiceInput(
|
|
303
|
-
silenceThresholdMs: Double?, maxDurationMs: Double?, listeningText: String?
|
|
304
|
-
): Promise<ArrayBuffer> {
|
|
305
|
-
return Promise.async {
|
|
306
|
-
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) {
|
|
307
|
-
throw UnsupportedOperationException("startVoiceInput requires at least API level ${Build.VERSION_CODES.O}")
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
val manager = VoiceInputManager(AndroidAutoSession.getRootContext())
|
|
311
|
-
voiceInputManager = manager
|
|
312
|
-
|
|
313
|
-
try {
|
|
314
|
-
val pcmBytes = manager.start(
|
|
315
|
-
silenceThresholdMs = silenceThresholdMs?.toLong() ?: 1_500L,
|
|
316
|
-
maxDurationMs = maxDurationMs?.toLong() ?: 10_000L,
|
|
317
|
-
)
|
|
318
|
-
val directBuffer =
|
|
319
|
-
ByteBuffer.allocateDirect(pcmBytes.size).put(pcmBytes).rewind() as ByteBuffer
|
|
320
|
-
ArrayBuffer.wrap(directBuffer)
|
|
321
|
-
} finally {
|
|
322
|
-
voiceInputManager = null
|
|
323
|
-
manager.dispose()
|
|
324
|
-
}
|
|
325
|
-
}
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
override fun stopVoiceInput() {
|
|
329
|
-
voiceInputManager?.stop()
|
|
330
|
-
}
|
|
331
|
-
|
|
332
246
|
companion object {
|
|
333
247
|
const val TAG = "HybridAutoPlay"
|
|
334
248
|
|
|
@@ -339,9 +253,6 @@ class HybridAutoPlay : HybridAutoPlaySpec() {
|
|
|
339
253
|
|
|
340
254
|
private val voiceInputListeners = CopyOnWriteArrayList<(Location?, String?) -> Unit>()
|
|
341
255
|
|
|
342
|
-
@Volatile
|
|
343
|
-
private var voiceInputManager: VoiceInputManager? = null
|
|
344
|
-
|
|
345
256
|
private val safeAreaInsetsListeners =
|
|
346
257
|
ConcurrentHashMap<String, CopyOnWriteArrayList<(SafeAreaInsets) -> Unit>>()
|
|
347
258
|
|
package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridVoice.kt
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
package com.margelo.nitro.swe.iternio.reactnativeautoplay
|
|
2
|
+
|
|
3
|
+
import android.content.pm.PackageManager
|
|
4
|
+
import android.os.Build
|
|
5
|
+
import androidx.core.content.ContextCompat
|
|
6
|
+
import com.facebook.react.modules.core.PermissionAwareActivity
|
|
7
|
+
import com.facebook.react.modules.core.PermissionListener
|
|
8
|
+
import com.margelo.nitro.NitroModules
|
|
9
|
+
import com.margelo.nitro.core.Promise
|
|
10
|
+
import kotlinx.coroutines.suspendCancellableCoroutine
|
|
11
|
+
import kotlin.coroutines.resume
|
|
12
|
+
|
|
13
|
+
class HybridVoice : HybridVoiceSpec() {
|
|
14
|
+
@Volatile
|
|
15
|
+
private var voiceInputManager: VoiceInputManager? = null
|
|
16
|
+
|
|
17
|
+
override fun hasVoiceInputPermission(): Boolean {
|
|
18
|
+
return VoiceInputManager.hasVoiceInputPermission()
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
override fun requestVoiceInputPermission(): Promise<Boolean> {
|
|
22
|
+
return Promise.async {
|
|
23
|
+
if (hasVoiceInputPermission()) {
|
|
24
|
+
return@async true
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
val carContext = AndroidAutoSession.getRootContext()
|
|
28
|
+
|
|
29
|
+
if (carContext != null) {
|
|
30
|
+
suspendCancellableCoroutine { cont ->
|
|
31
|
+
carContext.requestPermissions(
|
|
32
|
+
listOf(android.Manifest.permission.RECORD_AUDIO)
|
|
33
|
+
) { approved, _ ->
|
|
34
|
+
cont.resume(approved.contains(android.Manifest.permission.RECORD_AUDIO))
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
} else {
|
|
38
|
+
val context = NitroModules.applicationContext ?: return@async false
|
|
39
|
+
val activity =
|
|
40
|
+
context.currentActivity as? PermissionAwareActivity ?: return@async false
|
|
41
|
+
val code = (Math.random() * 10000).toInt()
|
|
42
|
+
|
|
43
|
+
suspendCancellableCoroutine { cont ->
|
|
44
|
+
activity.requestPermissions(
|
|
45
|
+
arrayOf(android.Manifest.permission.RECORD_AUDIO),
|
|
46
|
+
code,
|
|
47
|
+
PermissionListener { requestCode, _, grantResults ->
|
|
48
|
+
if (requestCode != code) {
|
|
49
|
+
return@PermissionListener false
|
|
50
|
+
}
|
|
51
|
+
cont.resume(
|
|
52
|
+
grantResults.isNotEmpty() &&
|
|
53
|
+
grantResults.first() == PackageManager.PERMISSION_GRANTED
|
|
54
|
+
)
|
|
55
|
+
true
|
|
56
|
+
}
|
|
57
|
+
)
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
override fun startVoiceInput(
|
|
64
|
+
silenceThresholdMs: Double?,
|
|
65
|
+
maxDurationMs: Double?,
|
|
66
|
+
listeningText: String?,
|
|
67
|
+
preferSpeechToText: Boolean?,
|
|
68
|
+
onChunk: ((chunk: VoiceInputChunk) -> Unit)?,
|
|
69
|
+
): Promise<VoiceInputResult> {
|
|
70
|
+
return Promise.async {
|
|
71
|
+
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) {
|
|
72
|
+
throw UnsupportedOperationException("startVoiceInput requires at least API level ${Build.VERSION_CODES.O}")
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
val manager = VoiceInputManager(AndroidAutoSession.getRootContext())
|
|
76
|
+
voiceInputManager = manager
|
|
77
|
+
|
|
78
|
+
try {
|
|
79
|
+
manager.start(
|
|
80
|
+
silenceThresholdMs = silenceThresholdMs?.toLong() ?: 1_500L,
|
|
81
|
+
maxDurationMs = maxDurationMs?.toLong() ?: 10_000L,
|
|
82
|
+
preferSpeechToText = preferSpeechToText ?: false,
|
|
83
|
+
onChunk = onChunk,
|
|
84
|
+
)
|
|
85
|
+
} finally {
|
|
86
|
+
voiceInputManager = null
|
|
87
|
+
manager.dispose()
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
override fun stopVoiceInput() {
|
|
93
|
+
voiceInputManager?.stop()
|
|
94
|
+
}
|
|
95
|
+
}
|
package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/VoiceInputManager.kt
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
package com.margelo.nitro.swe.iternio.reactnativeautoplay
|
|
2
2
|
|
|
3
|
+
import android.annotation.SuppressLint
|
|
4
|
+
import android.content.Intent
|
|
3
5
|
import android.content.pm.PackageManager
|
|
4
6
|
import android.media.AudioAttributes
|
|
5
7
|
import android.media.AudioFocusRequest
|
|
@@ -8,18 +10,28 @@ import android.media.AudioManager
|
|
|
8
10
|
import android.media.AudioRecord
|
|
9
11
|
import android.media.MediaRecorder
|
|
10
12
|
import android.os.Build
|
|
13
|
+
import android.os.Bundle
|
|
14
|
+
import android.os.ParcelFileDescriptor
|
|
15
|
+
import android.speech.RecognitionListener
|
|
16
|
+
import android.speech.RecognizerIntent
|
|
17
|
+
import android.speech.SpeechRecognizer
|
|
11
18
|
import androidx.annotation.RequiresApi
|
|
12
19
|
import androidx.car.app.CarContext
|
|
13
20
|
import androidx.car.app.media.CarAudioRecord
|
|
14
21
|
import androidx.core.content.ContextCompat
|
|
22
|
+
import com.facebook.react.bridge.UiThreadUtil
|
|
15
23
|
import com.margelo.nitro.NitroModules
|
|
24
|
+
import com.margelo.nitro.core.ArrayBuffer
|
|
25
|
+
import com.margelo.nitro.swe.iternio.reactnativeautoplay.utils.ThreadUtil
|
|
16
26
|
import kotlinx.coroutines.CoroutineScope
|
|
17
27
|
import kotlinx.coroutines.Dispatchers
|
|
18
28
|
import kotlinx.coroutines.Job
|
|
29
|
+
import kotlinx.coroutines.async
|
|
19
30
|
import kotlinx.coroutines.cancel
|
|
20
31
|
import kotlinx.coroutines.launch
|
|
21
32
|
import kotlinx.coroutines.suspendCancellableCoroutine
|
|
22
33
|
import java.io.ByteArrayOutputStream
|
|
34
|
+
import java.nio.ByteBuffer
|
|
23
35
|
import kotlin.coroutines.Continuation
|
|
24
36
|
import kotlin.coroutines.resume
|
|
25
37
|
import kotlin.coroutines.resumeWithException
|
|
@@ -27,43 +39,264 @@ import kotlin.math.abs
|
|
|
27
39
|
|
|
28
40
|
/**
|
|
29
41
|
* Captures 16-bit PCM audio (16 kHz, mono).
|
|
30
|
-
* When [carContext] is provided uses CarAudioRecord (Android Auto/Automotive)
|
|
31
|
-
*
|
|
42
|
+
* When [carContext] is provided uses CarAudioRecord (Android Auto/Automotive),
|
|
43
|
+
* otherwise falls back to standard AudioRecord.
|
|
44
|
+
*
|
|
45
|
+
* When preferSpeechToText is true and SpeechRecognizer is available, it owns
|
|
46
|
+
* the microphone and streams partial results; the PCM path is not used.
|
|
47
|
+
* When SpeechRecognizer is unavailable the manager falls back to PCM recording.
|
|
32
48
|
*/
|
|
33
49
|
class VoiceInputManager(
|
|
34
50
|
private val carContext: CarContext?,
|
|
35
51
|
) {
|
|
52
|
+
// PCM recording state
|
|
36
53
|
private var carAudioRecord: CarAudioRecord? = null
|
|
37
54
|
private var audioRecord: AudioRecord? = null
|
|
38
55
|
private var audioFocusRequest: AudioFocusRequest? = null
|
|
39
56
|
private var recordingJob: Job? = null
|
|
40
|
-
private var
|
|
57
|
+
private var pcmContinuation: Continuation<ByteArray>? = null
|
|
41
58
|
private val scope = CoroutineScope(Dispatchers.IO)
|
|
42
59
|
|
|
43
60
|
@Volatile
|
|
44
61
|
private var isRecording = false
|
|
45
62
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
*/
|
|
63
|
+
// STT state — only set when SpeechRecognizer owns the mic
|
|
64
|
+
@Volatile
|
|
65
|
+
private var activeSpeechRecognizer: SpeechRecognizer? = null
|
|
66
|
+
|
|
51
67
|
@RequiresApi(Build.VERSION_CODES.O)
|
|
52
68
|
suspend fun start(
|
|
53
69
|
silenceThresholdMs: Long = 1_500,
|
|
54
70
|
maxDurationMs: Long = 10_000,
|
|
71
|
+
preferSpeechToText: Boolean = false,
|
|
72
|
+
onChunk: ((chunk: VoiceInputChunk) -> Unit)? = null,
|
|
73
|
+
): VoiceInputResult {
|
|
74
|
+
if (preferSpeechToText) {
|
|
75
|
+
val context = NitroModules.applicationContext ?: throw IllegalArgumentException()
|
|
76
|
+
if (SpeechRecognizer.isRecognitionAvailable(context)) {
|
|
77
|
+
if (carContext != null) {
|
|
78
|
+
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
|
|
79
|
+
return startSTTFromCarAudio(silenceThresholdMs, maxDurationMs, onChunk)
|
|
80
|
+
}
|
|
81
|
+
// Car connected but API < 33: EXTRA_AUDIO_SOURCE unavailable, fall back to PCM
|
|
82
|
+
return startPCM(silenceThresholdMs, maxDurationMs, onChunk)
|
|
83
|
+
}
|
|
84
|
+
return ThreadUtil.postOnUiAndAwait { startSTT(context, onChunk) }.getOrThrow()
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return startPCM(silenceThresholdMs, maxDurationMs, onChunk)
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// MARK: - STT path (SpeechRecognizer owns the mic)
|
|
91
|
+
|
|
92
|
+
private suspend fun startSTT(
|
|
93
|
+
context: android.content.Context,
|
|
94
|
+
onChunk: ((chunk: VoiceInputChunk) -> Unit)?,
|
|
95
|
+
): VoiceInputResult = suspendCancellableCoroutine { cont ->
|
|
96
|
+
val recognizer = SpeechRecognizer.createSpeechRecognizer(context)
|
|
97
|
+
activeSpeechRecognizer = recognizer
|
|
98
|
+
|
|
99
|
+
recognizer.setRecognitionListener(object : RecognitionListener {
|
|
100
|
+
override fun onResults(results: Bundle?) {
|
|
101
|
+
activeSpeechRecognizer = null
|
|
102
|
+
recognizer.destroy()
|
|
103
|
+
val text =
|
|
104
|
+
results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)?.firstOrNull()
|
|
105
|
+
cont.resume(VoiceInputResult(transcription = text, audio = null))
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
override fun onError(error: Int) {
|
|
109
|
+
activeSpeechRecognizer = null
|
|
110
|
+
recognizer.destroy()
|
|
111
|
+
// Return empty transcription — caller sees null audio and null transcription
|
|
112
|
+
cont.resume(VoiceInputResult(transcription = null, audio = null))
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
override fun onPartialResults(partialResults: Bundle?) {
|
|
116
|
+
val text = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)
|
|
117
|
+
?.firstOrNull()
|
|
118
|
+
if (!text.isNullOrEmpty()) {
|
|
119
|
+
onChunk?.invoke(VoiceInputChunk(partial = text, audio = null))
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
override fun onReadyForSpeech(params: Bundle?) {}
|
|
124
|
+
override fun onBeginningOfSpeech() {}
|
|
125
|
+
override fun onRmsChanged(rmsdB: Float) {}
|
|
126
|
+
override fun onBufferReceived(buffer: ByteArray?) {}
|
|
127
|
+
override fun onEndOfSpeech() {}
|
|
128
|
+
override fun onEvent(eventType: Int, params: Bundle?) {}
|
|
129
|
+
})
|
|
130
|
+
|
|
131
|
+
val intent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply {
|
|
132
|
+
putExtra(
|
|
133
|
+
RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM
|
|
134
|
+
)
|
|
135
|
+
putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true)
|
|
136
|
+
putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 1)
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
recognizer.startListening(intent)
|
|
140
|
+
|
|
141
|
+
cont.invokeOnCancellation {
|
|
142
|
+
activeSpeechRecognizer = null
|
|
143
|
+
recognizer.destroy()
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// MARK: - STT path fed from CarAudioRecord via a pipe (API 33+)
|
|
148
|
+
@SuppressLint("MissingPermission")
|
|
149
|
+
@RequiresApi(Build.VERSION_CODES.TIRAMISU)
|
|
150
|
+
private suspend fun startSTTFromCarAudio(
|
|
151
|
+
silenceThresholdMs: Long,
|
|
152
|
+
maxDurationMs: Long,
|
|
153
|
+
onChunk: ((chunk: VoiceInputChunk) -> Unit)?,
|
|
154
|
+
): VoiceInputResult {
|
|
155
|
+
if (!hasVoiceInputPermission()) {
|
|
156
|
+
throw SecurityException("RECORD_AUDIO permission not granted")
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
val appContext = NitroModules.applicationContext ?: throw IllegalArgumentException()
|
|
160
|
+
val pipes = ParcelFileDescriptor.createPipe()
|
|
161
|
+
val readFd = pipes[0]
|
|
162
|
+
val pipeOut = ParcelFileDescriptor.AutoCloseOutputStream(pipes[1])
|
|
163
|
+
|
|
164
|
+
val sttDeferred = scope.async {
|
|
165
|
+
ThreadUtil.postOnUiAndAwait {
|
|
166
|
+
startSTTWithSource(appContext, readFd, silenceThresholdMs, onChunk)
|
|
167
|
+
}.getOrThrow()
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
try {
|
|
171
|
+
recordPCM(silenceThresholdMs, maxDurationMs) { chunk ->
|
|
172
|
+
chunk.audio?.let { ab ->
|
|
173
|
+
try {
|
|
174
|
+
pipeOut.write(ab.toByteArray())
|
|
175
|
+
} catch (_: Exception) {
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
} finally {
|
|
180
|
+
try {
|
|
181
|
+
pipeOut.close()
|
|
182
|
+
} catch (_: Exception) {
|
|
183
|
+
}
|
|
184
|
+
try {
|
|
185
|
+
readFd.close()
|
|
186
|
+
} catch (_: Exception) {
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
return sttDeferred.await()
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
@RequiresApi(Build.VERSION_CODES.TIRAMISU)
|
|
194
|
+
private suspend fun startSTTWithSource(
|
|
195
|
+
context: android.content.Context,
|
|
196
|
+
audioSource: ParcelFileDescriptor,
|
|
197
|
+
silenceThresholdMs: Long,
|
|
198
|
+
onChunk: ((chunk: VoiceInputChunk) -> Unit)?,
|
|
199
|
+
): VoiceInputResult = suspendCancellableCoroutine { cont ->
|
|
200
|
+
val recognizer = SpeechRecognizer.createSpeechRecognizer(context)
|
|
201
|
+
activeSpeechRecognizer = recognizer
|
|
202
|
+
// When EXTRA_AUDIO_SOURCE is used, onResults always returns an empty list — the actual
|
|
203
|
+
// transcription only arrives via onPartialResults. Track the last partial here.
|
|
204
|
+
var lastPartial: String? = null
|
|
205
|
+
|
|
206
|
+
recognizer.setRecognitionListener(object : RecognitionListener {
|
|
207
|
+
override fun onResults(results: Bundle?) {
|
|
208
|
+
activeSpeechRecognizer = null
|
|
209
|
+
recognizer.destroy()
|
|
210
|
+
val text =
|
|
211
|
+
results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)?.firstOrNull()
|
|
212
|
+
?: lastPartial
|
|
213
|
+
cont.resume(VoiceInputResult(transcription = text, audio = null))
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
override fun onError(error: Int) {
|
|
217
|
+
activeSpeechRecognizer = null
|
|
218
|
+
recognizer.destroy()
|
|
219
|
+
cont.resume(VoiceInputResult(transcription = lastPartial, audio = null))
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
override fun onPartialResults(partialResults: Bundle?) {
|
|
223
|
+
val text = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)
|
|
224
|
+
?.firstOrNull()
|
|
225
|
+
if (!text.isNullOrEmpty()) {
|
|
226
|
+
lastPartial = text
|
|
227
|
+
onChunk?.invoke(VoiceInputChunk(partial = text, audio = null))
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
override fun onReadyForSpeech(params: Bundle?) {}
|
|
232
|
+
override fun onBeginningOfSpeech() {}
|
|
233
|
+
override fun onRmsChanged(rmsdB: Float) {}
|
|
234
|
+
override fun onBufferReceived(buffer: ByteArray?) {}
|
|
235
|
+
override fun onEndOfSpeech() {}
|
|
236
|
+
override fun onEvent(eventType: Int, params: Bundle?) {}
|
|
237
|
+
})
|
|
238
|
+
|
|
239
|
+
val intent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply {
|
|
240
|
+
putExtra(
|
|
241
|
+
RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM
|
|
242
|
+
)
|
|
243
|
+
putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true)
|
|
244
|
+
putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 1)
|
|
245
|
+
putExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE, audioSource)
|
|
246
|
+
putExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE_CHANNEL_COUNT, 1)
|
|
247
|
+
putExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE_ENCODING, AudioFormat.ENCODING_PCM_16BIT)
|
|
248
|
+
putExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE_SAMPLING_RATE, SAMPLE_RATE)
|
|
249
|
+
putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_MINIMUM_LENGTH_MILLIS, WARMUP_MS)
|
|
250
|
+
putExtra(
|
|
251
|
+
RecognizerIntent.EXTRA_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS,
|
|
252
|
+
silenceThresholdMs
|
|
253
|
+
)
|
|
254
|
+
putExtra(
|
|
255
|
+
RecognizerIntent.EXTRA_SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS,
|
|
256
|
+
silenceThresholdMs / 2,
|
|
257
|
+
)
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
recognizer.startListening(intent)
|
|
261
|
+
|
|
262
|
+
cont.invokeOnCancellation {
|
|
263
|
+
activeSpeechRecognizer = null
|
|
264
|
+
recognizer.destroy()
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
// MARK: - PCM path
|
|
269
|
+
|
|
270
|
+
@RequiresApi(Build.VERSION_CODES.O)
|
|
271
|
+
private suspend fun startPCM(
|
|
272
|
+
silenceThresholdMs: Long,
|
|
273
|
+
maxDurationMs: Long,
|
|
274
|
+
onChunk: ((chunk: VoiceInputChunk) -> Unit)?,
|
|
275
|
+
): VoiceInputResult {
|
|
276
|
+
val pcmBytes = recordPCM(silenceThresholdMs, maxDurationMs, onChunk)
|
|
277
|
+
val directBuffer =
|
|
278
|
+
ByteBuffer.allocateDirect(pcmBytes.size).put(pcmBytes).rewind() as ByteBuffer
|
|
279
|
+
return VoiceInputResult(transcription = null, audio = ArrayBuffer.wrap(directBuffer))
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
@SuppressLint("MissingPermission")
|
|
283
|
+
@RequiresApi(Build.VERSION_CODES.O)
|
|
284
|
+
private suspend fun recordPCM(
|
|
285
|
+
silenceThresholdMs: Long,
|
|
286
|
+
maxDurationMs: Long,
|
|
287
|
+
onChunk: ((chunk: VoiceInputChunk) -> Unit)?,
|
|
55
288
|
): ByteArray = suspendCancellableCoroutine { cont ->
|
|
56
|
-
|
|
57
|
-
if (appContext == null || ContextCompat.checkSelfPermission(
|
|
58
|
-
appContext,
|
|
59
|
-
android.Manifest.permission.RECORD_AUDIO,
|
|
60
|
-
) != PackageManager.PERMISSION_GRANTED
|
|
61
|
-
) {
|
|
289
|
+
if (!hasVoiceInputPermission()) {
|
|
62
290
|
cont.resumeWithException(SecurityException("RECORD_AUDIO permission not granted"))
|
|
63
291
|
return@suspendCancellableCoroutine
|
|
64
292
|
}
|
|
65
293
|
|
|
66
|
-
|
|
294
|
+
val appContext = NitroModules.applicationContext ?: run {
|
|
295
|
+
cont.resumeWithException(SecurityException("Missing application context"))
|
|
296
|
+
return@suspendCancellableCoroutine
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
pcmContinuation = cont
|
|
67
300
|
|
|
68
301
|
val audioManager = appContext.getSystemService(AudioManager::class.java)
|
|
69
302
|
|
|
@@ -80,7 +313,7 @@ class VoiceInputManager(
|
|
|
80
313
|
}.build()
|
|
81
314
|
|
|
82
315
|
if (audioManager.requestAudioFocus(focusRequest) != AudioManager.AUDIOFOCUS_REQUEST_GRANTED) {
|
|
83
|
-
|
|
316
|
+
pcmContinuation = null
|
|
84
317
|
cont.resumeWithException(IllegalStateException("Audio focus request denied"))
|
|
85
318
|
return@suspendCancellableCoroutine
|
|
86
319
|
}
|
|
@@ -136,6 +369,13 @@ class VoiceInputManager(
|
|
|
136
369
|
if (read > 0) {
|
|
137
370
|
outputStream.write(buffer, 0, read)
|
|
138
371
|
|
|
372
|
+
onChunk?.let { cb ->
|
|
373
|
+
val chunk = ByteArray(read) { buffer[it] }
|
|
374
|
+
val direct =
|
|
375
|
+
ByteBuffer.allocateDirect(read).put(chunk).rewind() as ByteBuffer
|
|
376
|
+
cb(VoiceInputChunk(partial = null, audio = ArrayBuffer.wrap(direct)))
|
|
377
|
+
}
|
|
378
|
+
|
|
139
379
|
val now = System.currentTimeMillis()
|
|
140
380
|
val elapsedMs = now - recordingStart
|
|
141
381
|
|
|
@@ -151,7 +391,9 @@ class VoiceInputManager(
|
|
|
151
391
|
val sample =
|
|
152
392
|
(buffer[i].toInt() and 0xFF) or (buffer[i + 1].toInt() shl 8)
|
|
153
393
|
val absSample = abs(sample.toShort().toInt())
|
|
154
|
-
if (absSample > peak)
|
|
394
|
+
if (absSample > peak) {
|
|
395
|
+
peak = absSample
|
|
396
|
+
}
|
|
155
397
|
i += 2
|
|
156
398
|
}
|
|
157
399
|
|
|
@@ -170,14 +412,21 @@ class VoiceInputManager(
|
|
|
170
412
|
}
|
|
171
413
|
} finally {
|
|
172
414
|
releaseResources()
|
|
173
|
-
val
|
|
174
|
-
|
|
175
|
-
|
|
415
|
+
val captured = pcmContinuation
|
|
416
|
+
pcmContinuation = null
|
|
417
|
+
captured?.resume(outputStream.toByteArray())
|
|
176
418
|
}
|
|
177
419
|
}
|
|
178
420
|
}
|
|
179
421
|
|
|
180
422
|
fun stop() {
|
|
423
|
+
// STT path: stopListening() triggers onResults/onError which resolves the continuation
|
|
424
|
+
activeSpeechRecognizer?.let { recognizer ->
|
|
425
|
+
UiThreadUtil.runOnUiThread {
|
|
426
|
+
recognizer.stopListening()
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
// PCM path and car-audio STT pump
|
|
181
430
|
isRecording = false
|
|
182
431
|
carAudioRecord?.stopRecording()
|
|
183
432
|
audioRecord?.stop()
|
|
@@ -210,5 +459,12 @@ class VoiceInputManager(
|
|
|
210
459
|
private const val WARMUP_MS = 500L
|
|
211
460
|
private const val SAMPLE_RATE = 16_000
|
|
212
461
|
private const val PHONE_BUFFER_SIZE = 3_200 // ~100ms at 16kHz/16-bit/mono
|
|
462
|
+
|
|
463
|
+
fun hasVoiceInputPermission(): Boolean {
|
|
464
|
+
val context = NitroModules.applicationContext ?: return false
|
|
465
|
+
return ContextCompat.checkSelfPermission(
|
|
466
|
+
context, android.Manifest.permission.RECORD_AUDIO
|
|
467
|
+
) == PackageManager.PERMISSION_GRANTED
|
|
468
|
+
}
|
|
213
469
|
}
|
|
214
470
|
}
|
package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/utils/ThreadUtil.kt
CHANGED
|
@@ -1,19 +1,12 @@
|
|
|
1
1
|
package com.margelo.nitro.swe.iternio.reactnativeautoplay.utils
|
|
2
2
|
|
|
3
|
-
import
|
|
4
|
-
import kotlinx.coroutines.
|
|
5
|
-
import kotlin.coroutines.resume
|
|
3
|
+
import kotlinx.coroutines.Dispatchers
|
|
4
|
+
import kotlinx.coroutines.withContext
|
|
6
5
|
|
|
7
6
|
object ThreadUtil {
|
|
8
|
-
suspend fun <T> postOnUiAndAwait(block: () -> T): Result<T> =
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
try {
|
|
12
|
-
val result = block()
|
|
13
|
-
cont.resume(Result.success(result))
|
|
14
|
-
} catch (e: Exception) {
|
|
15
|
-
cont.resume(Result.failure(e))
|
|
16
|
-
}
|
|
17
|
-
}
|
|
7
|
+
suspend fun <T> postOnUiAndAwait(block: suspend () -> T): Result<T> = runCatching {
|
|
8
|
+
withContext(Dispatchers.Main) {
|
|
9
|
+
block()
|
|
18
10
|
}
|
|
11
|
+
}
|
|
19
12
|
}
|
|
@@ -21,7 +21,7 @@ class HybridAutoPlay: HybridAutoPlaySpec {
|
|
|
21
21
|
private static var listeners = [EventName: [StateListener]]()
|
|
22
22
|
private static var renderStateListeners = [String: [RenderStateListener]]()
|
|
23
23
|
private static var safeAreaInsetsListeners = [String: [SafeAreaListener]]()
|
|
24
|
-
|
|
24
|
+
|
|
25
25
|
|
|
26
26
|
override init() {
|
|
27
27
|
HybridAutoPlay.listeners.removeAll()
|
|
@@ -119,55 +119,10 @@ class HybridAutoPlay: HybridAutoPlaySpec {
|
|
|
119
119
|
func addListenerVoiceInput(
|
|
120
120
|
callback: @escaping (Location?, String?) -> Void
|
|
121
121
|
) throws -> () -> Void {
|
|
122
|
-
// iOS does not use the OS-triggered voice input path — use
|
|
122
|
+
// iOS does not use the OS-triggered voice input path — use HybridVoice instead.
|
|
123
123
|
return {}
|
|
124
124
|
}
|
|
125
125
|
|
|
126
|
-
func hasVoiceInputPermission() throws -> Bool {
|
|
127
|
-
return AVAudioSession.sharedInstance().recordPermission == .granted
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
func requestVoiceInputPermission() throws -> Promise<Bool> {
|
|
131
|
-
return Promise.async {
|
|
132
|
-
return await withCheckedContinuation { cont in
|
|
133
|
-
AVAudioSession.sharedInstance().requestRecordPermission { granted in
|
|
134
|
-
cont.resume(returning: granted)
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
func startVoiceInput(silenceThresholdMs: Double?, maxDurationMs: Double?, listeningText: String?) throws -> Promise<
|
|
141
|
-
ArrayBuffer
|
|
142
|
-
> {
|
|
143
|
-
return Promise.async {
|
|
144
|
-
let interfaceController = try? await RootModule.withInterfaceController { $0 }
|
|
145
|
-
|
|
146
|
-
let manager = VoiceInputManager()
|
|
147
|
-
HybridAutoPlay.voiceInputManager = manager
|
|
148
|
-
|
|
149
|
-
defer {
|
|
150
|
-
HybridAutoPlay.voiceInputManager = nil
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
let data = try await manager.start(
|
|
154
|
-
interfaceController: interfaceController,
|
|
155
|
-
silenceThresholdMs: silenceThresholdMs ?? 1_500,
|
|
156
|
-
maxDurationMs: maxDurationMs ?? 10_000,
|
|
157
|
-
listeningText: listeningText ?? "Listening..."
|
|
158
|
-
)
|
|
159
|
-
|
|
160
|
-
return try ArrayBuffer.copy(data: data)
|
|
161
|
-
}
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
func stopVoiceInput() throws {
|
|
165
|
-
Task { @MainActor in
|
|
166
|
-
let interfaceController = try? await RootModule.withInterfaceController { $0 }
|
|
167
|
-
HybridAutoPlay.voiceInputManager?.stop(interfaceController: interfaceController)
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
|
|
171
126
|
// MARK: set/push/pop templates
|
|
172
127
|
func setRootTemplate(templateId: String) throws -> Promise<Void> {
|
|
173
128
|
return Promise.async {
|