@iternio/react-native-auto-play 0.4.5 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridAutoPlay.kt +89 -0
- package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/VirtualRenderer.kt +12 -4
- package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/VoiceInputManager.kt +20 -276
- package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/utils/ThreadUtil.kt +13 -6
- package/ios/hybrid/HybridAutoPlay.swift +47 -2
- package/ios/utils/VoiceInputManager.swift +40 -141
- package/lib/index.d.ts +1 -3
- package/lib/index.js +1 -2
- package/lib/specs/AutoPlay.nitro.d.ts +29 -0
- package/lib/templates/MapTemplate.js +1 -6
- package/nitro.json +0 -10
- package/nitrogen/generated/android/ReactNativeAutoPlay+autolinking.cmake +0 -2
- package/nitrogen/generated/android/ReactNativeAutoPlayOnLoad.cpp +0 -18
- package/nitrogen/generated/android/c++/JHybridAutoPlaySpec.cpp +43 -0
- package/nitrogen/generated/android/c++/JHybridAutoPlaySpec.hpp +4 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridAutoPlaySpec.kt +17 -0
- package/nitrogen/generated/ios/ReactNativeAutoPlay-Swift-Cxx-Bridge.cpp +16 -41
- package/nitrogen/generated/ios/ReactNativeAutoPlay-Swift-Cxx-Bridge.hpp +126 -201
- package/nitrogen/generated/ios/ReactNativeAutoPlay-Swift-Cxx-Umbrella.hpp +0 -11
- package/nitrogen/generated/ios/ReactNativeAutoPlayAutolinking.mm +0 -8
- package/nitrogen/generated/ios/ReactNativeAutoPlayAutolinking.swift +0 -12
- package/nitrogen/generated/ios/c++/HybridAutoPlaySpecSwift.hpp +34 -0
- package/nitrogen/generated/ios/swift/Func_void_bool.swift +5 -5
- package/nitrogen/generated/ios/swift/{Func_void_VoiceInputResult.swift → Func_void_std__shared_ptr_ArrayBuffer_.swift} +10 -10
- package/nitrogen/generated/ios/swift/HybridAutoPlaySpec.swift +4 -0
- package/nitrogen/generated/ios/swift/HybridAutoPlaySpec_cxx.swift +82 -0
- package/nitrogen/generated/shared/c++/HybridAutoPlaySpec.cpp +4 -0
- package/nitrogen/generated/shared/c++/HybridAutoPlaySpec.hpp +5 -0
- package/package.json +1 -1
- package/src/index.ts +1 -3
- package/src/specs/AutoPlay.nitro.ts +37 -0
- package/src/templates/MapTemplate.ts +1 -6
- package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridVoice.kt +0 -95
- package/ios/hybrid/HybridVoice.swift +0 -63
- package/lib/HybridAutoPlay.d.ts +0 -2
- package/lib/HybridAutoPlay.js +0 -2
- package/lib/components/OnAppearedChildRenderer.d.ts +0 -10
- package/lib/components/OnAppearedChildRenderer.js +0 -26
- package/lib/hooks/useIsAutoPlayFocused.d.ts +0 -7
- package/lib/hooks/useIsAutoPlayFocused.js +0 -20
- package/lib/hybrid/HybridVoice.d.ts +0 -12
- package/lib/hybrid/HybridVoice.js +0 -13
- package/lib/hybrid.d.ts +0 -2
- package/lib/hybrid.js +0 -2
- package/lib/specs/AutomotivePermissionRequestTemplate.d.ts +0 -11
- package/lib/specs/AutomotivePermissionRequestTemplate.js +0 -1
- package/lib/specs/AutomotivePermissionRequestTemplate.nitro.d.ts +0 -11
- package/lib/specs/AutomotivePermissionRequestTemplate.nitro.js +0 -1
- package/lib/specs/Voice.nitro.d.ts +0 -51
- package/lib/specs/Voice.nitro.js +0 -1
- package/lib/templates/AutomotivePermissionRequestTemplate.d.ts +0 -23
- package/lib/templates/AutomotivePermissionRequestTemplate.js +0 -18
- package/lib/types/Glyphmap.d.ts +0 -4105
- package/lib/types/Glyphmap.js +0 -4105
- package/lib/types/Voice.d.ts +0 -15
- package/lib/types/Voice.js +0 -1
- package/nitrogen/generated/android/c++/JFunc_void_VoiceInputChunk.hpp +0 -81
- package/nitrogen/generated/android/c++/JHybridVoiceSpec.cpp +0 -104
- package/nitrogen/generated/android/c++/JHybridVoiceSpec.hpp +0 -66
- package/nitrogen/generated/android/c++/JVoiceInputChunk.hpp +0 -64
- package/nitrogen/generated/android/c++/JVoiceInputResult.hpp +0 -64
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/swe/iternio/reactnativeautoplay/Func_void_VoiceInputChunk.kt +0 -80
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridVoiceSpec.kt +0 -72
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/swe/iternio/reactnativeautoplay/VoiceInputChunk.kt +0 -41
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/swe/iternio/reactnativeautoplay/VoiceInputResult.kt +0 -41
- package/nitrogen/generated/ios/c++/HybridVoiceSpecSwift.cpp +0 -11
- package/nitrogen/generated/ios/c++/HybridVoiceSpecSwift.hpp +0 -116
- package/nitrogen/generated/ios/swift/Func_void_VoiceInputChunk.swift +0 -46
- package/nitrogen/generated/ios/swift/HybridVoiceSpec.swift +0 -58
- package/nitrogen/generated/ios/swift/HybridVoiceSpec_cxx.swift +0 -227
- package/nitrogen/generated/ios/swift/VoiceInputChunk.swift +0 -60
- package/nitrogen/generated/ios/swift/VoiceInputResult.swift +0 -60
- package/nitrogen/generated/shared/c++/HybridVoiceSpec.cpp +0 -24
- package/nitrogen/generated/shared/c++/HybridVoiceSpec.hpp +0 -73
- package/nitrogen/generated/shared/c++/VoiceInputChunk.hpp +0 -89
- package/nitrogen/generated/shared/c++/VoiceInputResult.hpp +0 -89
- package/src/components/OnAppearedChildRenderer.tsx +0 -37
- package/src/hybrid/HybridVoice.ts +0 -30
- package/src/specs/Voice.nitro.ts +0 -58
- package/src/types/Voice.ts +0 -17
package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridAutoPlay.kt
CHANGED
|
@@ -1,11 +1,19 @@
|
|
|
1
1
|
package com.margelo.nitro.swe.iternio.reactnativeautoplay
|
|
2
2
|
|
|
3
|
+
import android.content.pm.PackageManager
|
|
3
4
|
import android.os.Build
|
|
5
|
+
import androidx.core.content.ContextCompat
|
|
4
6
|
import com.facebook.react.bridge.UiThreadUtil
|
|
7
|
+
import com.facebook.react.modules.core.PermissionAwareActivity
|
|
8
|
+
import com.facebook.react.modules.core.PermissionListener
|
|
9
|
+
import com.margelo.nitro.NitroModules
|
|
10
|
+
import com.margelo.nitro.core.ArrayBuffer
|
|
5
11
|
import com.margelo.nitro.core.Promise
|
|
6
12
|
import com.margelo.nitro.swe.iternio.reactnativeautoplay.template.AndroidAutoTemplate
|
|
7
13
|
import com.margelo.nitro.swe.iternio.reactnativeautoplay.template.MessageTemplate
|
|
8
14
|
import com.margelo.nitro.swe.iternio.reactnativeautoplay.utils.ThreadUtil
|
|
15
|
+
import kotlinx.coroutines.suspendCancellableCoroutine
|
|
16
|
+
import java.nio.ByteBuffer
|
|
9
17
|
import java.util.concurrent.ConcurrentHashMap
|
|
10
18
|
import java.util.concurrent.CopyOnWriteArrayList
|
|
11
19
|
import kotlin.coroutines.resume
|
|
@@ -243,6 +251,84 @@ class HybridAutoPlay : HybridAutoPlaySpec() {
|
|
|
243
251
|
}
|
|
244
252
|
}
|
|
245
253
|
|
|
254
|
+
override fun hasVoiceInputPermission(): Boolean {
|
|
255
|
+
val context = NitroModules.applicationContext ?: return false
|
|
256
|
+
return ContextCompat.checkSelfPermission(
|
|
257
|
+
context, android.Manifest.permission.RECORD_AUDIO
|
|
258
|
+
) == PackageManager.PERMISSION_GRANTED
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
override fun requestVoiceInputPermission(): Promise<Boolean> {
|
|
262
|
+
return Promise.async {
|
|
263
|
+
if (hasVoiceInputPermission()) {
|
|
264
|
+
return@async true
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
val carContext = AndroidAutoSession.getRootContext()
|
|
268
|
+
|
|
269
|
+
if (carContext != null) {
|
|
270
|
+
suspendCancellableCoroutine {
|
|
271
|
+
carContext.requestPermissions(listOf(android.Manifest.permission.RECORD_AUDIO)) { approved, _ ->
|
|
272
|
+
it.resume(approved.contains(android.Manifest.permission.RECORD_AUDIO))
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
} else {
|
|
276
|
+
val context = NitroModules.applicationContext ?: return@async false
|
|
277
|
+
val activity =
|
|
278
|
+
context.currentActivity as? PermissionAwareActivity ?: return@async false
|
|
279
|
+
val code = (Math.random() * 10000).toInt()
|
|
280
|
+
|
|
281
|
+
suspendCancellableCoroutine {
|
|
282
|
+
activity.requestPermissions(
|
|
283
|
+
arrayOf(android.Manifest.permission.RECORD_AUDIO),
|
|
284
|
+
code,
|
|
285
|
+
PermissionListener { requestCode, _, grantResults ->
|
|
286
|
+
if (requestCode != code) {
|
|
287
|
+
return@PermissionListener false
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
val granted =
|
|
291
|
+
grantResults.isNotEmpty() && grantResults.first() == PackageManager.PERMISSION_GRANTED
|
|
292
|
+
|
|
293
|
+
it.resume(granted)
|
|
294
|
+
|
|
295
|
+
return@PermissionListener true
|
|
296
|
+
})
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
override fun startVoiceInput(
|
|
303
|
+
silenceThresholdMs: Double?, maxDurationMs: Double?, listeningText: String?
|
|
304
|
+
): Promise<ArrayBuffer> {
|
|
305
|
+
return Promise.async {
|
|
306
|
+
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) {
|
|
307
|
+
throw UnsupportedOperationException("startVoiceInput requires at least API level ${Build.VERSION_CODES.O}")
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
val manager = VoiceInputManager(AndroidAutoSession.getRootContext())
|
|
311
|
+
voiceInputManager = manager
|
|
312
|
+
|
|
313
|
+
try {
|
|
314
|
+
val pcmBytes = manager.start(
|
|
315
|
+
silenceThresholdMs = silenceThresholdMs?.toLong() ?: 1_500L,
|
|
316
|
+
maxDurationMs = maxDurationMs?.toLong() ?: 10_000L,
|
|
317
|
+
)
|
|
318
|
+
val directBuffer =
|
|
319
|
+
ByteBuffer.allocateDirect(pcmBytes.size).put(pcmBytes).rewind() as ByteBuffer
|
|
320
|
+
ArrayBuffer.wrap(directBuffer)
|
|
321
|
+
} finally {
|
|
322
|
+
voiceInputManager = null
|
|
323
|
+
manager.dispose()
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
override fun stopVoiceInput() {
|
|
329
|
+
voiceInputManager?.stop()
|
|
330
|
+
}
|
|
331
|
+
|
|
246
332
|
companion object {
|
|
247
333
|
const val TAG = "HybridAutoPlay"
|
|
248
334
|
|
|
@@ -253,6 +339,9 @@ class HybridAutoPlay : HybridAutoPlaySpec() {
|
|
|
253
339
|
|
|
254
340
|
private val voiceInputListeners = CopyOnWriteArrayList<(Location?, String?) -> Unit>()
|
|
255
341
|
|
|
342
|
+
@Volatile
|
|
343
|
+
private var voiceInputManager: VoiceInputManager? = null
|
|
344
|
+
|
|
256
345
|
private val safeAreaInsetsListeners =
|
|
257
346
|
ConcurrentHashMap<String, CopyOnWriteArrayList<(SafeAreaInsets) -> Unit>>()
|
|
258
347
|
|
package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/VirtualRenderer.kt
CHANGED
|
@@ -7,6 +7,7 @@ import android.graphics.Rect
|
|
|
7
7
|
import android.hardware.display.DisplayManager
|
|
8
8
|
import android.hardware.display.VirtualDisplay
|
|
9
9
|
import android.os.Bundle
|
|
10
|
+
import android.view.ContextThemeWrapper
|
|
10
11
|
import android.view.Display
|
|
11
12
|
import android.view.LayoutInflater
|
|
12
13
|
import android.view.View
|
|
@@ -78,6 +79,7 @@ class VirtualRenderer(
|
|
|
78
79
|
}
|
|
79
80
|
|
|
80
81
|
val manager = context.getSystemService(Context.DISPLAY_SERVICE) as DisplayManager
|
|
82
|
+
virtualDisplay?.release()
|
|
81
83
|
virtualDisplay = manager.createVirtualDisplay(
|
|
82
84
|
moduleName,
|
|
83
85
|
surfaceContainer.width,
|
|
@@ -282,8 +284,14 @@ class VirtualRenderer(
|
|
|
282
284
|
override fun onCreate(savedInstanceState: Bundle?) {
|
|
283
285
|
super.onCreate(savedInstanceState)
|
|
284
286
|
|
|
287
|
+
// Wrap applicationContext with the app theme so AppCompat widgets (e.g. ReactTextView)
|
|
288
|
+
// can resolve their required text-appearance attrs on OEM themes that don't define them
|
|
289
|
+
// (notably Polestar/Volvo Android Automotive: Theme.DeviceDefault.Light.DarkActionBar).
|
|
290
|
+
val appTheme = context.applicationContext.applicationInfo.theme
|
|
291
|
+
val themedContext = ContextThemeWrapper(context.applicationContext, appTheme)
|
|
292
|
+
|
|
285
293
|
if (!this@VirtualRenderer::reactSurfaceImpl.isInitialized) {
|
|
286
|
-
reactSurfaceImpl = ReactSurfaceImpl(
|
|
294
|
+
reactSurfaceImpl = ReactSurfaceImpl(themedContext, moduleName, initialProperties)
|
|
287
295
|
}
|
|
288
296
|
|
|
289
297
|
var splashScreenView: View? = null
|
|
@@ -292,9 +300,9 @@ class VirtualRenderer(
|
|
|
292
300
|
(it.parent as ViewGroup).removeView(it)
|
|
293
301
|
} ?: run {
|
|
294
302
|
splashScreenView =
|
|
295
|
-
if (isCluster) getClusterSplashScreen(
|
|
303
|
+
if (isCluster) getClusterSplashScreen(themedContext, height, width) else null
|
|
296
304
|
|
|
297
|
-
val surfaceView = ReactSurfaceView(
|
|
305
|
+
val surfaceView = ReactSurfaceView(themedContext, reactSurfaceImpl).apply {
|
|
298
306
|
layoutParams = FrameLayout.LayoutParams(
|
|
299
307
|
(width / reactNativeScale).toInt(), (height / reactNativeScale).toInt()
|
|
300
308
|
)
|
|
@@ -330,7 +338,7 @@ class VirtualRenderer(
|
|
|
330
338
|
}
|
|
331
339
|
|
|
332
340
|
|
|
333
|
-
val rootContainer = FrameLayout(
|
|
341
|
+
val rootContainer = FrameLayout(themedContext).apply {
|
|
334
342
|
layoutParams = FrameLayout.LayoutParams(
|
|
335
343
|
FrameLayout.LayoutParams.MATCH_PARENT, FrameLayout.LayoutParams.MATCH_PARENT
|
|
336
344
|
)
|
package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/VoiceInputManager.kt
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
package com.margelo.nitro.swe.iternio.reactnativeautoplay
|
|
2
2
|
|
|
3
|
-
import android.annotation.SuppressLint
|
|
4
|
-
import android.content.Intent
|
|
5
3
|
import android.content.pm.PackageManager
|
|
6
4
|
import android.media.AudioAttributes
|
|
7
5
|
import android.media.AudioFocusRequest
|
|
@@ -10,28 +8,18 @@ import android.media.AudioManager
|
|
|
10
8
|
import android.media.AudioRecord
|
|
11
9
|
import android.media.MediaRecorder
|
|
12
10
|
import android.os.Build
|
|
13
|
-
import android.os.Bundle
|
|
14
|
-
import android.os.ParcelFileDescriptor
|
|
15
|
-
import android.speech.RecognitionListener
|
|
16
|
-
import android.speech.RecognizerIntent
|
|
17
|
-
import android.speech.SpeechRecognizer
|
|
18
11
|
import androidx.annotation.RequiresApi
|
|
19
12
|
import androidx.car.app.CarContext
|
|
20
13
|
import androidx.car.app.media.CarAudioRecord
|
|
21
14
|
import androidx.core.content.ContextCompat
|
|
22
|
-
import com.facebook.react.bridge.UiThreadUtil
|
|
23
15
|
import com.margelo.nitro.NitroModules
|
|
24
|
-
import com.margelo.nitro.core.ArrayBuffer
|
|
25
|
-
import com.margelo.nitro.swe.iternio.reactnativeautoplay.utils.ThreadUtil
|
|
26
16
|
import kotlinx.coroutines.CoroutineScope
|
|
27
17
|
import kotlinx.coroutines.Dispatchers
|
|
28
18
|
import kotlinx.coroutines.Job
|
|
29
|
-
import kotlinx.coroutines.async
|
|
30
19
|
import kotlinx.coroutines.cancel
|
|
31
20
|
import kotlinx.coroutines.launch
|
|
32
21
|
import kotlinx.coroutines.suspendCancellableCoroutine
|
|
33
22
|
import java.io.ByteArrayOutputStream
|
|
34
|
-
import java.nio.ByteBuffer
|
|
35
23
|
import kotlin.coroutines.Continuation
|
|
36
24
|
import kotlin.coroutines.resume
|
|
37
25
|
import kotlin.coroutines.resumeWithException
|
|
@@ -39,264 +27,43 @@ import kotlin.math.abs
|
|
|
39
27
|
|
|
40
28
|
/**
|
|
41
29
|
* Captures 16-bit PCM audio (16 kHz, mono).
|
|
42
|
-
* When [carContext] is provided uses CarAudioRecord (Android Auto/Automotive)
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
* When preferSpeechToText is true and SpeechRecognizer is available, it owns
|
|
46
|
-
* the microphone and streams partial results; the PCM path is not used.
|
|
47
|
-
* When SpeechRecognizer is unavailable the manager falls back to PCM recording.
|
|
30
|
+
* When [carContext] is provided uses CarAudioRecord (Android Auto/Automotive).
|
|
31
|
+
* When [carContext] is null falls back to standard AudioRecord (phone-only).
|
|
48
32
|
*/
|
|
49
33
|
class VoiceInputManager(
|
|
50
34
|
private val carContext: CarContext?,
|
|
51
35
|
) {
|
|
52
|
-
// PCM recording state
|
|
53
36
|
private var carAudioRecord: CarAudioRecord? = null
|
|
54
37
|
private var audioRecord: AudioRecord? = null
|
|
55
38
|
private var audioFocusRequest: AudioFocusRequest? = null
|
|
56
39
|
private var recordingJob: Job? = null
|
|
57
|
-
private var
|
|
40
|
+
private var continuation: Continuation<ByteArray>? = null
|
|
58
41
|
private val scope = CoroutineScope(Dispatchers.IO)
|
|
59
42
|
|
|
60
43
|
@Volatile
|
|
61
44
|
private var isRecording = false
|
|
62
45
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
46
|
+
/**
|
|
47
|
+
* Acquires audio focus, starts recording, and suspends until stopped.
|
|
48
|
+
* Stops automatically after [silenceThresholdMs] of silence or [maxDurationMs] total.
|
|
49
|
+
* Returns the complete raw PCM buffer (Int16 LE, 16 kHz, mono).
|
|
50
|
+
*/
|
|
67
51
|
@RequiresApi(Build.VERSION_CODES.O)
|
|
68
52
|
suspend fun start(
|
|
69
53
|
silenceThresholdMs: Long = 1_500,
|
|
70
54
|
maxDurationMs: Long = 10_000,
|
|
71
|
-
preferSpeechToText: Boolean = false,
|
|
72
|
-
onChunk: ((chunk: VoiceInputChunk) -> Unit)? = null,
|
|
73
|
-
): VoiceInputResult {
|
|
74
|
-
if (preferSpeechToText) {
|
|
75
|
-
val context = NitroModules.applicationContext ?: throw IllegalArgumentException()
|
|
76
|
-
if (SpeechRecognizer.isRecognitionAvailable(context)) {
|
|
77
|
-
if (carContext != null) {
|
|
78
|
-
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
|
|
79
|
-
return startSTTFromCarAudio(silenceThresholdMs, maxDurationMs, onChunk)
|
|
80
|
-
}
|
|
81
|
-
// Car connected but API < 33: EXTRA_AUDIO_SOURCE unavailable, fall back to PCM
|
|
82
|
-
return startPCM(silenceThresholdMs, maxDurationMs, onChunk)
|
|
83
|
-
}
|
|
84
|
-
return ThreadUtil.postOnUiAndAwait { startSTT(context, onChunk) }.getOrThrow()
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
return startPCM(silenceThresholdMs, maxDurationMs, onChunk)
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
// MARK: - STT path (SpeechRecognizer owns the mic)
|
|
91
|
-
|
|
92
|
-
private suspend fun startSTT(
|
|
93
|
-
context: android.content.Context,
|
|
94
|
-
onChunk: ((chunk: VoiceInputChunk) -> Unit)?,
|
|
95
|
-
): VoiceInputResult = suspendCancellableCoroutine { cont ->
|
|
96
|
-
val recognizer = SpeechRecognizer.createSpeechRecognizer(context)
|
|
97
|
-
activeSpeechRecognizer = recognizer
|
|
98
|
-
|
|
99
|
-
recognizer.setRecognitionListener(object : RecognitionListener {
|
|
100
|
-
override fun onResults(results: Bundle?) {
|
|
101
|
-
activeSpeechRecognizer = null
|
|
102
|
-
recognizer.destroy()
|
|
103
|
-
val text =
|
|
104
|
-
results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)?.firstOrNull()
|
|
105
|
-
cont.resume(VoiceInputResult(transcription = text, audio = null))
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
override fun onError(error: Int) {
|
|
109
|
-
activeSpeechRecognizer = null
|
|
110
|
-
recognizer.destroy()
|
|
111
|
-
// Return empty transcription — caller sees null audio and null transcription
|
|
112
|
-
cont.resume(VoiceInputResult(transcription = null, audio = null))
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
override fun onPartialResults(partialResults: Bundle?) {
|
|
116
|
-
val text = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)
|
|
117
|
-
?.firstOrNull()
|
|
118
|
-
if (!text.isNullOrEmpty()) {
|
|
119
|
-
onChunk?.invoke(VoiceInputChunk(partial = text, audio = null))
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
override fun onReadyForSpeech(params: Bundle?) {}
|
|
124
|
-
override fun onBeginningOfSpeech() {}
|
|
125
|
-
override fun onRmsChanged(rmsdB: Float) {}
|
|
126
|
-
override fun onBufferReceived(buffer: ByteArray?) {}
|
|
127
|
-
override fun onEndOfSpeech() {}
|
|
128
|
-
override fun onEvent(eventType: Int, params: Bundle?) {}
|
|
129
|
-
})
|
|
130
|
-
|
|
131
|
-
val intent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply {
|
|
132
|
-
putExtra(
|
|
133
|
-
RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM
|
|
134
|
-
)
|
|
135
|
-
putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true)
|
|
136
|
-
putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 1)
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
recognizer.startListening(intent)
|
|
140
|
-
|
|
141
|
-
cont.invokeOnCancellation {
|
|
142
|
-
activeSpeechRecognizer = null
|
|
143
|
-
recognizer.destroy()
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
// MARK: - STT path fed from CarAudioRecord via a pipe (API 33+)
|
|
148
|
-
@SuppressLint("MissingPermission")
|
|
149
|
-
@RequiresApi(Build.VERSION_CODES.TIRAMISU)
|
|
150
|
-
private suspend fun startSTTFromCarAudio(
|
|
151
|
-
silenceThresholdMs: Long,
|
|
152
|
-
maxDurationMs: Long,
|
|
153
|
-
onChunk: ((chunk: VoiceInputChunk) -> Unit)?,
|
|
154
|
-
): VoiceInputResult {
|
|
155
|
-
if (!hasVoiceInputPermission()) {
|
|
156
|
-
throw SecurityException("RECORD_AUDIO permission not granted")
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
val appContext = NitroModules.applicationContext ?: throw IllegalArgumentException()
|
|
160
|
-
val pipes = ParcelFileDescriptor.createPipe()
|
|
161
|
-
val readFd = pipes[0]
|
|
162
|
-
val pipeOut = ParcelFileDescriptor.AutoCloseOutputStream(pipes[1])
|
|
163
|
-
|
|
164
|
-
val sttDeferred = scope.async {
|
|
165
|
-
ThreadUtil.postOnUiAndAwait {
|
|
166
|
-
startSTTWithSource(appContext, readFd, silenceThresholdMs, onChunk)
|
|
167
|
-
}.getOrThrow()
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
try {
|
|
171
|
-
recordPCM(silenceThresholdMs, maxDurationMs) { chunk ->
|
|
172
|
-
chunk.audio?.let { ab ->
|
|
173
|
-
try {
|
|
174
|
-
pipeOut.write(ab.toByteArray())
|
|
175
|
-
} catch (_: Exception) {
|
|
176
|
-
}
|
|
177
|
-
}
|
|
178
|
-
}
|
|
179
|
-
} finally {
|
|
180
|
-
try {
|
|
181
|
-
pipeOut.close()
|
|
182
|
-
} catch (_: Exception) {
|
|
183
|
-
}
|
|
184
|
-
try {
|
|
185
|
-
readFd.close()
|
|
186
|
-
} catch (_: Exception) {
|
|
187
|
-
}
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
return sttDeferred.await()
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
@RequiresApi(Build.VERSION_CODES.TIRAMISU)
|
|
194
|
-
private suspend fun startSTTWithSource(
|
|
195
|
-
context: android.content.Context,
|
|
196
|
-
audioSource: ParcelFileDescriptor,
|
|
197
|
-
silenceThresholdMs: Long,
|
|
198
|
-
onChunk: ((chunk: VoiceInputChunk) -> Unit)?,
|
|
199
|
-
): VoiceInputResult = suspendCancellableCoroutine { cont ->
|
|
200
|
-
val recognizer = SpeechRecognizer.createSpeechRecognizer(context)
|
|
201
|
-
activeSpeechRecognizer = recognizer
|
|
202
|
-
// When EXTRA_AUDIO_SOURCE is used, onResults always returns an empty list — the actual
|
|
203
|
-
// transcription only arrives via onPartialResults. Track the last partial here.
|
|
204
|
-
var lastPartial: String? = null
|
|
205
|
-
|
|
206
|
-
recognizer.setRecognitionListener(object : RecognitionListener {
|
|
207
|
-
override fun onResults(results: Bundle?) {
|
|
208
|
-
activeSpeechRecognizer = null
|
|
209
|
-
recognizer.destroy()
|
|
210
|
-
val text =
|
|
211
|
-
results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)?.firstOrNull()
|
|
212
|
-
?: lastPartial
|
|
213
|
-
cont.resume(VoiceInputResult(transcription = text, audio = null))
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
override fun onError(error: Int) {
|
|
217
|
-
activeSpeechRecognizer = null
|
|
218
|
-
recognizer.destroy()
|
|
219
|
-
cont.resume(VoiceInputResult(transcription = lastPartial, audio = null))
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
override fun onPartialResults(partialResults: Bundle?) {
|
|
223
|
-
val text = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)
|
|
224
|
-
?.firstOrNull()
|
|
225
|
-
if (!text.isNullOrEmpty()) {
|
|
226
|
-
lastPartial = text
|
|
227
|
-
onChunk?.invoke(VoiceInputChunk(partial = text, audio = null))
|
|
228
|
-
}
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
override fun onReadyForSpeech(params: Bundle?) {}
|
|
232
|
-
override fun onBeginningOfSpeech() {}
|
|
233
|
-
override fun onRmsChanged(rmsdB: Float) {}
|
|
234
|
-
override fun onBufferReceived(buffer: ByteArray?) {}
|
|
235
|
-
override fun onEndOfSpeech() {}
|
|
236
|
-
override fun onEvent(eventType: Int, params: Bundle?) {}
|
|
237
|
-
})
|
|
238
|
-
|
|
239
|
-
val intent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply {
|
|
240
|
-
putExtra(
|
|
241
|
-
RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM
|
|
242
|
-
)
|
|
243
|
-
putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true)
|
|
244
|
-
putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 1)
|
|
245
|
-
putExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE, audioSource)
|
|
246
|
-
putExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE_CHANNEL_COUNT, 1)
|
|
247
|
-
putExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE_ENCODING, AudioFormat.ENCODING_PCM_16BIT)
|
|
248
|
-
putExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE_SAMPLING_RATE, SAMPLE_RATE)
|
|
249
|
-
putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_MINIMUM_LENGTH_MILLIS, WARMUP_MS)
|
|
250
|
-
putExtra(
|
|
251
|
-
RecognizerIntent.EXTRA_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS,
|
|
252
|
-
silenceThresholdMs
|
|
253
|
-
)
|
|
254
|
-
putExtra(
|
|
255
|
-
RecognizerIntent.EXTRA_SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS,
|
|
256
|
-
silenceThresholdMs / 2,
|
|
257
|
-
)
|
|
258
|
-
}
|
|
259
|
-
|
|
260
|
-
recognizer.startListening(intent)
|
|
261
|
-
|
|
262
|
-
cont.invokeOnCancellation {
|
|
263
|
-
activeSpeechRecognizer = null
|
|
264
|
-
recognizer.destroy()
|
|
265
|
-
}
|
|
266
|
-
}
|
|
267
|
-
|
|
268
|
-
// MARK: - PCM path
|
|
269
|
-
|
|
270
|
-
@RequiresApi(Build.VERSION_CODES.O)
|
|
271
|
-
private suspend fun startPCM(
|
|
272
|
-
silenceThresholdMs: Long,
|
|
273
|
-
maxDurationMs: Long,
|
|
274
|
-
onChunk: ((chunk: VoiceInputChunk) -> Unit)?,
|
|
275
|
-
): VoiceInputResult {
|
|
276
|
-
val pcmBytes = recordPCM(silenceThresholdMs, maxDurationMs, onChunk)
|
|
277
|
-
val directBuffer =
|
|
278
|
-
ByteBuffer.allocateDirect(pcmBytes.size).put(pcmBytes).rewind() as ByteBuffer
|
|
279
|
-
return VoiceInputResult(transcription = null, audio = ArrayBuffer.wrap(directBuffer))
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
@SuppressLint("MissingPermission")
|
|
283
|
-
@RequiresApi(Build.VERSION_CODES.O)
|
|
284
|
-
private suspend fun recordPCM(
|
|
285
|
-
silenceThresholdMs: Long,
|
|
286
|
-
maxDurationMs: Long,
|
|
287
|
-
onChunk: ((chunk: VoiceInputChunk) -> Unit)?,
|
|
288
55
|
): ByteArray = suspendCancellableCoroutine { cont ->
|
|
289
|
-
|
|
56
|
+
val appContext = NitroModules.applicationContext
|
|
57
|
+
if (appContext == null || ContextCompat.checkSelfPermission(
|
|
58
|
+
appContext,
|
|
59
|
+
android.Manifest.permission.RECORD_AUDIO,
|
|
60
|
+
) != PackageManager.PERMISSION_GRANTED
|
|
61
|
+
) {
|
|
290
62
|
cont.resumeWithException(SecurityException("RECORD_AUDIO permission not granted"))
|
|
291
63
|
return@suspendCancellableCoroutine
|
|
292
64
|
}
|
|
293
65
|
|
|
294
|
-
|
|
295
|
-
cont.resumeWithException(SecurityException("Missing application context"))
|
|
296
|
-
return@suspendCancellableCoroutine
|
|
297
|
-
}
|
|
298
|
-
|
|
299
|
-
pcmContinuation = cont
|
|
66
|
+
continuation = cont
|
|
300
67
|
|
|
301
68
|
val audioManager = appContext.getSystemService(AudioManager::class.java)
|
|
302
69
|
|
|
@@ -313,7 +80,7 @@ class VoiceInputManager(
|
|
|
313
80
|
}.build()
|
|
314
81
|
|
|
315
82
|
if (audioManager.requestAudioFocus(focusRequest) != AudioManager.AUDIOFOCUS_REQUEST_GRANTED) {
|
|
316
|
-
|
|
83
|
+
continuation = null
|
|
317
84
|
cont.resumeWithException(IllegalStateException("Audio focus request denied"))
|
|
318
85
|
return@suspendCancellableCoroutine
|
|
319
86
|
}
|
|
@@ -369,13 +136,6 @@ class VoiceInputManager(
|
|
|
369
136
|
if (read > 0) {
|
|
370
137
|
outputStream.write(buffer, 0, read)
|
|
371
138
|
|
|
372
|
-
onChunk?.let { cb ->
|
|
373
|
-
val chunk = ByteArray(read) { buffer[it] }
|
|
374
|
-
val direct =
|
|
375
|
-
ByteBuffer.allocateDirect(read).put(chunk).rewind() as ByteBuffer
|
|
376
|
-
cb(VoiceInputChunk(partial = null, audio = ArrayBuffer.wrap(direct)))
|
|
377
|
-
}
|
|
378
|
-
|
|
379
139
|
val now = System.currentTimeMillis()
|
|
380
140
|
val elapsedMs = now - recordingStart
|
|
381
141
|
|
|
@@ -391,9 +151,7 @@ class VoiceInputManager(
|
|
|
391
151
|
val sample =
|
|
392
152
|
(buffer[i].toInt() and 0xFF) or (buffer[i + 1].toInt() shl 8)
|
|
393
153
|
val absSample = abs(sample.toShort().toInt())
|
|
394
|
-
if (absSample > peak)
|
|
395
|
-
peak = absSample
|
|
396
|
-
}
|
|
154
|
+
if (absSample > peak) peak = absSample
|
|
397
155
|
i += 2
|
|
398
156
|
}
|
|
399
157
|
|
|
@@ -412,21 +170,14 @@ class VoiceInputManager(
|
|
|
412
170
|
}
|
|
413
171
|
} finally {
|
|
414
172
|
releaseResources()
|
|
415
|
-
val
|
|
416
|
-
|
|
417
|
-
|
|
173
|
+
val capturedContinuation = continuation
|
|
174
|
+
continuation = null
|
|
175
|
+
capturedContinuation?.resume(outputStream.toByteArray())
|
|
418
176
|
}
|
|
419
177
|
}
|
|
420
178
|
}
|
|
421
179
|
|
|
422
180
|
fun stop() {
|
|
423
|
-
// STT path: stopListening() triggers onResults/onError which resolves the continuation
|
|
424
|
-
activeSpeechRecognizer?.let { recognizer ->
|
|
425
|
-
UiThreadUtil.runOnUiThread {
|
|
426
|
-
recognizer.stopListening()
|
|
427
|
-
}
|
|
428
|
-
}
|
|
429
|
-
// PCM path and car-audio STT pump
|
|
430
181
|
isRecording = false
|
|
431
182
|
carAudioRecord?.stopRecording()
|
|
432
183
|
audioRecord?.stop()
|
|
@@ -459,12 +210,5 @@ class VoiceInputManager(
|
|
|
459
210
|
private const val WARMUP_MS = 500L
|
|
460
211
|
private const val SAMPLE_RATE = 16_000
|
|
461
212
|
private const val PHONE_BUFFER_SIZE = 3_200 // ~100ms at 16kHz/16-bit/mono
|
|
462
|
-
|
|
463
|
-
fun hasVoiceInputPermission(): Boolean {
|
|
464
|
-
val context = NitroModules.applicationContext ?: return false
|
|
465
|
-
return ContextCompat.checkSelfPermission(
|
|
466
|
-
context, android.Manifest.permission.RECORD_AUDIO
|
|
467
|
-
) == PackageManager.PERMISSION_GRANTED
|
|
468
|
-
}
|
|
469
213
|
}
|
|
470
214
|
}
|
package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/utils/ThreadUtil.kt
CHANGED
|
@@ -1,12 +1,19 @@
|
|
|
1
1
|
package com.margelo.nitro.swe.iternio.reactnativeautoplay.utils
|
|
2
2
|
|
|
3
|
-
import
|
|
4
|
-
import kotlinx.coroutines.
|
|
3
|
+
import com.facebook.react.bridge.UiThreadUtil
|
|
4
|
+
import kotlinx.coroutines.suspendCancellableCoroutine
|
|
5
|
+
import kotlin.coroutines.resume
|
|
5
6
|
|
|
6
7
|
object ThreadUtil {
|
|
7
|
-
suspend fun <T> postOnUiAndAwait(block:
|
|
8
|
-
|
|
9
|
-
|
|
8
|
+
suspend fun <T> postOnUiAndAwait(block: () -> T): Result<T> =
|
|
9
|
+
suspendCancellableCoroutine { cont ->
|
|
10
|
+
UiThreadUtil.runOnUiThread {
|
|
11
|
+
try {
|
|
12
|
+
val result = block()
|
|
13
|
+
cont.resume(Result.success(result))
|
|
14
|
+
} catch (e: Exception) {
|
|
15
|
+
cont.resume(Result.failure(e))
|
|
16
|
+
}
|
|
17
|
+
}
|
|
10
18
|
}
|
|
11
|
-
}
|
|
12
19
|
}
|
|
@@ -21,7 +21,7 @@ class HybridAutoPlay: HybridAutoPlaySpec {
|
|
|
21
21
|
private static var listeners = [EventName: [StateListener]]()
|
|
22
22
|
private static var renderStateListeners = [String: [RenderStateListener]]()
|
|
23
23
|
private static var safeAreaInsetsListeners = [String: [SafeAreaListener]]()
|
|
24
|
-
|
|
24
|
+
private static var voiceInputManager: VoiceInputManager?
|
|
25
25
|
|
|
26
26
|
override init() {
|
|
27
27
|
HybridAutoPlay.listeners.removeAll()
|
|
@@ -119,10 +119,55 @@ class HybridAutoPlay: HybridAutoPlaySpec {
|
|
|
119
119
|
func addListenerVoiceInput(
|
|
120
120
|
callback: @escaping (Location?, String?) -> Void
|
|
121
121
|
) throws -> () -> Void {
|
|
122
|
-
// iOS does not use the OS-triggered voice input path — use
|
|
122
|
+
// iOS does not use the OS-triggered voice input path — use startVoiceInput() instead.
|
|
123
123
|
return {}
|
|
124
124
|
}
|
|
125
125
|
|
|
126
|
+
func hasVoiceInputPermission() throws -> Bool {
|
|
127
|
+
return AVAudioSession.sharedInstance().recordPermission == .granted
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
func requestVoiceInputPermission() throws -> Promise<Bool> {
|
|
131
|
+
return Promise.async {
|
|
132
|
+
return await withCheckedContinuation { cont in
|
|
133
|
+
AVAudioSession.sharedInstance().requestRecordPermission { granted in
|
|
134
|
+
cont.resume(returning: granted)
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
func startVoiceInput(silenceThresholdMs: Double?, maxDurationMs: Double?, listeningText: String?) throws -> Promise<
|
|
141
|
+
ArrayBuffer
|
|
142
|
+
> {
|
|
143
|
+
return Promise.async {
|
|
144
|
+
let interfaceController = try? await RootModule.withInterfaceController { $0 }
|
|
145
|
+
|
|
146
|
+
let manager = VoiceInputManager()
|
|
147
|
+
HybridAutoPlay.voiceInputManager = manager
|
|
148
|
+
|
|
149
|
+
defer {
|
|
150
|
+
HybridAutoPlay.voiceInputManager = nil
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
let data = try await manager.start(
|
|
154
|
+
interfaceController: interfaceController,
|
|
155
|
+
silenceThresholdMs: silenceThresholdMs ?? 1_500,
|
|
156
|
+
maxDurationMs: maxDurationMs ?? 10_000,
|
|
157
|
+
listeningText: listeningText ?? "Listening..."
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
return try ArrayBuffer.copy(data: data)
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
func stopVoiceInput() throws {
|
|
165
|
+
Task { @MainActor in
|
|
166
|
+
let interfaceController = try? await RootModule.withInterfaceController { $0 }
|
|
167
|
+
HybridAutoPlay.voiceInputManager?.stop(interfaceController: interfaceController)
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
126
171
|
// MARK: set/push/pop templates
|
|
127
172
|
func setRootTemplate(templateId: String) throws -> Promise<Void> {
|
|
128
173
|
return Promise.async {
|