@iternio/react-native-auto-play 0.5.3 → 0.5.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -16
- package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/AutoPlayError.kt +6 -1
- package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridVoice.kt +9 -3
- package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/VoiceInputManager.kt +103 -45
- package/ios/Types.swift +2 -1
- package/ios/extensions/CarPlayTemplateExtensions.swift +7 -0
- package/ios/hybrid/HybridAutoPlay.swift +0 -1
- package/ios/hybrid/HybridVoice.swift +10 -2
- package/ios/templates/Parser.swift +95 -0
- package/ios/templates/VoiceInputTemplate.swift +33 -0
- package/ios/utils/VoiceInputManager.swift +224 -29
- package/lib/hybrid/HybridVoice.js +7 -2
- package/lib/specs/Voice.nitro.d.ts +2 -1
- package/lib/templates/Template.d.ts +1 -1
- package/lib/types/Image.d.ts +16 -0
- package/lib/types/Voice.d.ts +9 -0
- package/lib/utils/ErrorUtil.d.ts +1 -0
- package/lib/utils/ErrorUtil.js +15 -3
- package/nitrogen/generated/android/c++/JHybridVoiceSpec.cpp +21 -3
- package/nitrogen/generated/android/c++/JHybridVoiceSpec.hpp +1 -1
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridVoiceSpec.kt +3 -3
- package/nitrogen/generated/ios/c++/HybridVoiceSpecSwift.hpp +15 -2
- package/nitrogen/generated/ios/swift/HybridVoiceSpec.swift +1 -1
- package/nitrogen/generated/ios/swift/HybridVoiceSpec_cxx.swift +44 -1
- package/nitrogen/generated/shared/c++/HybridVoiceSpec.hpp +11 -1
- package/package.json +1 -1
- package/src/hybrid/HybridVoice.ts +16 -1
- package/src/specs/Voice.nitro.ts +6 -1
- package/src/templates/Template.ts +1 -1
- package/src/types/Image.ts +19 -0
- package/src/types/Voice.ts +10 -0
- package/src/utils/ErrorUtil.ts +19 -3
package/README.md
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
## Features
|
|
15
15
|
|
|
16
16
|
- **Cross-Platform:** Write once, run on both Apple CarPlay and Android Auto.
|
|
17
|
-
-
|
|
17
|
+
- **New Architecture:** Supports React Native new architecture only.
|
|
18
18
|
- **Template-Based UI:** Utilize a rich set of templates like `MapTemplate`, `ListTemplate`, `GridTemplate`, and more to build UIs that comply with automotive design guidelines.
|
|
19
19
|
- **Navigation APIs:** Build full-featured navigation experiences with APIs for trip management, maneuvers, and route guidance.
|
|
20
20
|
- **Dashboard & Cluster Support:** Extend your app's presence to the CarPlay Dashboard (CarPlay only) and instrument cluster displays (CarPlay & Android Auto).
|
|
@@ -837,36 +837,95 @@ new ListTemplate({
|
|
|
837
837
|
|
|
838
838
|
### Voice Input
|
|
839
839
|
|
|
840
|
-
The library provides a cross-platform in-app voice recording API built on top of the car microphone (when connected) or the device microphone (when no car is connected).
|
|
840
|
+
The library provides a cross-platform in-app voice recording API built on top of the car microphone (when connected) or the device microphone (when no car is connected). The voice API lives in `HybridVoice`.
|
|
841
841
|
|
|
842
842
|
#### Permission
|
|
843
843
|
|
|
844
844
|
```ts
|
|
845
|
-
|
|
846
|
-
|
|
845
|
+
import { HybridVoice } from '@iternio/react-native-auto-play';
|
|
846
|
+
|
|
847
|
+
// Check whether permission is already granted (synchronous)
|
|
848
|
+
const granted = HybridVoice.hasVoiceInputPermission();
|
|
847
849
|
|
|
848
850
|
// Request permission if not yet granted
|
|
849
|
-
const granted = await
|
|
851
|
+
const granted = await HybridVoice.requestVoiceInputPermission();
|
|
850
852
|
```
|
|
851
853
|
|
|
854
|
+
On **iOS**: checks/requests both microphone and speech recognition authorization.
|
|
855
|
+
On **Android**: checks/requests `RECORD_AUDIO` via the car context when connected, otherwise via the RN application context.
|
|
856
|
+
|
|
852
857
|
#### Recording
|
|
853
858
|
|
|
854
859
|
```ts
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
860
|
+
import { HybridVoice, ErrorUtil } from '@iternio/react-native-auto-play';
|
|
861
|
+
|
|
862
|
+
try {
|
|
863
|
+
const result = await HybridVoice.startVoiceInput({
|
|
864
|
+
silenceThresholdMs: 1500, // ms of silence before auto-stop (default 1500)
|
|
865
|
+
maxDurationMs: 10_000, // hard cap on recording duration (default 10 000)
|
|
866
|
+
listeningText: 'Listening…', // iOS CarPlay: text shown on CPVoiceControlTemplate
|
|
867
|
+
preferSpeechToText: false, // true → STT transcription; false → raw PCM (default)
|
|
868
|
+
startSound: require('./beep_start.mp3'), // played just before recording starts
|
|
869
|
+
endSound: require('./beep_end.mp3'), // played just after recording stops
|
|
870
|
+
onChunk: (chunk) => {
|
|
871
|
+
// chunk.audio — raw PCM ArrayBuffer chunk (PCM mode)
|
|
872
|
+
// chunk.partial — partial transcription string (STT mode)
|
|
873
|
+
},
|
|
874
|
+
});
|
|
875
|
+
|
|
876
|
+
if (result.transcription) {
|
|
877
|
+
console.log('Transcription:', result.transcription);
|
|
878
|
+
} else if (result.audio) {
|
|
879
|
+
console.log(`PCM audio: ${result.audio.byteLength} bytes`);
|
|
880
|
+
}
|
|
881
|
+
} catch (e) {
|
|
882
|
+
if (ErrorUtil.isVoiceInputCanceledError(e)) {
|
|
883
|
+
// User pressed the cancel button on the car screen
|
|
884
|
+
console.log('Voice input cancelled');
|
|
885
|
+
} else {
|
|
886
|
+
console.error(e);
|
|
887
|
+
}
|
|
888
|
+
}
|
|
862
889
|
|
|
863
|
-
// Stop recording early — resolves startVoiceInput with
|
|
864
|
-
|
|
890
|
+
// Stop recording early — resolves startVoiceInput with audio captured so far
|
|
891
|
+
HybridVoice.stopVoiceInput();
|
|
865
892
|
```
|
|
866
893
|
|
|
867
|
-
|
|
894
|
+
| Option | Type | Default | Description |
|
|
895
|
+
|---|---|---|---|
|
|
896
|
+
| `silenceThresholdMs` | `number` | `1500` | Auto-stop after this many ms of silence |
|
|
897
|
+
| `maxDurationMs` | `number` | `10000` | Hard recording time limit |
|
|
898
|
+
| `listeningText` | `string` | — | iOS only — text shown on `CPVoiceControlTemplate` |
|
|
899
|
+
| `listeningImage` | `VoiceInputImage` | — | iOS only — animated image in the CarPlay overlay |
|
|
900
|
+
| `preferSpeechToText` | `boolean` | `false` | `true` → resolve with `{ transcription }`; `false` → resolve with `{ audio }` |
|
|
901
|
+
| `startSound` | `number` | — | Metro asset (`require('./beep.mp3')`) played before recording. Takes audio focus so other apps pause. |
|
|
902
|
+
| `endSound` | `number` | — | Metro asset played after recording stops |
|
|
903
|
+
| `onChunk` | `(chunk) => void` | — | Streaming callback: `chunk.audio` (PCM) or `chunk.partial` (STT) |
|
|
904
|
+
| `language` | `string` | system | BCP-47 language tag for the STT recognizer |
|
|
905
|
+
|
|
906
|
+
**PCM result** (`preferSpeechToText: false`, default): resolves with `{ audio: ArrayBuffer }` — raw 16 kHz, 16-bit, mono PCM.
|
|
907
|
+
|
|
908
|
+
**STT result** (`preferSpeechToText: true`): resolves with `{ transcription: string }` on success, or falls back to `{ audio }` if recognition is unavailable.
|
|
868
909
|
|
|
869
|
-
On **
|
|
910
|
+
On **Android**: uses `CarAudioRecord` when Android Auto is connected, otherwise falls back to standard `AudioRecord`. STT uses `SpeechRecognizer`.
|
|
911
|
+
|
|
912
|
+
On **iOS**: presents `CPVoiceControlTemplate` on the car screen when CarPlay is connected, and captures audio via `AVAudioEngine`. STT uses `SFSpeechRecognizer`.
|
|
913
|
+
|
|
914
|
+
#### Cancel detection
|
|
915
|
+
|
|
916
|
+
When the user presses the cancel button on the car screen, `startVoiceInput` rejects with a `voiceInputCancelled` error on both platforms. Use `ErrorUtil.isVoiceInputCanceledError` to distinguish it from other errors:
|
|
917
|
+
|
|
918
|
+
```ts
|
|
919
|
+
import { ErrorUtil } from '@iternio/react-native-auto-play';
|
|
920
|
+
|
|
921
|
+
HybridVoice.startVoiceInput().catch((e) => {
|
|
922
|
+
if (ErrorUtil.isVoiceInputCanceledError(e)) {
|
|
923
|
+
// user dismissed — no action needed
|
|
924
|
+
} else {
|
|
925
|
+
throw e;
|
|
926
|
+
}
|
|
927
|
+
});
|
|
928
|
+
```
|
|
870
929
|
|
|
871
930
|
#### OS-triggered voice input (Android only)
|
|
872
931
|
|
package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/AutoPlayError.kt
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
package com.margelo.nitro.swe.iternio.reactnativeautoplay
|
|
2
2
|
|
|
3
|
-
class TemplateNotFoundException(private val templateId: String):
|
|
3
|
+
class TemplateNotFoundException(private val templateId: String) :
|
|
4
|
+
Exception("templateNotFound(\"$templateId\")") {
|
|
4
5
|
override fun toString(): String = "templateNotFound(\"$templateId\")"
|
|
5
6
|
}
|
|
7
|
+
|
|
8
|
+
class VoiceInputCancelledException : Exception("voiceInputCancelled") {
|
|
9
|
+
override fun toString(): String = "voiceInputCancelled"
|
|
10
|
+
}
|
package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridVoice.kt
CHANGED
|
@@ -50,7 +50,7 @@ class HybridVoice : HybridVoiceSpec() {
|
|
|
50
50
|
}
|
|
51
51
|
cont.resume(
|
|
52
52
|
grantResults.isNotEmpty() &&
|
|
53
|
-
|
|
53
|
+
grantResults.first() == PackageManager.PERMISSION_GRANTED
|
|
54
54
|
)
|
|
55
55
|
true
|
|
56
56
|
}
|
|
@@ -64,9 +64,13 @@ class HybridVoice : HybridVoiceSpec() {
|
|
|
64
64
|
silenceThresholdMs: Double?,
|
|
65
65
|
maxDurationMs: Double?,
|
|
66
66
|
listeningText: String?,
|
|
67
|
+
listeningImage: Variant_GlyphImage_AssetImage_RemoteImage?,
|
|
68
|
+
listeningImageRepeats: Boolean?,
|
|
67
69
|
preferSpeechToText: Boolean?,
|
|
68
70
|
onChunk: ((chunk: VoiceInputChunk) -> Unit)?,
|
|
69
|
-
language: String
|
|
71
|
+
language: String?,
|
|
72
|
+
startSoundUri: String?,
|
|
73
|
+
endSoundUri: String?
|
|
70
74
|
): Promise<VoiceInputResult> {
|
|
71
75
|
return Promise.async {
|
|
72
76
|
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) {
|
|
@@ -82,7 +86,9 @@ class HybridVoice : HybridVoiceSpec() {
|
|
|
82
86
|
maxDurationMs = maxDurationMs?.toLong() ?: 10_000L,
|
|
83
87
|
preferSpeechToText = preferSpeechToText ?: false,
|
|
84
88
|
onChunk = onChunk,
|
|
85
|
-
language = language
|
|
89
|
+
language = language,
|
|
90
|
+
startSoundUri = startSoundUri,
|
|
91
|
+
endSoundUri = endSoundUri,
|
|
86
92
|
)
|
|
87
93
|
} finally {
|
|
88
94
|
voiceInputManager = null
|
package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/VoiceInputManager.kt
CHANGED
|
@@ -10,7 +10,9 @@ import android.media.AudioFocusRequest
|
|
|
10
10
|
import android.media.AudioFormat
|
|
11
11
|
import android.media.AudioManager
|
|
12
12
|
import android.media.AudioRecord
|
|
13
|
+
import android.media.MediaPlayer
|
|
13
14
|
import android.media.MediaRecorder
|
|
15
|
+
import android.net.Uri
|
|
14
16
|
import android.os.Build
|
|
15
17
|
import android.os.Bundle
|
|
16
18
|
import android.os.ParcelFileDescriptor
|
|
@@ -62,6 +64,9 @@ class VoiceInputManager(
|
|
|
62
64
|
@Volatile
|
|
63
65
|
private var isRecording = false
|
|
64
66
|
|
|
67
|
+
@Volatile
|
|
68
|
+
private var cancelledByUser = false
|
|
69
|
+
|
|
65
70
|
// STT state — only set when SpeechRecognizer owns the mic
|
|
66
71
|
@Volatile
|
|
67
72
|
private var activeSpeechRecognizer: SpeechRecognizer? = null
|
|
@@ -72,22 +77,41 @@ class VoiceInputManager(
|
|
|
72
77
|
maxDurationMs: Long = 10_000,
|
|
73
78
|
preferSpeechToText: Boolean = false,
|
|
74
79
|
onChunk: ((chunk: VoiceInputChunk) -> Unit)? = null,
|
|
75
|
-
language: String? = null
|
|
80
|
+
language: String? = null,
|
|
81
|
+
startSoundUri: String? = null,
|
|
82
|
+
endSoundUri: String? = null,
|
|
76
83
|
): VoiceInputResult {
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
84
|
+
cancelledByUser = false
|
|
85
|
+
if (!requestAudioFocus()) {
|
|
86
|
+
throw IllegalStateException("Audio focus request denied")
|
|
87
|
+
}
|
|
88
|
+
try {
|
|
89
|
+
startSoundUri?.let { playSound(it) }
|
|
90
|
+
val result = if (preferSpeechToText) {
|
|
91
|
+
val context = NitroModules.applicationContext ?: throw IllegalArgumentException()
|
|
92
|
+
if (SpeechRecognizer.isRecognitionAvailable(context)) {
|
|
93
|
+
if (carContext != null) {
|
|
94
|
+
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
|
|
95
|
+
startSTTFromCarAudio(silenceThresholdMs, maxDurationMs, onChunk, language)
|
|
96
|
+
} else {
|
|
97
|
+
// Car connected but API < 33: EXTRA_AUDIO_SOURCE unavailable, fall back to PCM
|
|
98
|
+
startPCM(silenceThresholdMs, maxDurationMs, onChunk)
|
|
99
|
+
}
|
|
100
|
+
} else {
|
|
101
|
+
ThreadUtil.postOnUiAndAwait { startSTT(context, onChunk, language) }.getOrThrow()
|
|
83
102
|
}
|
|
84
|
-
|
|
85
|
-
|
|
103
|
+
} else {
|
|
104
|
+
startPCM(silenceThresholdMs, maxDurationMs, onChunk)
|
|
86
105
|
}
|
|
87
|
-
|
|
106
|
+
} else {
|
|
107
|
+
startPCM(silenceThresholdMs, maxDurationMs, onChunk)
|
|
88
108
|
}
|
|
109
|
+
if (cancelledByUser) throw VoiceInputCancelledException()
|
|
110
|
+
endSoundUri?.let { playSound(it) }
|
|
111
|
+
return result
|
|
112
|
+
} finally {
|
|
113
|
+
abandonAudioFocus()
|
|
89
114
|
}
|
|
90
|
-
return startPCM(silenceThresholdMs, maxDurationMs, onChunk)
|
|
91
115
|
}
|
|
92
116
|
|
|
93
117
|
// MARK: - STT path (SpeechRecognizer owns the mic)
|
|
@@ -309,35 +333,8 @@ class VoiceInputManager(
|
|
|
309
333
|
return@suspendCancellableCoroutine
|
|
310
334
|
}
|
|
311
335
|
|
|
312
|
-
val appContext = NitroModules.applicationContext ?: run {
|
|
313
|
-
cont.resumeWithException(SecurityException("Missing application context"))
|
|
314
|
-
return@suspendCancellableCoroutine
|
|
315
|
-
}
|
|
316
|
-
|
|
317
336
|
pcmContinuation = cont
|
|
318
337
|
|
|
319
|
-
val audioManager = appContext.getSystemService(AudioManager::class.java)
|
|
320
|
-
|
|
321
|
-
val audioAttributes =
|
|
322
|
-
AudioAttributes.Builder().setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
|
|
323
|
-
.setUsage(AudioAttributes.USAGE_ASSISTANCE_NAVIGATION_GUIDANCE).build()
|
|
324
|
-
|
|
325
|
-
val focusRequest =
|
|
326
|
-
AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE)
|
|
327
|
-
.setAudioAttributes(audioAttributes).setOnAudioFocusChangeListener { state ->
|
|
328
|
-
if (state == AudioManager.AUDIOFOCUS_LOSS) {
|
|
329
|
-
stop()
|
|
330
|
-
}
|
|
331
|
-
}.build()
|
|
332
|
-
|
|
333
|
-
if (audioManager.requestAudioFocus(focusRequest) != AudioManager.AUDIOFOCUS_REQUEST_GRANTED) {
|
|
334
|
-
pcmContinuation = null
|
|
335
|
-
cont.resumeWithException(IllegalStateException("Audio focus request denied"))
|
|
336
|
-
return@suspendCancellableCoroutine
|
|
337
|
-
}
|
|
338
|
-
|
|
339
|
-
audioFocusRequest = focusRequest
|
|
340
|
-
|
|
341
338
|
val bufferSize: Int
|
|
342
339
|
|
|
343
340
|
if (carContext != null) {
|
|
@@ -381,6 +378,8 @@ class VoiceInputManager(
|
|
|
381
378
|
) ?: -1
|
|
382
379
|
|
|
383
380
|
if (read < 0) {
|
|
381
|
+
// Whenever the user dismisses the microphone on the car screen, the next call to read will return -1
|
|
382
|
+
cancelledByUser = carAudioRecord != null && read == -1
|
|
384
383
|
break
|
|
385
384
|
}
|
|
386
385
|
|
|
@@ -450,6 +449,72 @@ class VoiceInputManager(
|
|
|
450
449
|
audioRecord?.stop()
|
|
451
450
|
}
|
|
452
451
|
|
|
452
|
+
@RequiresApi(Build.VERSION_CODES.O)
|
|
453
|
+
private fun requestAudioFocus(): Boolean {
|
|
454
|
+
val appContext = NitroModules.applicationContext ?: return false
|
|
455
|
+
val audioManager = appContext.getSystemService(AudioManager::class.java)
|
|
456
|
+
val audioAttributes = AudioAttributes.Builder()
|
|
457
|
+
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
|
|
458
|
+
.setUsage(AudioAttributes.USAGE_ASSISTANCE_NAVIGATION_GUIDANCE)
|
|
459
|
+
.build()
|
|
460
|
+
val focusRequest = AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE)
|
|
461
|
+
.setAudioAttributes(audioAttributes)
|
|
462
|
+
.setOnAudioFocusChangeListener { state ->
|
|
463
|
+
if (state == AudioManager.AUDIOFOCUS_LOSS) { stop() }
|
|
464
|
+
}
|
|
465
|
+
.build()
|
|
466
|
+
return if (audioManager.requestAudioFocus(focusRequest) == AudioManager.AUDIOFOCUS_REQUEST_GRANTED) {
|
|
467
|
+
audioFocusRequest = focusRequest
|
|
468
|
+
true
|
|
469
|
+
} else {
|
|
470
|
+
false
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
@RequiresApi(Build.VERSION_CODES.O)
|
|
475
|
+
private fun abandonAudioFocus() {
|
|
476
|
+
audioFocusRequest?.let {
|
|
477
|
+
val audioManager = (NitroModules.applicationContext ?: carContext)
|
|
478
|
+
?.getSystemService(AudioManager::class.java)
|
|
479
|
+
audioManager?.abandonAudioFocusRequest(it)
|
|
480
|
+
}
|
|
481
|
+
audioFocusRequest = null
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
private suspend fun playSound(uri: String) = suspendCancellableCoroutine<Unit> { cont ->
|
|
485
|
+
val context = NitroModules.applicationContext ?: run {
|
|
486
|
+
cont.resume(Unit)
|
|
487
|
+
return@suspendCancellableCoroutine
|
|
488
|
+
}
|
|
489
|
+
val player = MediaPlayer()
|
|
490
|
+
try {
|
|
491
|
+
player.setAudioAttributes(
|
|
492
|
+
AudioAttributes.Builder()
|
|
493
|
+
.setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION)
|
|
494
|
+
.setUsage(AudioAttributes.USAGE_ASSISTANCE_NAVIGATION_GUIDANCE)
|
|
495
|
+
.build()
|
|
496
|
+
)
|
|
497
|
+
player.setDataSource(context, Uri.parse(uri))
|
|
498
|
+
player.setOnCompletionListener {
|
|
499
|
+
it.release()
|
|
500
|
+
if (cont.isActive) { cont.resume(Unit) }
|
|
501
|
+
}
|
|
502
|
+
player.setOnErrorListener { mp, _, _ ->
|
|
503
|
+
mp.release()
|
|
504
|
+
if (cont.isActive) { cont.resume(Unit) }
|
|
505
|
+
true
|
|
506
|
+
}
|
|
507
|
+
player.prepare()
|
|
508
|
+
player.start()
|
|
509
|
+
} catch (_: Exception) {
|
|
510
|
+
try { player.release() } catch (_: Exception) {}
|
|
511
|
+
if (cont.isActive) { cont.resume(Unit) }
|
|
512
|
+
}
|
|
513
|
+
cont.invokeOnCancellation {
|
|
514
|
+
try { player.release() } catch (_: Exception) {}
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
|
|
453
518
|
@RequiresApi(Build.VERSION_CODES.O)
|
|
454
519
|
private fun releaseResources() {
|
|
455
520
|
carAudioRecord?.stopRecording()
|
|
@@ -458,13 +523,6 @@ class VoiceInputManager(
|
|
|
458
523
|
audioRecord?.release()
|
|
459
524
|
audioRecord = null
|
|
460
525
|
recordingJob = null
|
|
461
|
-
audioFocusRequest?.let {
|
|
462
|
-
val audioManager = (NitroModules.applicationContext ?: carContext)?.getSystemService(
|
|
463
|
-
AudioManager::class.java,
|
|
464
|
-
)
|
|
465
|
-
audioManager?.abandonAudioFocusRequest(it)
|
|
466
|
-
}
|
|
467
|
-
audioFocusRequest = null
|
|
468
526
|
}
|
|
469
527
|
|
|
470
528
|
fun dispose() {
|
package/ios/Types.swift
CHANGED
|
@@ -10,7 +10,7 @@ struct TemplateEventPayload {
|
|
|
10
10
|
let state: VisibilityState
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
-
enum AutoPlayError:
|
|
13
|
+
enum AutoPlayError: LocalizedError {
|
|
14
14
|
case templateNotFound(String)
|
|
15
15
|
case interfaceControllerNotFound(String)
|
|
16
16
|
case invalidTemplateError(String)
|
|
@@ -19,4 +19,5 @@ enum AutoPlayError: Error {
|
|
|
19
19
|
case invalidTemplateType(String)
|
|
20
20
|
case noUiWindow(String)
|
|
21
21
|
case initReactRootViewFailed(String)
|
|
22
|
+
case voiceInputCancelled
|
|
22
23
|
}
|
|
@@ -85,3 +85,10 @@ extension CPAlertTemplate {
|
|
|
85
85
|
initTemplate(template: self, id: id)
|
|
86
86
|
}
|
|
87
87
|
}
|
|
88
|
+
|
|
89
|
+
extension CPVoiceControlTemplate {
|
|
90
|
+
convenience init(voiceControlStates: [CPVoiceControlState], id: String) {
|
|
91
|
+
self.init(voiceControlStates: voiceControlStates)
|
|
92
|
+
initTemplate(template: self, id: id)
|
|
93
|
+
}
|
|
94
|
+
}
|
|
@@ -22,7 +22,6 @@ class HybridAutoPlay: HybridAutoPlaySpec {
|
|
|
22
22
|
private static var renderStateListeners = [String: [RenderStateListener]]()
|
|
23
23
|
private static var safeAreaInsetsListeners = [String: [SafeAreaListener]]()
|
|
24
24
|
|
|
25
|
-
|
|
26
25
|
override init() {
|
|
27
26
|
HybridAutoPlay.listeners.removeAll()
|
|
28
27
|
HybridAutoPlay.renderStateListeners.removeAll()
|
|
@@ -32,9 +32,13 @@ class HybridVoice: HybridVoiceSpec {
|
|
|
32
32
|
silenceThresholdMs: Double?,
|
|
33
33
|
maxDurationMs: Double?,
|
|
34
34
|
listeningText: String?,
|
|
35
|
+
listeningImage: Variant_GlyphImage_AssetImage_RemoteImage?,
|
|
36
|
+
listeningImageRepeats: Bool?,
|
|
35
37
|
preferSpeechToText: Bool?,
|
|
36
38
|
onChunk: ((_ chunk: VoiceInputChunk) -> Void)?,
|
|
37
|
-
language: String
|
|
39
|
+
language: String?,
|
|
40
|
+
startSoundUri: String?,
|
|
41
|
+
endSoundUri: String?
|
|
38
42
|
) throws -> Promise<VoiceInputResult> {
|
|
39
43
|
return Promise.async {
|
|
40
44
|
let interfaceController = try? await RootModule.withInterfaceController { $0 }
|
|
@@ -49,9 +53,13 @@ class HybridVoice: HybridVoiceSpec {
|
|
|
49
53
|
silenceThresholdMs: silenceThresholdMs ?? 1_500,
|
|
50
54
|
maxDurationMs: maxDurationMs ?? 10_000,
|
|
51
55
|
listeningText: listeningText ?? "Listening...",
|
|
56
|
+
listeningImage: listeningImage,
|
|
57
|
+
listeningImageRepeats: listeningImageRepeats,
|
|
52
58
|
preferSpeechToText: preferSpeechToText ?? false,
|
|
53
59
|
onChunk: onChunk,
|
|
54
|
-
language: language
|
|
60
|
+
language: language,
|
|
61
|
+
startSoundUri: startSoundUri,
|
|
62
|
+
endSoundUri: endSoundUri
|
|
55
63
|
)
|
|
56
64
|
}
|
|
57
65
|
}
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
//
|
|
7
7
|
|
|
8
8
|
import CarPlay
|
|
9
|
+
import ImageIO
|
|
9
10
|
import UIKit
|
|
10
11
|
|
|
11
12
|
struct HeaderActions {
|
|
@@ -939,6 +940,100 @@ class Parser {
|
|
|
939
940
|
)
|
|
940
941
|
}
|
|
941
942
|
|
|
943
|
+
// MARK: - Animated image decoding
|
|
944
|
+
|
|
945
|
+
/// Decodes raw image data via ImageIO, walking every frame so animated GIF/APNG/WebP all
|
|
946
|
+
/// animate. UIImage(data:) only ever decodes the first frame for any of these formats.
|
|
947
|
+
/// `maxDuration` caps the assembled cycle length; pass `.greatestFiniteMagnitude` to skip capping.
|
|
948
|
+
static func decodeImage(data: Data, scale: CGFloat, maxDuration: TimeInterval = .greatestFiniteMagnitude)
|
|
949
|
+
-> UIImage?
|
|
950
|
+
{
|
|
951
|
+
guard let source = CGImageSourceCreateWithData(data as CFData, nil) else { return nil }
|
|
952
|
+
let frameCount = CGImageSourceGetCount(source)
|
|
953
|
+
guard frameCount > 0 else { return nil }
|
|
954
|
+
|
|
955
|
+
guard frameCount > 1 else {
|
|
956
|
+
guard let cgImage = CGImageSourceCreateImageAtIndex(source, 0, nil) else { return nil }
|
|
957
|
+
return UIImage(cgImage: cgImage, scale: scale, orientation: .up)
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
var frames: [UIImage] = []
|
|
961
|
+
var totalDuration: TimeInterval = 0
|
|
962
|
+
for index in 0..<frameCount {
|
|
963
|
+
guard let cgImage = CGImageSourceCreateImageAtIndex(source, index, nil) else { continue }
|
|
964
|
+
totalDuration += frameDuration(source: source, index: index)
|
|
965
|
+
frames.append(UIImage(cgImage: cgImage, scale: scale, orientation: .up))
|
|
966
|
+
}
|
|
967
|
+
guard !frames.isEmpty else { return nil }
|
|
968
|
+
return UIImage.animatedImage(with: frames, duration: min(totalDuration, maxDuration))
|
|
969
|
+
}
|
|
970
|
+
|
|
971
|
+
/// Reads the per-frame delay from whichever format dictionary ImageIO populated
|
|
972
|
+
/// (GIF, APNG, or WebP), falling back to a sane default if none is present.
|
|
973
|
+
private static func frameDuration(source: CGImageSource, index: Int) -> TimeInterval {
|
|
974
|
+
let defaultDuration: TimeInterval = 0.1
|
|
975
|
+
guard
|
|
976
|
+
let properties = CGImageSourceCopyPropertiesAtIndex(source, index, nil) as? [CFString: Any]
|
|
977
|
+
else { return defaultDuration }
|
|
978
|
+
|
|
979
|
+
if let gif = properties[kCGImagePropertyGIFDictionary] as? [CFString: Any] {
|
|
980
|
+
if let unclamped = gif[kCGImagePropertyGIFUnclampedDelayTime] as? Double, unclamped > 0 {
|
|
981
|
+
return unclamped
|
|
982
|
+
}
|
|
983
|
+
if let delay = gif[kCGImagePropertyGIFDelayTime] as? Double, delay > 0 {
|
|
984
|
+
return delay
|
|
985
|
+
}
|
|
986
|
+
}
|
|
987
|
+
|
|
988
|
+
if let png = properties[kCGImagePropertyPNGDictionary] as? [CFString: Any] {
|
|
989
|
+
if let unclamped = png[kCGImagePropertyAPNGUnclampedDelayTime] as? Double, unclamped > 0 {
|
|
990
|
+
return unclamped
|
|
991
|
+
}
|
|
992
|
+
if let delay = png[kCGImagePropertyAPNGDelayTime] as? Double, delay > 0 {
|
|
993
|
+
return delay
|
|
994
|
+
}
|
|
995
|
+
}
|
|
996
|
+
|
|
997
|
+
if let webp = properties[kCGImagePropertyWebPDictionary] as? [CFString: Any],
|
|
998
|
+
let delay = webp[kCGImagePropertyWebPDelayTime] as? Double, delay > 0
|
|
999
|
+
{
|
|
1000
|
+
return delay
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
return defaultDuration
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
private static func targetSize(for size: CGSize, max maxSize: CGSize) -> CGSize {
|
|
1007
|
+
guard size.width > maxSize.width || size.height > maxSize.height else { return size }
|
|
1008
|
+
let scale = min(maxSize.width / size.width, maxSize.height / size.height)
|
|
1009
|
+
return CGSize(width: (size.width * scale).rounded(), height: (size.height * scale).rounded())
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
static func resize(_ image: UIImage, max maxSize: CGSize) -> UIImage {
|
|
1013
|
+
let target = targetSize(for: image.size, max: maxSize)
|
|
1014
|
+
guard target != image.size else { return image }
|
|
1015
|
+
return UIGraphicsImageRenderer(size: target).image { _ in
|
|
1016
|
+
image.draw(in: CGRect(origin: .zero, size: target))
|
|
1017
|
+
}
|
|
1018
|
+
}
|
|
1019
|
+
|
|
1020
|
+
/// Resizes every frame of an animated UIImage while preserving the per-frame timing.
|
|
1021
|
+
/// UIImage.draw(in:) only renders the current frame, so resize() alone would collapse
|
|
1022
|
+
/// the animation to a still image.
|
|
1023
|
+
static func resizeAnimated(_ image: UIImage, max maxSize: CGSize) -> UIImage {
|
|
1024
|
+
guard let frames = image.images, !frames.isEmpty else {
|
|
1025
|
+
return resize(image, max: maxSize)
|
|
1026
|
+
}
|
|
1027
|
+
let target = targetSize(for: image.size, max: maxSize)
|
|
1028
|
+
guard target != image.size else { return image }
|
|
1029
|
+
let resizedFrames = frames.map { frame in
|
|
1030
|
+
UIGraphicsImageRenderer(size: target).image { _ in
|
|
1031
|
+
frame.draw(in: CGRect(origin: .zero, size: target))
|
|
1032
|
+
}
|
|
1033
|
+
}
|
|
1034
|
+
return UIImage.animatedImage(with: resizedFrames, duration: image.duration) ?? image
|
|
1035
|
+
}
|
|
1036
|
+
|
|
942
1037
|
static func imageFromLanes(
|
|
943
1038
|
laneImages: Array<NitroImage>.SubSequence,
|
|
944
1039
|
traitCollection: UITraitCollection
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import CarPlay
|
|
2
|
+
|
|
3
|
+
class VoiceInputTemplate: AutoPlayTemplate {
|
|
4
|
+
let template: CPVoiceControlTemplate
|
|
5
|
+
private let onDidDisappearCallback: () -> Void
|
|
6
|
+
|
|
7
|
+
override func getTemplate() -> CPTemplate {
|
|
8
|
+
return template
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
init(
|
|
12
|
+
voiceControlStates: [CPVoiceControlState],
|
|
13
|
+
id: String,
|
|
14
|
+
onDidDisappear: @escaping () -> Void
|
|
15
|
+
) {
|
|
16
|
+
self.template = CPVoiceControlTemplate(voiceControlStates: voiceControlStates, id: id)
|
|
17
|
+
self.onDidDisappearCallback = onDidDisappear
|
|
18
|
+
|
|
19
|
+
super.init()
|
|
20
|
+
|
|
21
|
+
try? RootModule.withTemplateStore { templateStore in
|
|
22
|
+
templateStore.addTemplate(template: self, templateId: id)
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
override func onDidDisappear(animated: Bool) {
|
|
27
|
+
onDidDisappearCallback()
|
|
28
|
+
|
|
29
|
+
try? RootModule.withTemplateStore { templateStore in
|
|
30
|
+
templateStore.removeTemplate(templateId: self.template.id)
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|