@iternio/react-native-auto-play 0.4.4 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridAutoPlay.kt +0 -89
- package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridVoice.kt +95 -0
- package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/VoiceInputManager.kt +276 -20
- package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/utils/ThreadUtil.kt +6 -13
- package/ios/hybrid/HybridAutoPlay.swift +2 -47
- package/ios/hybrid/HybridVoice.swift +63 -0
- package/ios/utils/VoiceInputManager.swift +141 -40
- package/lib/HybridAutoPlay.d.ts +2 -0
- package/lib/HybridAutoPlay.js +2 -0
- package/lib/hooks/useIsAutoPlayFocused.d.ts +7 -0
- package/lib/hooks/useIsAutoPlayFocused.js +20 -0
- package/lib/hybrid/HybridVoice.d.ts +12 -0
- package/lib/hybrid/HybridVoice.js +13 -0
- package/lib/hybrid.d.ts +2 -0
- package/lib/hybrid.js +2 -0
- package/lib/index.d.ts +3 -1
- package/lib/index.js +2 -1
- package/lib/specs/AutoPlay.nitro.d.ts +0 -29
- package/lib/specs/AutomotivePermissionRequestTemplate.d.ts +11 -0
- package/lib/specs/AutomotivePermissionRequestTemplate.js +1 -0
- package/lib/specs/AutomotivePermissionRequestTemplate.nitro.d.ts +11 -0
- package/lib/specs/AutomotivePermissionRequestTemplate.nitro.js +1 -0
- package/lib/specs/Voice.nitro.d.ts +51 -0
- package/lib/specs/Voice.nitro.js +1 -0
- package/lib/templates/AutomotivePermissionRequestTemplate.d.ts +23 -0
- package/lib/templates/AutomotivePermissionRequestTemplate.js +18 -0
- package/lib/types/Glyphmap.d.ts +4105 -0
- package/lib/types/Glyphmap.js +4105 -0
- package/lib/types/Voice.d.ts +15 -0
- package/lib/types/Voice.js +1 -0
- package/nitro.json +10 -0
- package/nitrogen/generated/android/ReactNativeAutoPlay+autolinking.cmake +2 -0
- package/nitrogen/generated/android/ReactNativeAutoPlayOnLoad.cpp +18 -0
- package/nitrogen/generated/android/c++/JFunc_void_VoiceInputChunk.hpp +81 -0
- package/nitrogen/generated/android/c++/JHybridAutoPlaySpec.cpp +0 -43
- package/nitrogen/generated/android/c++/JHybridAutoPlaySpec.hpp +0 -4
- package/nitrogen/generated/android/c++/JHybridVoiceSpec.cpp +104 -0
- package/nitrogen/generated/android/c++/JHybridVoiceSpec.hpp +66 -0
- package/nitrogen/generated/android/c++/JVoiceInputChunk.hpp +64 -0
- package/nitrogen/generated/android/c++/JVoiceInputResult.hpp +64 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/swe/iternio/reactnativeautoplay/Func_void_VoiceInputChunk.kt +80 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridAutoPlaySpec.kt +0 -17
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridVoiceSpec.kt +72 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/swe/iternio/reactnativeautoplay/VoiceInputChunk.kt +41 -0
- package/nitrogen/generated/android/kotlin/com/margelo/nitro/swe/iternio/reactnativeautoplay/VoiceInputResult.kt +41 -0
- package/nitrogen/generated/ios/ReactNativeAutoPlay-Swift-Cxx-Bridge.cpp +41 -16
- package/nitrogen/generated/ios/ReactNativeAutoPlay-Swift-Cxx-Bridge.hpp +201 -126
- package/nitrogen/generated/ios/ReactNativeAutoPlay-Swift-Cxx-Umbrella.hpp +11 -0
- package/nitrogen/generated/ios/ReactNativeAutoPlayAutolinking.mm +8 -0
- package/nitrogen/generated/ios/ReactNativeAutoPlayAutolinking.swift +12 -0
- package/nitrogen/generated/ios/c++/HybridAutoPlaySpecSwift.hpp +0 -34
- package/nitrogen/generated/ios/c++/HybridVoiceSpecSwift.cpp +11 -0
- package/nitrogen/generated/ios/c++/HybridVoiceSpecSwift.hpp +116 -0
- package/nitrogen/generated/ios/swift/Func_void_VoiceInputChunk.swift +46 -0
- package/nitrogen/generated/ios/swift/{Func_void_std__shared_ptr_ArrayBuffer_.swift → Func_void_VoiceInputResult.swift} +10 -10
- package/nitrogen/generated/ios/swift/Func_void_bool.swift +5 -5
- package/nitrogen/generated/ios/swift/HybridAutoPlaySpec.swift +0 -4
- package/nitrogen/generated/ios/swift/HybridAutoPlaySpec_cxx.swift +0 -82
- package/nitrogen/generated/ios/swift/HybridVoiceSpec.swift +58 -0
- package/nitrogen/generated/ios/swift/HybridVoiceSpec_cxx.swift +227 -0
- package/nitrogen/generated/ios/swift/VoiceInputChunk.swift +60 -0
- package/nitrogen/generated/ios/swift/VoiceInputResult.swift +60 -0
- package/nitrogen/generated/shared/c++/HybridAutoPlaySpec.cpp +0 -4
- package/nitrogen/generated/shared/c++/HybridAutoPlaySpec.hpp +0 -5
- package/nitrogen/generated/shared/c++/HybridVoiceSpec.cpp +24 -0
- package/nitrogen/generated/shared/c++/HybridVoiceSpec.hpp +73 -0
- package/nitrogen/generated/shared/c++/VoiceInputChunk.hpp +89 -0
- package/nitrogen/generated/shared/c++/VoiceInputResult.hpp +89 -0
- package/package.json +1 -1
- package/src/hybrid/HybridVoice.ts +30 -0
- package/src/index.ts +3 -1
- package/src/specs/AutoPlay.nitro.ts +0 -37
- package/src/specs/Voice.nitro.ts +58 -0
- package/src/types/Voice.ts +17 -0
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import AVFoundation
|
|
2
|
+
import NitroModules
|
|
3
|
+
import Speech
|
|
4
|
+
|
|
5
|
+
class HybridVoice: HybridVoiceSpec {
|
|
6
|
+
private var voiceInputManager: VoiceInputManager?
|
|
7
|
+
|
|
8
|
+
func hasVoiceInputPermission() throws -> Bool {
|
|
9
|
+
let micGranted = AVAudioSession.sharedInstance().recordPermission == .granted
|
|
10
|
+
let speechGranted = SFSpeechRecognizer.authorizationStatus() == .authorized
|
|
11
|
+
return micGranted && speechGranted
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
func requestVoiceInputPermission() throws -> Promise<Bool> {
|
|
15
|
+
return Promise.async {
|
|
16
|
+
let micGranted = await withCheckedContinuation { cont in
|
|
17
|
+
AVAudioSession.sharedInstance().requestRecordPermission { granted in
|
|
18
|
+
cont.resume(returning: granted)
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
guard micGranted else { return false }
|
|
22
|
+
|
|
23
|
+
return await withCheckedContinuation { cont in
|
|
24
|
+
SFSpeechRecognizer.requestAuthorization { status in
|
|
25
|
+
cont.resume(returning: status == .authorized)
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
func startVoiceInput(
|
|
32
|
+
silenceThresholdMs: Double?,
|
|
33
|
+
maxDurationMs: Double?,
|
|
34
|
+
listeningText: String?,
|
|
35
|
+
preferSpeechToText: Bool?,
|
|
36
|
+
onChunk: ((_ chunk: VoiceInputChunk) -> Void)?
|
|
37
|
+
) throws -> Promise<VoiceInputResult> {
|
|
38
|
+
return Promise.async {
|
|
39
|
+
let interfaceController = try? await RootModule.withInterfaceController { $0 }
|
|
40
|
+
|
|
41
|
+
let manager = VoiceInputManager()
|
|
42
|
+
self.voiceInputManager = manager
|
|
43
|
+
|
|
44
|
+
defer { self.voiceInputManager = nil }
|
|
45
|
+
|
|
46
|
+
return try await manager.start(
|
|
47
|
+
interfaceController: interfaceController,
|
|
48
|
+
silenceThresholdMs: silenceThresholdMs ?? 1_500,
|
|
49
|
+
maxDurationMs: maxDurationMs ?? 10_000,
|
|
50
|
+
listeningText: listeningText ?? "Listening...",
|
|
51
|
+
preferSpeechToText: preferSpeechToText ?? false,
|
|
52
|
+
onChunk: onChunk
|
|
53
|
+
)
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
func stopVoiceInput() throws {
|
|
58
|
+
Task { @MainActor in
|
|
59
|
+
let interfaceController = try? await RootModule.withInterfaceController { $0 }
|
|
60
|
+
self.voiceInputManager?.stop(interfaceController: interfaceController)
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
@@ -1,16 +1,47 @@
|
|
|
1
1
|
import AVFoundation
|
|
2
2
|
import CarPlay
|
|
3
|
+
import NitroModules
|
|
4
|
+
import Speech
|
|
3
5
|
|
|
4
|
-
///
|
|
5
|
-
///
|
|
6
|
+
/// Wraps CheckedContinuation so it can only be resumed once even when
|
|
7
|
+
/// shared between a stop() call and an async recognition task callback.
|
|
8
|
+
private final class ResultBox: @unchecked Sendable {
|
|
9
|
+
private var continuation: CheckedContinuation<VoiceInputResult, Error>?
|
|
10
|
+
private let lock = NSLock()
|
|
11
|
+
|
|
12
|
+
init(_ continuation: CheckedContinuation<VoiceInputResult, Error>) {
|
|
13
|
+
self.continuation = continuation
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
func resume(returning result: VoiceInputResult) {
|
|
17
|
+
lock.lock()
|
|
18
|
+
defer { lock.unlock() }
|
|
19
|
+
continuation?.resume(returning: result)
|
|
20
|
+
continuation = nil
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
func resume(throwing error: Error) {
|
|
24
|
+
lock.lock()
|
|
25
|
+
defer { lock.unlock() }
|
|
26
|
+
continuation?.resume(throwing: error)
|
|
27
|
+
continuation = nil
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/// Captures audio from the car microphone and buffers raw 16 kHz / 16-bit / mono PCM,
|
|
32
|
+
/// or transcribes it via SFSpeechRecognizer when preferSpeechToText is true.
|
|
6
33
|
class VoiceInputManager {
|
|
7
34
|
private var audioEngine: AVAudioEngine?
|
|
8
35
|
private var voiceControlTemplate: CPVoiceControlTemplate?
|
|
9
|
-
private var
|
|
36
|
+
private var resultBox: ResultBox?
|
|
10
37
|
private var samples: [Int16] = []
|
|
11
38
|
private var isStopping = false
|
|
12
39
|
private let stopLock = NSLock()
|
|
13
40
|
|
|
41
|
+
// STT
|
|
42
|
+
private var recognitionRequest: SFSpeechAudioBufferRecognitionRequest?
|
|
43
|
+
private var isSTTMode = false
|
|
44
|
+
|
|
14
45
|
// Timing
|
|
15
46
|
private var recordingStart: Date?
|
|
16
47
|
private var silenceStart: Date?
|
|
@@ -33,30 +64,33 @@ class VoiceInputManager {
|
|
|
33
64
|
interfaceController: AutoPlayInterfaceController?,
|
|
34
65
|
silenceThresholdMs: Double,
|
|
35
66
|
maxDurationMs: Double,
|
|
36
|
-
listeningText: String
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
67
|
+
listeningText: String,
|
|
68
|
+
preferSpeechToText: Bool,
|
|
69
|
+
onChunk: ((_ chunk: VoiceInputChunk) -> Void)?
|
|
70
|
+
) async throws -> VoiceInputResult {
|
|
71
|
+
return try await withCheckedThrowingContinuation { cont in
|
|
72
|
+
let box = ResultBox(cont)
|
|
73
|
+
self.resultBox = box
|
|
41
74
|
self.samples = []
|
|
42
75
|
self.isStopping = false
|
|
76
|
+
self.isSTTMode = preferSpeechToText
|
|
43
77
|
|
|
44
78
|
do {
|
|
45
79
|
try self.startCapture(
|
|
46
80
|
interfaceController: interfaceController,
|
|
47
81
|
silenceThresholdMs: silenceThresholdMs,
|
|
48
82
|
maxDurationMs: maxDurationMs,
|
|
49
|
-
listeningText: listeningText
|
|
83
|
+
listeningText: listeningText,
|
|
84
|
+
preferSpeechToText: preferSpeechToText,
|
|
85
|
+
onChunk: onChunk,
|
|
86
|
+
box: box
|
|
50
87
|
)
|
|
51
88
|
}
|
|
52
89
|
catch {
|
|
53
|
-
self.
|
|
54
|
-
|
|
55
|
-
cont.resume(throwing: error)
|
|
90
|
+
self.cleanup(interfaceController: interfaceController)
|
|
91
|
+
box.resume(throwing: error)
|
|
56
92
|
}
|
|
57
93
|
}
|
|
58
|
-
|
|
59
|
-
return samplesAsData(samples)
|
|
60
94
|
}
|
|
61
95
|
|
|
62
96
|
func stop(interfaceController: AutoPlayInterfaceController? = nil) {
|
|
@@ -66,14 +100,22 @@ class VoiceInputManager {
|
|
|
66
100
|
return
|
|
67
101
|
}
|
|
68
102
|
isStopping = true
|
|
69
|
-
let
|
|
103
|
+
let wasSTTMode = isSTTMode
|
|
104
|
+
let box = resultBox
|
|
70
105
|
let capturedSamples = samples
|
|
71
|
-
|
|
106
|
+
resultBox = nil
|
|
72
107
|
samples = []
|
|
73
108
|
stopLock.unlock()
|
|
74
109
|
|
|
75
|
-
|
|
76
|
-
|
|
110
|
+
if wasSTTMode {
|
|
111
|
+
// endAudio() causes the recognition task to fire its final result,
|
|
112
|
+
// which resumes the box. Engine teardown happens there too.
|
|
113
|
+
recognitionRequest?.endAudio()
|
|
114
|
+
}
|
|
115
|
+
else {
|
|
116
|
+
cleanup(interfaceController: interfaceController)
|
|
117
|
+
box?.resume(returning: makePCMResult(from: capturedSamples))
|
|
118
|
+
}
|
|
77
119
|
}
|
|
78
120
|
|
|
79
121
|
// MARK: - Private
|
|
@@ -82,13 +124,15 @@ class VoiceInputManager {
|
|
|
82
124
|
interfaceController: AutoPlayInterfaceController?,
|
|
83
125
|
silenceThresholdMs: Double,
|
|
84
126
|
maxDurationMs: Double,
|
|
85
|
-
listeningText: String
|
|
127
|
+
listeningText: String,
|
|
128
|
+
preferSpeechToText: Bool,
|
|
129
|
+
onChunk: ((_ chunk: VoiceInputChunk) -> Void)?,
|
|
130
|
+
box: ResultBox
|
|
86
131
|
) throws {
|
|
87
132
|
guard AVAudioSession.sharedInstance().recordPermission == .granted else {
|
|
88
133
|
throw VoiceInputError.microphonePermissionDenied
|
|
89
134
|
}
|
|
90
135
|
|
|
91
|
-
// Activate the session first so inputNode reports the correct hardware format
|
|
92
136
|
let session = AVAudioSession.sharedInstance()
|
|
93
137
|
try session.setCategory(.playAndRecord, mode: .measurement, options: [])
|
|
94
138
|
try session.setActive(true)
|
|
@@ -97,12 +141,59 @@ class VoiceInputManager {
|
|
|
97
141
|
presentVoiceTemplate(interfaceController: interfaceController, listeningText: listeningText)
|
|
98
142
|
}
|
|
99
143
|
|
|
144
|
+
var activeRecognitionRequest: SFSpeechAudioBufferRecognitionRequest? = nil
|
|
145
|
+
|
|
146
|
+
if preferSpeechToText, SFSpeechRecognizer.authorizationStatus() == .authorized,
|
|
147
|
+
let recognizer = SFSpeechRecognizer(locale: Locale.current),
|
|
148
|
+
recognizer.isAvailable
|
|
149
|
+
{
|
|
150
|
+
let request = SFSpeechAudioBufferRecognitionRequest()
|
|
151
|
+
request.shouldReportPartialResults = true
|
|
152
|
+
recognitionRequest = request
|
|
153
|
+
activeRecognitionRequest = request
|
|
154
|
+
|
|
155
|
+
recognizer.recognitionTask(with: request) { [weak self] result, error in
|
|
156
|
+
guard let self else { return }
|
|
157
|
+
|
|
158
|
+
if error != nil {
|
|
159
|
+
// STT failed — fall back to whatever PCM was accumulated
|
|
160
|
+
self.stopLock.lock()
|
|
161
|
+
let capturedSamples = self.samples
|
|
162
|
+
self.samples = []
|
|
163
|
+
self.stopLock.unlock()
|
|
164
|
+
|
|
165
|
+
self.cleanup(interfaceController: interfaceController)
|
|
166
|
+
box.resume(returning: self.makePCMResult(from: capturedSamples))
|
|
167
|
+
return
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
guard let result else { return }
|
|
171
|
+
|
|
172
|
+
if result.isFinal {
|
|
173
|
+
self.stopLock.lock()
|
|
174
|
+
self.isStopping = true
|
|
175
|
+
self.samples = []
|
|
176
|
+
self.stopLock.unlock()
|
|
177
|
+
|
|
178
|
+
self.cleanup(interfaceController: interfaceController)
|
|
179
|
+
box.resume(
|
|
180
|
+
returning: VoiceInputResult(
|
|
181
|
+
transcription: result.bestTranscription.formattedString,
|
|
182
|
+
audio: nil
|
|
183
|
+
)
|
|
184
|
+
)
|
|
185
|
+
}
|
|
186
|
+
else {
|
|
187
|
+
onChunk?(VoiceInputChunk(partial: result.bestTranscription.formattedString, audio: nil))
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
100
192
|
let engine = AVAudioEngine()
|
|
101
193
|
let inputNode = engine.inputNode
|
|
102
194
|
let nativeFormat = inputNode.outputFormat(forBus: 0)
|
|
103
195
|
|
|
104
|
-
let
|
|
105
|
-
guard let converter = AVAudioConverter(from: nativeFormat, to: targetFormat) else {
|
|
196
|
+
guard let converter = AVAudioConverter(from: nativeFormat, to: VoiceInputManager.targetFormat) else {
|
|
106
197
|
throw VoiceInputError.converterUnavailable
|
|
107
198
|
}
|
|
108
199
|
|
|
@@ -116,36 +207,43 @@ class VoiceInputManager {
|
|
|
116
207
|
) { [weak self] buffer, _ in
|
|
117
208
|
guard let self, !self.isStopping else { return }
|
|
118
209
|
|
|
210
|
+
// Feed STT if active
|
|
211
|
+
activeRecognitionRequest?.append(buffer)
|
|
212
|
+
|
|
213
|
+
// Convert to 16kHz int16 for accumulation and PCM chunks
|
|
119
214
|
let outputFrameCapacity = AVAudioFrameCount(
|
|
120
|
-
Double(buffer.frameLength)
|
|
121
|
-
* VoiceInputManager.sampleRate
|
|
122
|
-
/ nativeFormat.sampleRate
|
|
215
|
+
Double(buffer.frameLength) * VoiceInputManager.sampleRate / nativeFormat.sampleRate
|
|
123
216
|
)
|
|
124
|
-
|
|
125
217
|
guard
|
|
126
218
|
let outputBuffer = AVAudioPCMBuffer(
|
|
127
|
-
pcmFormat: targetFormat,
|
|
219
|
+
pcmFormat: VoiceInputManager.targetFormat,
|
|
128
220
|
frameCapacity: outputFrameCapacity
|
|
129
221
|
)
|
|
130
222
|
else { return }
|
|
131
223
|
|
|
132
224
|
var conversionError: NSError?
|
|
133
|
-
let status = converter.convert(to: outputBuffer, error: &conversionError) {
|
|
134
|
-
_,
|
|
135
|
-
outStatus in
|
|
225
|
+
let status = converter.convert(to: outputBuffer, error: &conversionError) { _, outStatus in
|
|
136
226
|
outStatus.pointee = .haveData
|
|
137
227
|
return buffer
|
|
138
228
|
}
|
|
139
|
-
|
|
140
229
|
guard status != .error, let int16Data = outputBuffer.int16ChannelData else { return }
|
|
141
230
|
|
|
142
231
|
let frameCount = Int(outputBuffer.frameLength)
|
|
143
232
|
let newSamples = Array(UnsafeBufferPointer(start: int16Data[0], count: frameCount))
|
|
144
233
|
self.samples.append(contentsOf: newSamples)
|
|
145
234
|
|
|
235
|
+
// PCM chunk callback
|
|
236
|
+
if activeRecognitionRequest == nil, let onChunk {
|
|
237
|
+
if let chunkBuffer = try? ArrayBuffer.copy(
|
|
238
|
+
data: newSamples.withUnsafeBufferPointer { Data(buffer: $0) }
|
|
239
|
+
) {
|
|
240
|
+
onChunk(VoiceInputChunk(partial: nil, audio: chunkBuffer))
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
146
244
|
let now = Date()
|
|
147
245
|
|
|
148
|
-
// Max duration
|
|
246
|
+
// Max duration — applies in both modes
|
|
149
247
|
if let start = self.recordingStart,
|
|
150
248
|
now.timeIntervalSince(start) * 1000 >= maxDurationMs
|
|
151
249
|
{
|
|
@@ -160,7 +258,9 @@ class VoiceInputManager {
|
|
|
160
258
|
{
|
|
161
259
|
let peak = newSamples.reduce(0) { max($0, abs(Int($1))) }
|
|
162
260
|
if peak < VoiceInputManager.silenceAmplitudeThreshold {
|
|
163
|
-
if self.silenceStart == nil {
|
|
261
|
+
if self.silenceStart == nil {
|
|
262
|
+
self.silenceStart = now
|
|
263
|
+
}
|
|
164
264
|
if let silenceBegin = self.silenceStart,
|
|
165
265
|
now.timeIntervalSince(silenceBegin) * 1000 >= silenceThresholdMs
|
|
166
266
|
{
|
|
@@ -183,10 +283,11 @@ class VoiceInputManager {
|
|
|
183
283
|
}
|
|
184
284
|
}
|
|
185
285
|
|
|
186
|
-
private func
|
|
286
|
+
private func cleanup(interfaceController: AutoPlayInterfaceController?) {
|
|
187
287
|
audioEngine?.inputNode.removeTap(onBus: 0)
|
|
188
288
|
audioEngine?.stop()
|
|
189
289
|
audioEngine = nil
|
|
290
|
+
recognitionRequest = nil
|
|
190
291
|
recordingStart = nil
|
|
191
292
|
silenceStart = nil
|
|
192
293
|
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
|
|
@@ -195,6 +296,12 @@ class VoiceInputManager {
|
|
|
195
296
|
}
|
|
196
297
|
}
|
|
197
298
|
|
|
299
|
+
private func makePCMResult(from samples: [Int16]) -> VoiceInputResult {
|
|
300
|
+
let data = samples.withUnsafeBufferPointer { Data(buffer: $0) }
|
|
301
|
+
let buffer = try? ArrayBuffer.copy(data: data)
|
|
302
|
+
return VoiceInputResult(transcription: nil, audio: buffer)
|
|
303
|
+
}
|
|
304
|
+
|
|
198
305
|
private func presentVoiceTemplate(interfaceController: AutoPlayInterfaceController, listeningText: String) {
|
|
199
306
|
let listeningState = CPVoiceControlState(
|
|
200
307
|
identifier: "listening",
|
|
@@ -218,12 +325,6 @@ class VoiceInputManager {
|
|
|
218
325
|
}
|
|
219
326
|
voiceControlTemplate = nil
|
|
220
327
|
}
|
|
221
|
-
|
|
222
|
-
private func samplesAsData(_ samples: [Int16]) -> Data {
|
|
223
|
-
samples.withUnsafeBufferPointer { ptr in
|
|
224
|
-
Data(buffer: ptr)
|
|
225
|
-
}
|
|
226
|
-
}
|
|
227
328
|
}
|
|
228
329
|
|
|
229
330
|
enum VoiceInputError: Error {
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A hook to determine if the CarPlay/Android Auto screen is currently focused (visible).
|
|
3
|
+
*
|
|
4
|
+
* @param moduleName The name of the module to listen to.
|
|
5
|
+
* @returns `true` if the screen is focused, `false` otherwise.
|
|
6
|
+
*/
|
|
7
|
+
export declare function useIsAutoPlayFocused(moduleName: string): boolean;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { useEffect, useState } from 'react';
|
|
2
|
+
import { HybridAutoPlay } from '..';
|
|
3
|
+
/**
|
|
4
|
+
* A hook to determine if the CarPlay/Android Auto screen is currently focused (visible).
|
|
5
|
+
*
|
|
6
|
+
* @param moduleName The name of the module to listen to.
|
|
7
|
+
* @returns `true` if the screen is focused, `false` otherwise.
|
|
8
|
+
*/
|
|
9
|
+
export function useIsAutoPlayFocused(moduleName) {
|
|
10
|
+
const [isFocused, setIsFocused] = useState(false);
|
|
11
|
+
useEffect(() => {
|
|
12
|
+
const remove = HybridAutoPlay.addListenerRenderState(moduleName, (state) => {
|
|
13
|
+
setIsFocused(state === 'didAppear');
|
|
14
|
+
});
|
|
15
|
+
return () => {
|
|
16
|
+
remove();
|
|
17
|
+
};
|
|
18
|
+
}, [moduleName]);
|
|
19
|
+
return isFocused;
|
|
20
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { VoiceInputOptions, VoiceInputResult } from '../types/Voice';
|
|
2
|
+
type StartVoiceInput = {
|
|
3
|
+
(options: VoiceInputOptions & Required<Pick<VoiceInputOptions, 'onChunk'>>): Promise<void>;
|
|
4
|
+
(options?: Omit<VoiceInputOptions, 'onChunk'>): Promise<VoiceInputResult>;
|
|
5
|
+
};
|
|
6
|
+
export declare const HybridVoice: {
|
|
7
|
+
hasVoiceInputPermission: () => boolean;
|
|
8
|
+
requestVoiceInputPermission: () => Promise<boolean>;
|
|
9
|
+
startVoiceInput: StartVoiceInput;
|
|
10
|
+
stopVoiceInput: () => void;
|
|
11
|
+
};
|
|
12
|
+
export {};
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { NitroModules } from 'react-native-nitro-modules';
|
|
2
|
+
const _native = NitroModules.createHybridObject('Voice');
|
|
3
|
+
const startVoiceInput = (async (options) => {
|
|
4
|
+
const { onChunk, silenceThresholdMs, maxDurationMs, listeningText, preferSpeechToText } = options ?? {};
|
|
5
|
+
const result = await _native.startVoiceInput(silenceThresholdMs, maxDurationMs, listeningText, preferSpeechToText, onChunk);
|
|
6
|
+
return onChunk !== undefined ? undefined : result;
|
|
7
|
+
});
|
|
8
|
+
export const HybridVoice = {
|
|
9
|
+
hasVoiceInputPermission: () => _native.hasVoiceInputPermission(),
|
|
10
|
+
requestVoiceInputPermission: () => _native.requestVoiceInputPermission(),
|
|
11
|
+
startVoiceInput,
|
|
12
|
+
stopVoiceInput: () => _native.stopVoiceInput(),
|
|
13
|
+
};
|
package/lib/hybrid.d.ts
ADDED
package/lib/hybrid.js
ADDED
package/lib/index.d.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { HybridAndroidAutoTelemetry } from './hybrid/HybridAndroidAutoTelemetry';
|
|
2
2
|
import { HybridAutoPlay } from './hybrid/HybridAutoPlay';
|
|
3
|
+
import { HybridVoice } from './hybrid/HybridVoice';
|
|
3
4
|
import type { AndroidAutomotive } from './specs/AndroidAutomotive.nitro';
|
|
4
|
-
export { HybridAndroidAutoTelemetry, HybridAutoPlay };
|
|
5
|
+
export { HybridAndroidAutoTelemetry, HybridAutoPlay, HybridVoice };
|
|
5
6
|
export declare const HybridAndroidAutomotive: AndroidAutomotive | null;
|
|
6
7
|
/**
|
|
7
8
|
* These are the static module names for the app running on the mobile device, head unit screen and the CarPlay dashboard.
|
|
@@ -39,6 +40,7 @@ export * from './types/SignInMethod';
|
|
|
39
40
|
export * from './types/Telemetry';
|
|
40
41
|
export * from './types/Text';
|
|
41
42
|
export * from './types/Trip';
|
|
43
|
+
export type { VoiceInputChunk, VoiceInputOptions, VoiceInputResult } from './types/Voice';
|
|
42
44
|
export type { AlertPriority, NavigationAlert as Alert, NavigationAlertAction as AlertAction, } from './utils/NitroAlert';
|
|
43
45
|
export type { ThemedColor } from './utils/NitroColor';
|
|
44
46
|
export type { GridButton } from './utils/NitroGrid';
|
package/lib/index.js
CHANGED
|
@@ -3,8 +3,9 @@ import { NitroModules } from 'react-native-nitro-modules';
|
|
|
3
3
|
import AutoPlayHeadlessJsTask from './AutoPlayHeadlessJsTask';
|
|
4
4
|
import { HybridAndroidAutoTelemetry } from './hybrid/HybridAndroidAutoTelemetry';
|
|
5
5
|
import { HybridAutoPlay } from './hybrid/HybridAutoPlay';
|
|
6
|
+
import { HybridVoice } from './hybrid/HybridVoice';
|
|
6
7
|
AutoPlayHeadlessJsTask.registerHeadlessTask(HybridAutoPlay);
|
|
7
|
-
export { HybridAndroidAutoTelemetry, HybridAutoPlay };
|
|
8
|
+
export { HybridAndroidAutoTelemetry, HybridAutoPlay, HybridVoice };
|
|
8
9
|
export const HybridAndroidAutomotive = Platform.OS === 'android'
|
|
9
10
|
? NitroModules.createHybridObject('AndroidAutomotive')
|
|
10
11
|
: null;
|
|
@@ -31,35 +31,6 @@ export interface AutoPlay extends HybridObject<{
|
|
|
31
31
|
* @namespace Android
|
|
32
32
|
*/
|
|
33
33
|
addListenerVoiceInput(callback: (coordinates: Location | undefined, query: string | undefined) => void): CleanupCallback;
|
|
34
|
-
/**
|
|
35
|
-
* Returns true if microphone permission has already been granted.
|
|
36
|
-
*/
|
|
37
|
-
hasVoiceInputPermission(): boolean;
|
|
38
|
-
/**
|
|
39
|
-
* Request microphone permission from the user.
|
|
40
|
-
* On Android: uses the car context when Android Auto is connected, otherwise
|
|
41
|
-
* falls back to the React Native application context.
|
|
42
|
-
* On iOS: uses AVAudioApplication (iOS 17+) or AVAudioSession (iOS 15–16).
|
|
43
|
-
* Returns true if permission was granted, false if denied.
|
|
44
|
-
*/
|
|
45
|
-
requestVoiceInputPermission(): Promise<boolean>;
|
|
46
|
-
/**
|
|
47
|
-
* Start an in-app voice recording session.
|
|
48
|
-
* On Android: acquires audio focus and captures via CarAudioRecord when
|
|
49
|
-
* Android Auto is connected, otherwise uses standard AudioRecord.
|
|
50
|
-
* On iOS: presents CPVoiceControlTemplate (when a car is connected) and
|
|
51
|
-
* captures audio via AVAudioEngine.
|
|
52
|
-
* Resolves with the complete raw PCM buffer (16 kHz, 16-bit, mono) when
|
|
53
|
-
* silence is detected, the max duration is reached, or stopVoiceInput() is called.
|
|
54
|
-
* Rejects if microphone permission has not been granted or recording fails to start.
|
|
55
|
-
*/
|
|
56
|
-
startVoiceInput(silenceThresholdMs?: number, maxDurationMs?: number, listeningText?: string): Promise<ArrayBuffer>;
|
|
57
|
-
/**
|
|
58
|
-
* Stop the active voice recording session early. Causes the Promise returned
|
|
59
|
-
* by startVoiceInput() to resolve with the audio captured so far.
|
|
60
|
-
* No-op if no recording is in progress.
|
|
61
|
-
*/
|
|
62
|
-
stopVoiceInput(): void;
|
|
63
34
|
/**
|
|
64
35
|
* sets the specified template as root template, initializes a new stack
|
|
65
36
|
* Promise might contain an error message in case setting root template failed
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { HybridObject } from 'react-native-nitro-modules';
|
|
2
|
+
import type { NitroAutomotivePermissionRequestTemplateConfig } from '../templates/AutomotivePermissionRequestTemplate';
|
|
3
|
+
import type { NitroTemplateConfig } from './AutoPlay.nitro';
|
|
4
|
+
interface AutomotivePermissionRequestTemplateConfig extends NitroTemplateConfig, NitroAutomotivePermissionRequestTemplateConfig {
|
|
5
|
+
}
|
|
6
|
+
export interface AutomotivePermissionRequestTemplate extends HybridObject<{
|
|
7
|
+
android: 'kotlin';
|
|
8
|
+
}> {
|
|
9
|
+
createAutomotivePermissionRequestTemplate(config: AutomotivePermissionRequestTemplateConfig): void;
|
|
10
|
+
}
|
|
11
|
+
export {};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { HybridObject } from 'react-native-nitro-modules';
|
|
2
|
+
import type { NitroAutomotivePermissionRequestTemplateConfig } from '../templates/AutomotivePermissionRequestTemplate';
|
|
3
|
+
import type { NitroTemplateConfig } from './AutoPlay.nitro';
|
|
4
|
+
interface AutomotivePermissionRequestTemplateConfig extends NitroTemplateConfig, NitroAutomotivePermissionRequestTemplateConfig {
|
|
5
|
+
}
|
|
6
|
+
export interface AutomotivePermissionRequestTemplate extends HybridObject<{
|
|
7
|
+
android: 'kotlin';
|
|
8
|
+
}> {
|
|
9
|
+
createAutomotivePermissionRequestTemplate(config: AutomotivePermissionRequestTemplateConfig): void;
|
|
10
|
+
}
|
|
11
|
+
export {};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import type { HybridObject } from 'react-native-nitro-modules';
|
|
2
|
+
import type { VoiceInputChunk, VoiceInputResult } from '../types/Voice';
|
|
3
|
+
export interface Voice extends HybridObject<{
|
|
4
|
+
android: 'kotlin';
|
|
5
|
+
ios: 'swift';
|
|
6
|
+
}> {
|
|
7
|
+
/**
|
|
8
|
+
* Returns true if all permissions required for voice input are granted.
|
|
9
|
+
* On iOS: checks both microphone and speech recognition authorization.
|
|
10
|
+
* On Android: checks RECORD_AUDIO permission.
|
|
11
|
+
*/
|
|
12
|
+
hasVoiceInputPermission(): boolean;
|
|
13
|
+
/**
|
|
14
|
+
* Request all permissions required for voice input.
|
|
15
|
+
* On iOS: requests microphone permission then speech recognition authorization.
|
|
16
|
+
* On Android: requests RECORD_AUDIO via car context when connected, otherwise
|
|
17
|
+
* via the React Native application context.
|
|
18
|
+
* Returns true only if all required permissions were granted.
|
|
19
|
+
*/
|
|
20
|
+
requestVoiceInputPermission(): Promise<boolean>;
|
|
21
|
+
/**
|
|
22
|
+
* Start an in-app voice session.
|
|
23
|
+
*
|
|
24
|
+
* When preferSpeechToText is true:
|
|
25
|
+
* iOS — streams audio buffers into SFSpeechRecognizer during recording;
|
|
26
|
+
* onChunk fires with partial transcription results; resolves with
|
|
27
|
+
* { transcription } or falls back to { audio } if unavailable.
|
|
28
|
+
* Android — checks SpeechRecognizer availability upfront; if available it
|
|
29
|
+
* owns the mic and streams partial results via onChunk; if not
|
|
30
|
+
* available falls back to PCM recording.
|
|
31
|
+
*
|
|
32
|
+
* When preferSpeechToText is false (default):
|
|
33
|
+
* Both platforms record raw PCM; onChunk fires with audio chunks;
|
|
34
|
+
* resolves with { audio }.
|
|
35
|
+
*
|
|
36
|
+
* @param silenceThresholdMs ms of silence before auto-stop (default 1500)
|
|
37
|
+
* @param maxDurationMs hard cap on recording duration (default 10000)
|
|
38
|
+
* @param listeningText iOS only — text shown on CPVoiceControlTemplate
|
|
39
|
+
* @param preferSpeechToText request STT transcription instead of raw PCM
|
|
40
|
+
* @param onChunk optional streaming callback; when set the
|
|
41
|
+
* promise resolves with an empty VoiceInputResult
|
|
42
|
+
*/
|
|
43
|
+
startVoiceInput(silenceThresholdMs?: number, maxDurationMs?: number, listeningText?: string, preferSpeechToText?: boolean, onChunk?: (chunk: VoiceInputChunk) => void): Promise<VoiceInputResult>;
|
|
44
|
+
/**
|
|
45
|
+
* Stop the active voice session early.
|
|
46
|
+
* For PCM mode: resolves startVoiceInput with audio captured so far.
|
|
47
|
+
* For STT mode: finalises the recognition request.
|
|
48
|
+
* No-op if no session is active.
|
|
49
|
+
*/
|
|
50
|
+
stopVoiceInput(): void;
|
|
51
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { CustomActionButtonAndroid } from '../types/Button';
|
|
2
|
+
import type { AutoText } from '../types/Text';
|
|
3
|
+
import { type NitroAction } from '../utils/NitroAction';
|
|
4
|
+
import { type HeaderActions, Template, type TemplateConfig } from './Template';
|
|
5
|
+
export interface NitroAutomotivePermissionRequestTemplateConfig extends TemplateConfig {
|
|
6
|
+
headerActions?: Array<NitroAction>;
|
|
7
|
+
title: AutoText;
|
|
8
|
+
actions: Array<NitroAction>;
|
|
9
|
+
}
|
|
10
|
+
export type AutomotivePermissionRequestTemplateConfig = Omit<NitroAutomotivePermissionRequestTemplateConfig, 'headerActions' | 'buttons'> & {
|
|
11
|
+
/**
|
|
12
|
+
* action buttons, usually at the the top right on Android and a top bar on iOS
|
|
13
|
+
*/
|
|
14
|
+
headerActions?: HeaderActions<AutomotivePermissionRequestTemplate>;
|
|
15
|
+
actions: [CustomActionButtonAndroid<AutomotivePermissionRequestTemplate>] | [
|
|
16
|
+
CustomActionButtonAndroid<AutomotivePermissionRequestTemplate>,
|
|
17
|
+
CustomActionButtonAndroid<AutomotivePermissionRequestTemplate>
|
|
18
|
+
];
|
|
19
|
+
};
|
|
20
|
+
export declare class AutomotivePermissionRequestTemplate extends Template<AutomotivePermissionRequestTemplateConfig, HeaderActions<AutomotivePermissionRequestTemplate>> {
|
|
21
|
+
private template;
|
|
22
|
+
constructor(config: AutomotivePermissionRequestTemplateConfig);
|
|
23
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { NitroModules } from 'react-native-nitro-modules';
|
|
2
|
+
import { NitroActionUtil } from '../utils/NitroAction';
|
|
3
|
+
import { Template, } from './Template';
|
|
4
|
+
const HybridAutomotivePermissionRequestTemplate = NitroModules.createHybridObject('AutomotivePermissionRequestTemplate');
|
|
5
|
+
export class AutomotivePermissionRequestTemplate extends Template {
|
|
6
|
+
template = this;
|
|
7
|
+
constructor(config) {
|
|
8
|
+
super(config);
|
|
9
|
+
const { headerActions, actions, ...rest } = config;
|
|
10
|
+
const nitroConfig = {
|
|
11
|
+
...rest,
|
|
12
|
+
id: this.id,
|
|
13
|
+
headerActions: NitroActionUtil.convert(this.template, headerActions),
|
|
14
|
+
actions: NitroActionUtil.convert(this.template, actions),
|
|
15
|
+
};
|
|
16
|
+
HybridAutomotivePermissionRequestTemplate.createAutomotivePermissionRequestTemplate(nitroConfig);
|
|
17
|
+
}
|
|
18
|
+
}
|