@iternio/react-native-auto-play 0.3.11 → 0.4.0

package/README.md

@@ -304,7 +304,49 @@ When using build variants, Android Studio may not be aware of the selected varia
 To work around this and allow for debugging or enhancing the Android Automotive-specific implementation, you can temporarily set the automotive flags in your `gradle.properties` file or your default `.env` file before running a Gradle sync.
 
 ## Icons
-The library is using [Material Symbols](https://fonts.google.com/icons) for iconography. The font is bundled with the library, so no extra setup is required. You can use these icons on both Android Auto and CarPlay.
+The library does **not** bundle any icon font — the consuming app must provide one.
+
+### Setup
+
+1. Add a `.ttf` font file to your native projects:
+   - **iOS** — add `<name>.ttf` to your app bundle (no `UIAppFonts` entry needed — the library registers it via CoreText automatically).
+   - **Android** — place `<name>.ttf` in `res/font/`.
+
+   For cross-platform compatibility use **lowercase names with underscores only** (e.g. `material_symbols`).
+
+2. Register the font and an optional glyph map at startup:
+
+   ```ts
+   import { setIconFont } from '@iternio/react-native-auto-play';
+   import { glyphMap } from './assets/Glyphmap';
+
+   setIconFont('material_symbols', glyphMap);
+   ```
+
+3. Use glyph images by name or code point:
+
+   ```ts
+   { type: 'glyph', name: 'directions_car' }
+   { type: 'glyph', codepoint: 0xe531 }
+   ```
+
+`setIconFont` must be called once before the first glyph is used (subsequent calls are ignored). If no font is registered, the library throws an error when a glyph image is rendered.
+
+### Type-safe glyph names
+
+To get autocompletion and type checking for glyph names, create a declaration file in your app (e.g. `autoplay-glyphs.d.ts`):
+
+```ts
+import type { GlyphName } from './assets/Glyphmap';
+
+declare module '@iternio/react-native-auto-play' {
+  interface AutoPlayGlyphMap extends Record<GlyphName, number> {}
+}
+```
+
+Without this augmentation, `name` accepts any `string`. With it, only keys from your glyph map are allowed and you get full autocompletion.
+
+The example app uses [Material Symbols](https://fonts.google.com/icons). See `apps/example/assets/symbolFont/` for the glyph map generation script.
 
 It is also possible to use custom bundled images (e.g. PNG, WEBP or Vector Drawables). Make sure to add them to your native projects.
 - iOS: Add to your `Images.xcassets`
@@ -613,7 +655,7 @@ This section lists the available listeners and lifecycle callbacks so you can wi
 | --- | --- | --- |
 | `HybridAutoPlay.addListener(event, cb)` | event: `'didConnect'` `'didDisconnect'` | Connection changes for the head unit. |
 | `HybridAutoPlay.addListenerRenderState(moduleName, cb)` | `cb(visibility: 'willAppear' \| 'didAppear' \| 'willDisappear' \| 'didDisappear')` | Use `AutoPlayModules.*` or a cluster UUID. |
-| `HybridAutoPlay.addListenerVoiceInput(cb)` | `cb(location?, query?)` | Android-only voice input. |
+| `HybridAutoPlay.addListenerVoiceInput(cb)` | `cb(location?, query?)` | Android-only. Fires when the OS triggers a voice input event (e.g. "Hey Google, navigate to…"). For in-app recording use `startVoiceInput` instead. |
 | `HybridAutoPlay.addSafeAreaInsetsListener(moduleName, cb)` | `cb(insets)` | Safe area inset changes for any module. |
 
 ```ts
@@ -760,10 +802,68 @@ new ListTemplate({
 }).push();
 ```
 
+### Voice Input
+
+The library provides a cross-platform in-app voice recording API built on top of the car microphone (when connected) or the device microphone (when no car is connected).
+
+#### Permission
+
+```ts
+// Check whether permission is already granted
+const granted = HybridAutoPlay.hasVoiceInputPermission();
+
+// Request permission if not yet granted
+const granted = await HybridAutoPlay.requestVoiceInputPermission();
+```
+
+#### Recording
+
+```ts
+// Start recording — resolves with a raw PCM ArrayBuffer (16 kHz, 16-bit, mono)
+// Recording stops automatically on silence or when maxDurationMs is reached
+const pcmBuffer = await HybridAutoPlay.startVoiceInput(
+  1500,       // silenceThresholdMs (default 1500)
+  10_000,     // maxDurationMs (default 10 000)
+  'Listening...' // text shown on the car screen while recording (iOS CarPlay)
+);
+
+// Stop recording early — resolves startVoiceInput with the audio captured so far
+HybridAutoPlay.stopVoiceInput();
+```
+
+On **Android**: uses `CarAudioRecord` when Android Auto is connected, otherwise falls back to standard `AudioRecord`.
+
+On **iOS**: presents `CPVoiceControlTemplate` on the car screen when CarPlay is connected, and captures audio via `AVAudioEngine`.
+
+#### OS-triggered voice input (Android only)
+
+`addListenerVoiceInput` fires when the OS itself initiates a voice action (e.g. "Hey Google, navigate to…"). It is a no-op on iOS — use `startVoiceInput` for in-app recording on both platforms.
+
+```ts
+const cleanup = HybridAutoPlay.addListenerVoiceInput((location, query) => {
+  console.log('Voice query:', query, 'near', location);
+});
+```
+
+#### useVoiceInput hook (Android only)
+
+A convenience hook that wires up `addListenerVoiceInput` and exposes the latest `location` and `query` values reactively.
+
+```tsx
+import { useVoiceInput } from '@iternio/react-native-auto-play';
+
+const MyScreen = () => {
+  const { location, query } = useVoiceInput();
+  return <Text>{query ?? 'Say something…'}</Text>;
+};
+```
+
+---
+
 ### Hooks
 
 -   `useMapTemplate()`: Get a reference to the parent `MapTemplate` instance.
--   `useVoiceInput()`: Access voice input functionality - Android Auto only.
+-   `useVoiceInput()`: Reactively exposes the latest OS-triggered voice input (`location`, `query`). Android only — for in-app recording use `startVoiceInput` / `stopVoiceInput` directly.
 -   `useSafeAreaInsets()`: Get safe area insets for any root component.
 -   `useFocusedEffect()`: A useEffect alternative that executes when the specified component is visible to the user - use any of the `AutoPlayModules` enum or a cluster uuid to sepcify the component the effect should listen for.
 -   `useAndroidAutoTelemetry()`: Access to car telemetry data on Android Auto and Android Automotive.

package/ReactNativeAutoPlay.podspec

@@ -25,10 +25,6 @@ Pod::Spec.new do |s|
   # react helpers like RCTConvert
   s.public_header_files = Array(s.attributes_hash['public_header_files']) + ["ios/ReactHelpers/*.h"]
 
-  s.resource_bundles = {
-    "ReactNativeAutoPlay" => ['ios/Assets/**/*.ttf']
-  }
-  
   s.pod_target_xcconfig = {
     # C++ compiler flags, mainly for folly.
     "GCC_PREPROCESSOR_DEFINITIONS" => "$(inherited) FOLLY_NO_CONFIG FOLLY_CFG_NO_COROUTINES"

package/android/src/automotive/AndroidManifest.xml

@@ -19,6 +19,7 @@
     <uses-permission android:name="android.permission.FOREGROUND_SERVICE_LOCATION" />
 
     <uses-permission android:name="android.permission.WAKE_LOCK" />
+    <uses-permission android:name="android.permission.RECORD_AUDIO" />
 
 
     <!-- Android Automotive specific permissions -->

package/android/src/main/AndroidManifest.xml

@@ -12,6 +12,7 @@
     <uses-permission android:name="android.permission.FOREGROUND_SERVICE_LOCATION" />
 
     <uses-permission android:name="android.permission.WAKE_LOCK" />
+    <uses-permission android:name="android.permission.RECORD_AUDIO" />
 
     <!-- Android Auto specific permissions -->
     <uses-permission android:name="com.google.android.gms.permission.CAR_FUEL" />

package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/HybridAutoPlay.kt

@@ -1,12 +1,22 @@
 package com.margelo.nitro.swe.iternio.reactnativeautoplay
 
+import android.content.pm.PackageManager
+import android.os.Build
+import androidx.core.content.ContextCompat
 import com.facebook.react.bridge.UiThreadUtil
+import com.facebook.react.modules.core.PermissionAwareActivity
+import com.facebook.react.modules.core.PermissionListener
+import com.margelo.nitro.NitroModules
+import com.margelo.nitro.core.ArrayBuffer
 import com.margelo.nitro.core.Promise
 import com.margelo.nitro.swe.iternio.reactnativeautoplay.template.AndroidAutoTemplate
 import com.margelo.nitro.swe.iternio.reactnativeautoplay.template.MessageTemplate
 import com.margelo.nitro.swe.iternio.reactnativeautoplay.utils.ThreadUtil
+import kotlinx.coroutines.suspendCancellableCoroutine
+import java.nio.ByteBuffer
 import java.util.concurrent.ConcurrentHashMap
 import java.util.concurrent.CopyOnWriteArrayList
+import kotlin.coroutines.resume
 
 class HybridAutoPlay : HybridAutoPlaySpec() {
     init {
@@ -241,6 +251,84 @@ class HybridAutoPlay : HybridAutoPlaySpec() {
         }
     }
 
+    override fun hasVoiceInputPermission(): Boolean {
+        val context = NitroModules.applicationContext ?: return false
+        return ContextCompat.checkSelfPermission(
+            context, android.Manifest.permission.RECORD_AUDIO
+        ) == PackageManager.PERMISSION_GRANTED
+    }
+
+    override fun requestVoiceInputPermission(): Promise<Boolean> {
+        return Promise.async {
+            if (hasVoiceInputPermission()) {
+                return@async true
+            }
+
+            val carContext = AndroidAutoSession.getRootContext()
+
+            if (carContext != null) {
+                suspendCancellableCoroutine {
+                    carContext.requestPermissions(listOf(android.Manifest.permission.RECORD_AUDIO)) { approved, _ ->
+                        it.resume(approved.contains(android.Manifest.permission.RECORD_AUDIO))
+                    }
+                }
+            } else {
+                val context = NitroModules.applicationContext ?: return@async false
+                val activity =
+                    context.currentActivity as? PermissionAwareActivity ?: return@async false
+                val code = (Math.random() * 10000).toInt()
+
+                suspendCancellableCoroutine {
+                    activity.requestPermissions(
+                        arrayOf(android.Manifest.permission.RECORD_AUDIO),
+                        code,
+                        PermissionListener { requestCode, _, grantResults ->
+                            if (requestCode != code) {
+                                return@PermissionListener false
+                            }
+
+                            val granted =
+                                grantResults.isNotEmpty() && grantResults.first() == PackageManager.PERMISSION_GRANTED
+
+                            it.resume(granted)
+
+                            return@PermissionListener true
+                        })
+                }
+            }
+        }
+    }
+
+    override fun startVoiceInput(
+        silenceThresholdMs: Double?, maxDurationMs: Double?, listeningText: String?
+    ): Promise<ArrayBuffer> {
+        return Promise.async {
+            if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) {
+                throw UnsupportedOperationException("startVoiceInput requires at least API level ${Build.VERSION_CODES.O}")
+            }
+
+            val manager = VoiceInputManager(AndroidAutoSession.getRootContext())
+            voiceInputManager = manager
+
+            try {
+                val pcmBytes = manager.start(
+                    silenceThresholdMs = silenceThresholdMs?.toLong() ?: 1_500L,
+                    maxDurationMs = maxDurationMs?.toLong() ?: 10_000L,
+                )
+                val directBuffer =
+                    ByteBuffer.allocateDirect(pcmBytes.size).put(pcmBytes).rewind() as ByteBuffer
+                ArrayBuffer.wrap(directBuffer)
+            } finally {
+                voiceInputManager = null
+                manager.dispose()
+            }
+        }
+    }
+
+    override fun stopVoiceInput() {
+        voiceInputManager?.stop()
+    }
+
     companion object {
         const val TAG = "HybridAutoPlay"
 
@@ -251,6 +339,9 @@ class HybridAutoPlay : HybridAutoPlaySpec() {
 
         private val voiceInputListeners = CopyOnWriteArrayList<(Location?, String?) -> Unit>()
 
+        @Volatile
+        private var voiceInputManager: VoiceInputManager? = null
+
         private val safeAreaInsetsListeners =
             ConcurrentHashMap<String, CopyOnWriteArrayList<(SafeAreaInsets) -> Unit>>()
 

package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/VoiceInputManager.kt

@@ -0,0 +1,214 @@
+package com.margelo.nitro.swe.iternio.reactnativeautoplay
+
+import android.content.pm.PackageManager
+import android.media.AudioAttributes
+import android.media.AudioFocusRequest
+import android.media.AudioFormat
+import android.media.AudioManager
+import android.media.AudioRecord
+import android.media.MediaRecorder
+import android.os.Build
+import androidx.annotation.RequiresApi
+import androidx.car.app.CarContext
+import androidx.car.app.media.CarAudioRecord
+import androidx.core.content.ContextCompat
+import com.margelo.nitro.NitroModules
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.cancel
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.suspendCancellableCoroutine
+import java.io.ByteArrayOutputStream
+import kotlin.coroutines.Continuation
+import kotlin.coroutines.resume
+import kotlin.coroutines.resumeWithException
+import kotlin.math.abs
+
+/**
+ * Captures 16-bit PCM audio (16 kHz, mono).
+ * When [carContext] is provided uses CarAudioRecord (Android Auto/Automotive).
+ * When [carContext] is null falls back to standard AudioRecord (phone-only).
+ */
+class VoiceInputManager(
+    private val carContext: CarContext?,
+) {
+    private var carAudioRecord: CarAudioRecord? = null
+    private var audioRecord: AudioRecord? = null
+    private var audioFocusRequest: AudioFocusRequest? = null
+    private var recordingJob: Job? = null
+    private var continuation: Continuation<ByteArray>? = null
+    private val scope = CoroutineScope(Dispatchers.IO)
+
+    @Volatile
+    private var isRecording = false
+
+    /**
+     * Acquires audio focus, starts recording, and suspends until stopped.
+     * Stops automatically after [silenceThresholdMs] of silence or [maxDurationMs] total.
+     * Returns the complete raw PCM buffer (Int16 LE, 16 kHz, mono).
+     */
+    @RequiresApi(Build.VERSION_CODES.O)
+    suspend fun start(
+        silenceThresholdMs: Long = 1_500,
+        maxDurationMs: Long = 10_000,
+    ): ByteArray = suspendCancellableCoroutine { cont ->
+        val appContext = NitroModules.applicationContext
+        if (appContext == null || ContextCompat.checkSelfPermission(
+                appContext,
+                android.Manifest.permission.RECORD_AUDIO,
+            ) != PackageManager.PERMISSION_GRANTED
+        ) {
+            cont.resumeWithException(SecurityException("RECORD_AUDIO permission not granted"))
+            return@suspendCancellableCoroutine
+        }
+
+        continuation = cont
+
+        val audioManager = appContext.getSystemService(AudioManager::class.java)
+
+        val audioAttributes =
+            AudioAttributes.Builder().setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
+                .setUsage(AudioAttributes.USAGE_ASSISTANCE_NAVIGATION_GUIDANCE).build()
+
+        val focusRequest =
+            AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_EXCLUSIVE)
+                .setAudioAttributes(audioAttributes).setOnAudioFocusChangeListener { state ->
+                    if (state == AudioManager.AUDIOFOCUS_LOSS) {
+                        stop()
+                    }
+                }.build()
+
+        if (audioManager.requestAudioFocus(focusRequest) != AudioManager.AUDIOFOCUS_REQUEST_GRANTED) {
+            continuation = null
+            cont.resumeWithException(IllegalStateException("Audio focus request denied"))
+            return@suspendCancellableCoroutine
+        }
+
+        audioFocusRequest = focusRequest
+
+        val bufferSize: Int
+
+        if (carContext != null) {
+            val record = CarAudioRecord.create(carContext)
+            carAudioRecord = record
+            bufferSize = CarAudioRecord.AUDIO_CONTENT_BUFFER_SIZE
+            isRecording = true
+            record.startRecording()
+        } else {
+            val minBuffer = AudioRecord.getMinBufferSize(
+                SAMPLE_RATE,
+                AudioFormat.CHANNEL_IN_MONO,
+                AudioFormat.ENCODING_PCM_16BIT,
+            )
+            bufferSize = maxOf(minBuffer, PHONE_BUFFER_SIZE)
+            val record = AudioRecord(
+                MediaRecorder.AudioSource.MIC,
+                SAMPLE_RATE,
+                AudioFormat.CHANNEL_IN_MONO,
+                AudioFormat.ENCODING_PCM_16BIT,
+                bufferSize,
+            )
+            audioRecord = record
+            isRecording = true
+            record.startRecording()
+        }
+
+        val outputStream = ByteArrayOutputStream()
+
+        recordingJob = scope.launch {
+            val buffer = ByteArray(bufferSize)
+            val recordingStart = System.currentTimeMillis()
+            var silenceStart: Long? = null
+
+            try {
+                while (isRecording) {
+                    val read = carAudioRecord?.read(buffer, 0, bufferSize) ?: audioRecord?.read(
+                        buffer,
+                        0,
+                        bufferSize,
+                    ) ?: -1
+
+                    if (read < 0) {
+                        break
+                    }
+
+                    if (read > 0) {
+                        outputStream.write(buffer, 0, read)
+
+                        val now = System.currentTimeMillis()
+                        val elapsedMs = now - recordingStart
+
+                        if (elapsedMs >= maxDurationMs) {
+                            break
+                        }
+
+                        // Silence detection — skip during warm-up
+                        if (elapsedMs >= WARMUP_MS) {
+                            var peak = 0
+                            var i = 0
+                            while (i < read - 1) {
+                                val sample =
+                                    (buffer[i].toInt() and 0xFF) or (buffer[i + 1].toInt() shl 8)
+                                val absSample = abs(sample.toShort().toInt())
+                                if (absSample > peak) peak = absSample
+                                i += 2
+                            }
+
+                            if (peak < SILENCE_AMPLITUDE_THRESHOLD) {
+                                if (silenceStart == null) {
+                                    silenceStart = now
+                                }
+                                if (now - silenceStart >= silenceThresholdMs) {
+                                    break
+                                }
+                            } else {
+                                silenceStart = null
+                            }
+                        }
+                    }
+                }
+            } finally {
+                releaseResources()
+                val capturedContinuation = continuation
+                continuation = null
+                capturedContinuation?.resume(outputStream.toByteArray())
+            }
+        }
+    }
+
+    fun stop() {
+        isRecording = false
+        carAudioRecord?.stopRecording()
+        audioRecord?.stop()
+    }
+
+    @RequiresApi(Build.VERSION_CODES.O)
+    private fun releaseResources() {
+        carAudioRecord?.stopRecording()
+        carAudioRecord = null
+        audioRecord?.stop()
+        audioRecord?.release()
+        audioRecord = null
+        recordingJob = null
+        audioFocusRequest?.let {
+            val audioManager = (NitroModules.applicationContext ?: carContext)?.getSystemService(
+                AudioManager::class.java,
+            )
+            audioManager?.abandonAudioFocusRequest(it)
+        }
+        audioFocusRequest = null
+    }
+
+    fun dispose() {
+        stop()
+        scope.cancel()
+    }
+
+    companion object {
+        private const val SILENCE_AMPLITUDE_THRESHOLD = 500
+        private const val WARMUP_MS = 500L
+        private const val SAMPLE_RATE = 16_000
+        private const val PHONE_BUFFER_SIZE = 3_200 // ~100ms at 16kHz/16-bit/mono
+    }
+}

package/android/src/main/java/com/margelo/nitro/swe/iternio/reactnativeautoplay/template/MapTemplate.kt

@@ -377,7 +377,8 @@ class MapTemplate(
                 val notificationIcon = Parser.parseImageToBitmap(
                     context,
                     current.symbolImage.asFirstOrNull(),
-                    current.symbolImage.asSecondOrNull()
+                    current.symbolImage.asSecondOrNull(),
+                    current.symbolImage.asThirdOrNull()
                 )
 
                 val notificationText = currentStep.cue?.toString()