@gmessier/nitro-speech 0.3.2 → 0.4.0

package/src/index.ts

@@ -1,182 +1,6 @@
-import React from 'react'
-import { NitroModules } from 'react-native-nitro-modules'
-import type {
-  NitroSpeech as NitroSpeechSpec,
-  Recognizer as RecognizerSpec,
-  SpeechToTextParams,
-} from './specs/NitroSpeech.nitro'
-
-const NitroSpeech =
-  NitroModules.createHybridObject<NitroSpeechSpec>('NitroSpeech')
-
-/**
- * Unsafe access to the Recognizer Session.
- */
-export const RecognizerSession = NitroSpeech.recognizer
-
-type RecognizerCallbacks = Pick<
-  RecognizerSpec,
-  | 'onReadyForSpeech'
-  | 'onRecordingStopped'
-  | 'onResult'
-  | 'onAutoFinishProgress'
-  | 'onError'
-  | 'onPermissionDenied'
-  | 'onVolumeChange'
->
-
-type RecognizerHandlers = Pick<
-  RecognizerSpec,
-  | 'startListening'
-  | 'stopListening'
-  | 'addAutoFinishTime'
-  | 'updateAutoFinishTime'
-  | 'getIsActive'
->
-
-const recognizerStartListening = (params: SpeechToTextParams) => {
-  RecognizerSession.startListening(params)
-}
-
-const recognizerStopListening = () => {
-  RecognizerSession.stopListening()
-}
-
-const recognizerAddAutoFinishTime = (additionalTimeMs?: number) => {
-  RecognizerSession.addAutoFinishTime(additionalTimeMs)
-}
-
-const recognizerUpdateAutoFinishTime = (
-  newTimeMs: number,
-  withRefresh?: boolean
-) => {
-  RecognizerSession.updateAutoFinishTime(newTimeMs, withRefresh)
-}
-
-const recognizerGetIsActive = () => {
-  return RecognizerSession.getIsActive()
-}
-
-const subscribers = new Set<RecognizerSpec['onVolumeChange']>()
-let currentVolume = 0
-
-/**
- * Subscription to the voice input volume changes
- *
- * Updates with arbitrary frequency (many times per second) while audio recording is active.
- *
- * @returns The current voice input volume normalized to a range of 0 to 1.
- */
-export const useVoiceInputVolume = () => {
-  return React.useSyncExternalStore(
-    (subscriber) => {
-      subscribers.add(subscriber)
-      return () => subscribers.delete(subscriber)
-    },
-    () => currentVolume
-  )
-}
-
-const handleVolumeChange: RecognizerSpec['onVolumeChange'] = (normVolume) => {
-  if (normVolume === currentVolume) return
-  currentVolume = normVolume
-  subscribers.forEach((subscriber) => subscriber?.(normVolume))
-}
-
-/**
- * Unsafe access to default Recognizer Session's volume change handler.
- *
- * In case you use static Recognizer Session:
- *
- * ```typescript
- * import { unsafe_onVolumeChange } from '@gmessier/nitro-speech'
- *
- * RecognizerSession.onVolumeChange = unsafe_onVolumeChange
- * ... // do something
- * RecognizerSession.startListening({ locale: 'en-US' })
- * ```
- */
-export const unsafe_onVolumeChange = handleVolumeChange
-
-/**
- * Safe, lifecycle-aware hook to use the recognizer.
- *
- * @param callbacks - The callbacks to use for the recognizer.
- * @param destroyDeps - The additional dependencies to use for the cleanup effect.
- *
- * Example: To cleanup when the screen is unfocused.
- *
- * ```typescript
- * const isFocused = useIsFocused()
- * useRecognizer({ ... }, [isFocused])
- * ```
- */
-export const useRecognizer = (
-  callbacks: RecognizerCallbacks,
-  destroyDeps: React.DependencyList = []
-): RecognizerHandlers => {
-  React.useEffect(() => {
-    if (callbacks.onVolumeChange) {
-      RecognizerSession.onVolumeChange = (normVolume: number) => {
-        callbacks.onVolumeChange?.(normVolume)
-      }
-    } else {
-      RecognizerSession.onVolumeChange = handleVolumeChange
-    }
-    RecognizerSession.onReadyForSpeech = () => {
-      callbacks.onReadyForSpeech?.()
-    }
-    RecognizerSession.onRecordingStopped = () => {
-      callbacks.onRecordingStopped?.()
-    }
-    RecognizerSession.onResult = (resultBatches: string[]) => {
-      callbacks.onResult?.(resultBatches)
-    }
-    RecognizerSession.onAutoFinishProgress = (timeLeftMs: number) => {
-      callbacks.onAutoFinishProgress?.(timeLeftMs)
-    }
-    RecognizerSession.onError = (message: string) => {
-      callbacks.onError?.(message)
-    }
-    RecognizerSession.onPermissionDenied = () => {
-      callbacks.onPermissionDenied?.()
-    }
-    return () => {
-      RecognizerSession.onReadyForSpeech = undefined
-      RecognizerSession.onRecordingStopped = undefined
-      RecognizerSession.onResult = undefined
-      RecognizerSession.onAutoFinishProgress = undefined
-      RecognizerSession.onError = undefined
-      RecognizerSession.onPermissionDenied = undefined
-      RecognizerSession.onVolumeChange = undefined
-    }
-  }, [callbacks])
-
-  React.useEffect(() => {
-    return () => {
-      RecognizerSession.stopListening()
-    }
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [...destroyDeps])
-
-  return {
-    startListening: recognizerStartListening,
-    stopListening: recognizerStopListening,
-    addAutoFinishTime: recognizerAddAutoFinishTime,
-    updateAutoFinishTime: recognizerUpdateAutoFinishTime,
-    getIsActive: recognizerGetIsActive,
-  }
-}
-
-/**
- * Safe reference to the Recognizer methods.
- */
-export const RecognizerRef: RecognizerHandlers = {
-  startListening: recognizerStartListening,
-  stopListening: recognizerStopListening,
-  addAutoFinishTime: recognizerAddAutoFinishTime,
-  updateAutoFinishTime: recognizerUpdateAutoFinishTime,
-  getIsActive: recognizerGetIsActive,
-}
-
-export type { RecognizerCallbacks, RecognizerHandlers, SpeechToTextParams }
+export * from './Recognizer/useRecognizer'
+export * from './Recognizer/useVoiceInputVolume'
+export * from './Recognizer/SpeechRecognizer'
+export * from './Recognizer/RecognizerRef'
+export * from './Recognizer/types'
+export * from './NitroSpeech'

package/src/specs/NitroSpeech.nitro.ts

@@ -1,166 +1,5 @@
-import { type HybridObject } from 'react-native-nitro-modules'
-
-interface ParamsAndroid {
-  /**
-   * Default - false
-   *
-   * Prefer quality over latency (may break autofinish timing, depends on engine)
-   *
-   * Android 13+
-   */
-  androidFormattingPreferQuality?: boolean
-  /**
-   * Default - false
-   *
-   * Language model based on web search terms. (may not work on some devices)
-   *
-   * Default - free form model
-   */
-  androidUseWebSearchModel?: boolean
-  /**
-   * Default - false.
-   *
-   * If required to handle batches non-default way.
-   *
-   * Will add lots of batches with empty or similar content to the result.
-   */
-  androidDisableBatchHandling?: boolean
-}
-
-interface ParamsIOS {
-  /**
-   * Default - true
-   *
-   * Adds punctuation to speech recognition results
-   *
-   * iOS 16+
-   */
-  iosAddPunctuation?: boolean
-}
-
-type HapticFeedbackStyle = 'light' | 'medium' | 'heavy' | 'none'
-
-export interface SpeechToTextParams extends ParamsAndroid, ParamsIOS {
-  /**
-   * Default - "en-US"
-   */
-  locale?: string
-  /**
-   * Default - 8s
-   */
-  autoFinishRecognitionMs?: number
-  /**
-   * Default - false
-   *
-   * Lots of repeating words in a row can be annoying
-   */
-  disableRepeatingFilter?: boolean
-  /**
-   * Default - empty array
-   *
-   * An array of strings that should be recognized, even if they are not in the system vocabulary.
-   */
-  contextualStrings?: string[]
-  /**
-   * Default - "medium"
-   *
-   * Haptic feedback style when microphone starts recording.
-   */
-  startHapticFeedbackStyle?: HapticFeedbackStyle
-  /**
-   * Default - "medium"
-   *
-   * Haptic feedback style when microphone stops recording.
-   */
-  stopHapticFeedbackStyle?: HapticFeedbackStyle
-  /**
-   * Default - false
-   *
-   * Android 13+
-   *
-   * iOS 26+ (iOS <26: always `false`)
-   */
-  maskOffensiveWords?: boolean
-}
-
-export interface Recognizer extends HybridObject<{
-  ios: 'swift'
-  android: 'kotlin'
-}> {
-  /**
-   * Tries to start the speech recognition.
-   *
-   * Not guaranteed to start the speech recognition.
-   */
-  startListening(params: SpeechToTextParams): void
-  /**
-   * Stops the speech recognition. if not started, does nothing.
-   *
-   * Not a sync operation for android, delay about 250ms to polish the result.
-   *
-   * Use onRecordingStopped to handle the stop event.
-   */
-  stopListening(): void
-
-  /**
-   * Manually adds time to the auto finish progress.
-   *
-   * If you want to give the user ability to manually increase time before timer calls stop.
-   *
-   * @param additionalTimeMs - time in ms to add to the current auto finish timer. If not set, will reset the timer to the original auto finish time.
-   */
-  addAutoFinishTime(additionalTimeMs?: number): void
-
-  /**
-   * Updates the auto finish time.
-   *
-   * Applies changes only within the current recognition session.
-   *
-   * @param newTimeMs - new time in ms for the auto finish timer.
-   * @param withRefresh - if true, will refresh the auto finish progress.
-   */
-  updateAutoFinishTime(newTimeMs: number, withRefresh?: boolean): void
-
-  /**
-   * Returns true if the speech recognition is active.
-   */
-  getIsActive(): boolean
-
-  /**
-   * The speech recognition has started.
-   */
-  onReadyForSpeech?: () => void
-  /**
-   * Audio recording has stopped.
-   */
-  onRecordingStopped?: () => void
-  /**
-   * Called each time either a new batch has been added or the last batch has been updated.
-   */
-  onResult?: (resultBatches: string[]) => void
-  /**
-   * Called each second while auto finish is in progress.
-   *
-   * Time left in milliseconds. Always more than 1000ms.
-   *
-   * TODO: Add for android
-   */
-  onAutoFinishProgress?: (timeLeftMs: number) => void
-  /**
-   * Error of the speech recognition.
-   */
-  onError?: (message: string) => void
-  /**
-   * Permission to record audio has been denied.
-   */
-  onPermissionDenied?: () => void
-  /**
-   * Called with arbitrary frequency (many times per second) while audio recording is active.
-   *
-   * Voice input volume normalized to a range of 0 to 1.
-   */
-  onVolumeChange?: (normVolume: number) => void
-}
+import type { HybridObject } from 'react-native-nitro-modules'
+import type { Recognizer } from './Recognizer.nitro'
 
 export interface NitroSpeech extends HybridObject<{
   ios: 'swift'

package/src/specs/Recognizer.nitro.ts

@@ -0,0 +1,110 @@
+import type { HybridObject } from 'react-native-nitro-modules'
+import type {
+  MutableSpeechRecognitionConfig,
+  SpeechRecognitionConfig,
+} from './SpeechRecognitionConfig'
+import type { VolumeChangeEvent } from './VolumeChangeEvent'
+
+export interface Recognizer extends HybridObject<{
+  ios: 'swift'
+  android: 'kotlin'
+}> {
+  /**
+   * Prepare the speech recognition engine and the model for the given parameters.
+   *
+   * Omit the {@linkcode Promise} result if want to run synchronously.
+   * {@linkcode startListening} will start and resolve it automatically.
+   * Only `await` if run beforehand and want to react on the success
+   */
+  prewarm(defaultParams?: SpeechRecognitionConfig): Promise<void>
+
+  /**
+   * Try to start the speech recognition.
+   *
+   * Not guaranteed to start the speech recognition.
+   *
+   * On success - {@linkcode onReadyForSpeech} is called
+   *
+   * On failure - {@linkcode onError} is called
+   */
+  startListening(params?: SpeechRecognitionConfig): void
+
+  /**
+   * Stops the speech recognition. if not started, does nothing.
+   *
+   * Not a sync operation for android, delay about 250ms to polish the result.
+   *
+   * Use {@linkcode onRecordingStopped} to handle the stop event.
+   */
+  stopListening(): void
+
+  /**
+   * Reset the auto finish timer to current {@linkcode SpeechRecognitionConfig.autoFinishRecognitionMs}.
+   */
+  resetAutoFinishTime(): void
+
+  /**
+   * Add time to the auto finish timer once without changing the timer threshold.
+   *
+   * @param additionalTimeMs - time in ms to add to the current auto finish timer. If not set, will reset the timer to the original {@linkcode SpeechRecognitionConfig.autoFinishRecognitionMs}.
+   */
+  addAutoFinishTime(additionalTimeMs?: number): void
+
+  /**
+   * Applies changes only within the active recognition session.
+   *
+   * @param newConfig - new dynamic params for the speech recognition.
+   * @param resetAutoFinishTime - if true, will reset auto finish time to actual {@linkcode SpeechRecognitionConfig.autoFinishRecognitionMs}.
+   */
+  updateConfig(
+    newConfig?: MutableSpeechRecognitionConfig,
+    resetAutoFinishTime?: boolean
+  ): void
+
+  /**
+   * Returns true if the speech recognition is active.
+   */
+  getIsActive(): boolean
+
+  /**
+   * Returns a list of supported locales.
+   *
+   * @platform iOS only
+   */
+  getSupportedLocalesIOS(): string[]
+
+  /**
+   * The speech recognition session has started.
+   */
+  onReadyForSpeech?: () => void
+  /**
+   * The speech recognition session has stopped.
+   */
+  onRecordingStopped?: () => void
+  /**
+   * Called each time either a new batch has been added or the last batch has been updated.
+   */
+  onResult?: (resultBatches: string[]) => void
+  /**
+   * Called every {@linkcode SpeechRecognitionConfig.autoFinishProgressIntervalMs} or 1000ms
+   *
+   * Time left in milliseconds until the timer stops.
+   *
+   * @note not implemented for Android yet.
+   */
+  onAutoFinishProgress?: (timeLeftMs: number) => void
+  /**
+   * Error of the speech recognition.
+   */
+  onError?: (message: string) => void
+  /**
+   * The permission to use the microphone or recognize speech has been denied.
+   */
+  onPermissionDenied?: () => void
+  /**
+   * Called with high and arbitrary frequency (many times per second) while audio recording is active.
+   *
+   * @warning overriding it will disable the built-in `useVoiceInputVolume` hook.
+   */
+  onVolumeChange?: (event: VolumeChangeEvent) => void
+}

package/src/specs/SpeechRecognitionConfig.ts

@@ -0,0 +1,167 @@
+interface ParamsAndroid {
+  /**
+   * Prefer quality over latency (may break autofinish, depends on device)
+   *
+   * @since Android 13+
+   *
+   * @default false
+   */
+  androidFormattingPreferQuality?: boolean
+  /**
+   * Language model based on web search terms. (may not work on some devices)
+   *
+   * Free form model by default
+   *
+   * @default false
+   */
+  androidUseWebSearchModel?: boolean
+  /**
+   * Without batch handling the result will contain empty or same content batches.
+   *
+   * @default false
+   */
+  androidDisableBatchHandling?: boolean
+}
+
+type IosPreset = 'shortform' | 'general'
+
+interface ParamsIOS {
+  /**
+   * Add punctuation to speech recognition results
+   *
+   * @since iOS 16.0+
+   *
+   * @default true
+   */
+  iosAddPunctuation?: boolean
+  /**
+   * `"shortForm"` - for a short phrase or sentence, also disables punctuation
+   *
+   * `"general"` - for longer speeches, more accurate but delayed response
+   *
+   * @since iOS 26.0+
+   *
+   * @default "general"
+   */
+  iosPreset?: IosPreset
+  /**
+   * A processing hint indicating that the audio is from a speaker with a heavy accent, lisp, or other confounding factor.
+   *
+   * @since iOS 26.0+
+   *
+   * @default false
+   */
+  iosAtypicalSpeech?: boolean
+}
+
+type HapticFeedbackStyle = 'light' | 'medium' | 'heavy' | 'none'
+
+export interface MutableSpeechRecognitionConfig {
+  /**
+   * Silence timer duration with no voice detected
+   *
+   * for Android - undefined behavior for > 300_000ms (5 minutes)
+   *
+   * @default 8000
+   */
+  autoFinishRecognitionMs?: number
+  /**
+   * The interval at which `onAutoFinishProgress` will be triggered.
+   *
+   * min: 50ms,
+   *
+   * max: {@link autoFinishRecognitionMs}
+   *
+   * @default 1000
+   */
+  autoFinishProgressIntervalMs?: number
+  /**
+   * The voice detector sensitivity to reset the auto finish timer.
+   *
+   * Adjust for your expected environment.
+   *
+   * `0.00` - `1.00`
+   *
+   * `0.10` - Quiet room level
+   *
+   * `0.40` - Default, street, medium restaurant background, office level
+   *
+   * `0.60` - Loud restaurants, noisy office, party, highly crowded places level
+   *
+   * `1` - Use to disable volume-based reset of the auto finish timer
+   *
+   * @default 0.40
+   */
+  resetAutoFinishVoiceSensitivity?: number
+  /**
+   * Lots of repeating words in a row can be annoying
+   *
+   * @default false
+   */
+  disableRepeatingFilter?: boolean
+  /**
+   * Haptic feedback level when microphone starts recording.
+   *
+   * Use `"none"` to disable.
+   *
+   * @default "medium"
+   */
+  startHapticFeedbackStyle?: HapticFeedbackStyle
+  /**
+   * Haptic feedback level when microphone stops recording.
+   *
+   * Use `"none"` to disable.
+   *
+   * @default "medium"
+   */
+  stopHapticFeedbackStyle?: HapticFeedbackStyle
+}
+
+/**
+ * @note Configuration for android and ios may behave differently
+ * for the same properties, test both platforms
+ *
+ * Use `Plarform.select()` to match expectations
+ */
+export interface SpeechRecognitionConfig
+  extends MutableSpeechRecognitionConfig, ParamsAndroid, ParamsIOS {
+  /**
+   * @default "en-US"
+   *
+   * @example
+   * "fr-FR"
+   * "pt-PT"
+   * "pt-BR"
+   *
+   */
+  locale?: string
+  /**
+   * An array of strings that should be recognized, even if they are not in the system vocabulary.
+   *
+   * Use this property to specify short custom phrases that are unique to your app.
+   *
+   * You might include phrases with the names of characters, products, or places that are specific to your app.
+   *
+   * You might also include domain-specific terminology or unusual or made-up words.
+   *
+   * Assigning custom phrases to this property improves the likelihood of those phrases being recognized.
+   *
+   * Keep phrases relatively brief, limiting them to one or two words whenever possible.
+   * Lengthy phrases are less likely to be recognized.
+   * In addition, try to limit each phrase to something the user can say without pausing.
+   *
+   * Limit the total number of phrases to no more than 100.
+   *
+   * @default []
+   */
+  contextualStrings?: string[]
+  /**
+   * Mask offensive words with asterisks.
+   *
+   * @since Android 13+
+   * @since iOS 26+ (iOS <26: always `false`)
+   *
+   * @default false
+   */
+  maskOffensiveWords?: boolean
+}

package/src/specs/VolumeChangeEvent.ts

@@ -0,0 +1,31 @@
+/**
+ * Contains data about each audio buffer volume.
+ *
+ * Emits with high frequency.
+ */
+export interface VolumeChangeEvent {
+  /**
+   * Smoothed voice input volume
+   *
+   * Normalized to a range of 0 to 1.
+   *
+   * Best choice for UI animations.
+   */
+  smoothedVolume: number
+  /**
+   * Raw voice input volume
+   *
+   * Normalized to a range of 0 to 1.
+   *
+   * Appropriate for internal logic, quick reactions, not UI.
+   */
+  rawVolume: number
+  /**
+   * Audio buffer volume in decibels.
+   *
+   * May vary on different devices and audio engines.
+   *
+   * db 0 is still a sound, undefined is no sound.
+   */
+  db?: number
+}

package/android/proguard-rules.pro

@@ -1 +0,0 @@
--keep class com.margelo.nitro.nitrospeech.** { *; }