@siteed/expo-audio-stream 1.0.3 → 1.0.5

package/src/AudioAnalysis/extractAudioAnalysis.ts

@@ -1,136 +1,147 @@
 // packages/expo-audio-stream/src/AudioAnalysis/extractAudioAnalysis.ts
-import { AudioAnalysisData } from "./AudioAnalysis.types";
-import ExpoAudioStreamModule from "../ExpoAudioStreamModule";
-import { isWeb } from "../constants";
-import { getLogger } from "../logger";
-import { ExtractMetadataProps } from "../useAudioRecorder";
-import { convertPCMToFloat32 } from "../utils/convertPCMToFloat32";
-import { getWavFileInfo } from "../utils/getWavFileInfo";
-import { InlineFeaturesExtractor } from "../workers/InlineFeaturesExtractor.web";
+import {
+    AmplitudeAlgorithm,
+    AudioAnalysis,
+    AudioFeaturesOptions,
+} from './AudioAnalysis.types'
+import ExpoAudioStreamModule from '../ExpoAudioStreamModule'
+import { isWeb } from '../constants'
+import { getLogger } from '../logger'
+import { convertPCMToFloat32 } from '../utils/convertPCMToFloat32'
+import { getWavFileInfo, WavFileInfo } from '../utils/getWavFileInfo'
+import { InlineFeaturesExtractor } from '../workers/InlineFeaturesExtractor.web'
 
-const logger = getLogger("extractAudioAnalysis");
+const logger = getLogger('extractAudioAnalysis')
+
+export interface ExtractAudioAnalysisProps {
+    fileUri?: string // should provide either fileUri or arrayBuffer
+    wavMetadata?: WavFileInfo
+    arrayBuffer?: ArrayBuffer
+    bitDepth?: number
+    skipWavHeader?: boolean
+    durationMs?: number
+    sampleRate?: number
+    numberOfChannels?: number
+    algorithm?: AmplitudeAlgorithm
+    position?: number // Optional number of bytes to skip. Default is 0
+    length?: number // Optional number of bytes to read.
+    pointsPerSecond?: number // Optional number of points per second. Use to reduce the number of points and compute the number of datapoints to return.
+    features?: AudioFeaturesOptions
+    featuresExtratorUrl?: string
+}
 
 export const extractAudioAnalysis = async ({
-  fileUri,
-  pointsPerSecond = 20,
-  arrayBuffer,
-  bitDepth,
-  skipWavHeader,
-  durationMs,
-  sampleRate,
-  numberOfChannels,
-  algorithm = "rms",
-  features,
-  featuresExtratorUrl,
-}: ExtractMetadataProps): Promise<AudioAnalysisData> => {
-  if (isWeb) {
-    if (!arrayBuffer && !fileUri) {
-      throw new Error("Either arrayBuffer or fileUri must be provided");
-    }
+    fileUri,
+    pointsPerSecond = 20,
+    arrayBuffer,
+    bitDepth,
+    skipWavHeader = true,
+    durationMs,
+    sampleRate,
+    numberOfChannels,
+    algorithm = 'rms',
+    features,
+    featuresExtratorUrl,
+}: ExtractAudioAnalysisProps): Promise<AudioAnalysis> => {
+    if (isWeb) {
+        if (!arrayBuffer && !fileUri) {
+            throw new Error('Either arrayBuffer or fileUri must be provided')
+        }
 
-    if (!arrayBuffer) {
-      logger.log(`fetching fileUri`, fileUri);
-      const response = await fetch(fileUri!);
+        if (!arrayBuffer) {
+            logger.log(`fetching fileUri`, fileUri)
+            const response = await fetch(fileUri!)
 
-      if (!response.ok) {
-        throw new Error(`Failed to fetch fileUri: ${response.statusText}`);
-      }
+            if (!response.ok) {
+                throw new Error(
+                    `Failed to fetch fileUri: ${response.statusText}`
+                )
+            }
 
-      arrayBuffer = (await response.arrayBuffer()).slice(0);
-      logger.log(`fetched fileUri`, arrayBuffer.byteLength, arrayBuffer);
-    }
+            arrayBuffer = await response.arrayBuffer()
+            logger.log(`fetched fileUri`, arrayBuffer.byteLength, arrayBuffer)
+        }
 
-    // Create a new copy of the ArrayBuffer to avoid detachment issues
-    const bufferCopy = arrayBuffer.slice(0);
-    logger.log(
-      `extractAudioAnalysis skipWavHeader=${skipWavHeader} bitDepth=${bitDepth} len=${bufferCopy.byteLength}`,
-      bufferCopy.slice(0, 100),
-    );
+        // Create a new copy of the ArrayBuffer to avoid detachment issues
+        const bufferCopy = arrayBuffer.slice(0)
+        logger.log(
+            `extractAudioAnalysis skipWavHeader=${skipWavHeader} bitDepth=${bitDepth} len=${bufferCopy.byteLength}`,
+            bufferCopy.slice(0, 100)
+        )
 
-    let actualBitDepth = bitDepth;
-    if (!actualBitDepth) {
-      logger.log(
-        `extractAudioAnalysis bitDepth not provided -- getting wav file info`,
-      );
-      const fileInfo = await getWavFileInfo(bufferCopy);
-      actualBitDepth = fileInfo.bitDepth;
-    }
-    logger.log(`extractAudioAnalysis actualBitDepth=${actualBitDepth}`);
-    // let copyChannelData: Float32Array;
-    // try {
-    //   const audioContext = new (window.AudioContext ||
-    //     // @ts-ignore
-    //     window.webkitAudioContext)();
-    //   const audioBuffer = await audioContext.decodeAudioData(bufferCopy);
-    //   const channelData = audioBuffer.getChannelData(0); // Use only the first channel
-    //   copyChannelData = new Float32Array(channelData); // Create a new Float32Array
-    // } catch (error) {
-    //   console.warn("Failed to decode audio data:", error);
-    //   // Fall back to creating a new Float32Array from the ArrayBuffer if decoding fails
-    //   copyChannelData = new Float32Array(arrayBuffer);
-    // }
+        let actualBitDepth = bitDepth
+        if (!actualBitDepth) {
+            logger.log(
+                `extractAudioAnalysis bitDepth not provided -- getting wav file info`
+            )
+            const fileInfo = await getWavFileInfo(bufferCopy)
+            actualBitDepth = fileInfo.bitDepth
+        }
+        logger.log(`extractAudioAnalysis actualBitDepth=${actualBitDepth}`)
 
-    const {
-      pcmValues: channelData,
-      min,
-      max,
-    } = convertPCMToFloat32({
-      buffer: arrayBuffer,
-      bitDepth: actualBitDepth,
-      skipWavHeader,
-    });
-    logger.log(
-      `extractAudioAnalysis skipWaveHeader=${skipWavHeader} convertPCMToFloat32 length=${channelData.length} range: [ ${min} :: ${max} ]`,
-    );
+        const {
+            pcmValues: channelData,
+            min,
+            max,
+        } = await convertPCMToFloat32({
+            buffer: arrayBuffer,
+            bitDepth: actualBitDepth,
+            skipWavHeader,
+        })
+        logger.log(
+            `extractAudioAnalysis skipWaveHeader=${skipWavHeader} convertPCMToFloat32 length=${channelData.length} range: [ ${min} :: ${max} ]`
+        )
 
-    return new Promise((resolve, reject) => {
-      let worker: Worker;
-      if (featuresExtratorUrl) {
-        worker = new Worker(new URL(featuresExtratorUrl, window.location.href));
-      } else {
-        const blob = new Blob([InlineFeaturesExtractor], {
-          type: "application/javascript",
-        });
-        const url = URL.createObjectURL(blob);
-        worker = new Worker(url);
-      }
+        return new Promise((resolve, reject) => {
+            let worker: Worker
+            if (featuresExtratorUrl) {
+                worker = new Worker(
+                    new URL(featuresExtratorUrl, window.location.href)
+                )
+            } else {
+                const blob = new Blob([InlineFeaturesExtractor], {
+                    type: 'application/javascript',
+                })
+                const url = URL.createObjectURL(blob)
+                worker = new Worker(url)
+            }
 
-      worker.onmessage = (event) => {
-        resolve(event.data.result);
-      };
+            worker.onmessage = (event) => {
+                resolve(event.data.result)
+            }
 
-      worker.onerror = (error) => {
-        reject(error);
-      };
+            worker.onerror = (error) => {
+                reject(error)
+            }
 
-      worker.postMessage({
-        command: "process",
-        channelData,
-        sampleRate,
-        pointsPerSecond,
-        algorithm,
-        bitDepth,
-        fullAudioDurationMs: durationMs,
-        numberOfChannels,
-      });
-    });
-  } else {
-    if (!fileUri) {
-      throw new Error("fileUri is required");
+            worker.postMessage({
+                command: 'process',
+                channelData,
+                sampleRate,
+                pointsPerSecond,
+                algorithm,
+                bitDepth,
+                fullAudioDurationMs: durationMs,
+                numberOfChannels,
+            })
+        })
+    } else {
+        if (!fileUri) {
+            throw new Error('fileUri is required')
+        }
+        logger.log(`extractAudioAnalysis`, {
+            fileUri,
+            pointsPerSecond,
+            algorithm,
+        })
+        const res = await ExpoAudioStreamModule.extractAudioAnalysis({
+            fileUri,
+            pointsPerSecond,
+            skipWavHeader,
+            algorithm,
+            features,
+        })
+        logger.log(`extractAudioAnalysis`, res)
+        return res
     }
-    logger.log(`extractAudioAnalysis`, {
-      fileUri,
-      pointsPerSecond,
-      algorithm,
-    });
-    const res = await ExpoAudioStreamModule.extractAudioAnalysis({
-      fileUri,
-      pointsPerSecond,
-      skipWavHeader,
-      algorithm,
-      features,
-    });
-    logger.log(`extractAudioAnalysis`, res);
-    return res;
-  }
-};
+}

package/src/AudioAnalysis/extractWaveform.ts

@@ -1,25 +1,25 @@
-import ExpoAudioStreamModule from "../ExpoAudioStreamModule";
-import { getLogger } from "../logger";
+import ExpoAudioStreamModule from '../ExpoAudioStreamModule'
+import { getLogger } from '../logger'
 
-const logger = getLogger("extractWaveform");
+const logger = getLogger('extractWaveform')
 export interface ExtractWaveformProps {
-  fileUri: string;
-  numberOfSamples: number;
-  offset?: number;
-  length?: number;
+    fileUri: string
+    numberOfSamples: number
+    offset?: number
+    length?: number
 }
 export const extractWaveform = async ({
-  fileUri,
-  numberOfSamples,
-  offset = 0,
-  length,
-}: ExtractWaveformProps): Promise<unknown> => {
-  const res = await ExpoAudioStreamModule.extractAudioAnalysis({
     fileUri,
     numberOfSamples,
-    offset,
+    offset = 0,
     length,
-  });
-  logger.log(`extractWaveform`, res);
-  return res;
-};
+}: ExtractWaveformProps): Promise<unknown> => {
+    const res = await ExpoAudioStreamModule.extractAudioAnalysis({
+        fileUri,
+        numberOfSamples,
+        offset,
+        length,
+    })
+    logger.log(`extractWaveform`, res)
+    return res
+}

package/src/AudioRecorder.provider.tsx

@@ -1,70 +1,70 @@
 // packages/expo-audio-stream/src/AudioRecorder.provider.tsx
-import React, { createContext, useContext } from "react";
+import React, { createContext, useContext } from 'react'
 
-import { AudioAnalysisData } from "./AudioAnalysis/AudioAnalysis.types";
+import { AudioAnalysis } from './AudioAnalysis/AudioAnalysis.types'
 import {
-  AudioRecordingResult,
-  RecordingConfig,
-  StartRecordingResult,
-} from "./ExpoAudioStream.types";
-import { UseAudioRecorderProps, useAudioRecorder } from "./useAudioRecorder";
+    AudioRecording,
+    RecordingConfig,
+    StartRecordingResult,
+} from './ExpoAudioStream.types'
+import { UseAudioRecorderProps, useAudioRecorder } from './useAudioRecorder'
 
 export interface UseAudioRecorderState {
-  startRecording: (_: RecordingConfig) => Promise<StartRecordingResult>;
-  stopRecording: () => Promise<AudioRecordingResult | null>;
-  pauseRecording: () => void;
-  resumeRecording: () => void;
-  isRecording: boolean;
-  isPaused: boolean;
-  durationMs: number; // Duration of the recording
-  size: number; // Size in bytes of the recorded audio
-  analysisData?: AudioAnalysisData;
+    startRecording: (_: RecordingConfig) => Promise<StartRecordingResult>
+    stopRecording: () => Promise<AudioRecording | null>
+    pauseRecording: () => void
+    resumeRecording: () => void
+    isRecording: boolean
+    isPaused: boolean
+    durationMs: number // Duration of the recording
+    size: number // Size in bytes of the recorded audio
+    analysisData?: AudioAnalysis // Analysis data for the recording depending on enableProcessing flag
 }
 
 const initContext: UseAudioRecorderState = {
-  isRecording: false,
-  isPaused: false,
-  durationMs: 0,
-  size: 0,
-  startRecording: async () => {
-    throw new Error("AudioRecorderProvider not found");
-  },
-  stopRecording: async () => {
-    throw new Error("AudioRecorderProvider not found");
-  },
-  pauseRecording: () => {
-    throw new Error("AudioRecorderProvider not found");
-  },
-  resumeRecording: () => {
-    throw new Error("AudioRecorderProvider not found");
-  },
-};
+    isRecording: false,
+    isPaused: false,
+    durationMs: 0,
+    size: 0,
+    startRecording: async () => {
+        throw new Error('AudioRecorderProvider not found')
+    },
+    stopRecording: async () => {
+        throw new Error('AudioRecorderProvider not found')
+    },
+    pauseRecording: () => {
+        throw new Error('AudioRecorderProvider not found')
+    },
+    resumeRecording: () => {
+        throw new Error('AudioRecorderProvider not found')
+    },
+}
 
-const AudioRecorderContext = createContext<UseAudioRecorderState>(initContext);
+const AudioRecorderContext = createContext<UseAudioRecorderState>(initContext)
 
 interface AudioRecorderProviderProps {
-  children: React.ReactNode;
-  config?: UseAudioRecorderProps;
+    children: React.ReactNode
+    config?: UseAudioRecorderProps
 }
 
 export const AudioRecorderProvider: React.FC<AudioRecorderProviderProps> = ({
-  children,
-  config = {},
+    children,
+    config = {},
 }) => {
-  const audioRecorder = useAudioRecorder(config);
-  return (
-    <AudioRecorderContext.Provider value={audioRecorder}>
-      {children}
-    </AudioRecorderContext.Provider>
-  );
-};
+    const audioRecorder = useAudioRecorder(config)
+    return (
+        <AudioRecorderContext.Provider value={audioRecorder}>
+            {children}
+        </AudioRecorderContext.Provider>
+    )
+}
 
 export const useSharedAudioRecorder = () => {
-  const context = useContext(AudioRecorderContext);
-  if (!context) {
-    throw new Error(
-      "useSharedAudioRecorder must be used within an AudioRecorderProvider",
-    );
-  }
-  return context;
-};
+    const context = useContext(AudioRecorderContext)
+    if (!context) {
+        throw new Error(
+            'useSharedAudioRecorder must be used within an AudioRecorderProvider'
+        )
+    }
+    return context
+}

package/src/ExpoAudioStream.native.ts

@@ -1,6 +1,6 @@
 // src/ExpoAudioStreamModule.ts
-import { requireNativeModule } from "expo-modules-core";
+import { requireNativeModule } from 'expo-modules-core'
 
 // It loads the native module object from the JSI or falls back to
 // the bridge module (from NativeModulesProxy) if the remote debugger is on.
-export default requireNativeModule("ExpoAudioStream");
+export default requireNativeModule('ExpoAudioStream')

package/src/ExpoAudioStream.types.ts

@@ -1,74 +1,77 @@
 // packages/expo-audio-stream/src/ExpoAudioStream.types.ts
 import {
-  AudioAnalysisData,
-  AudioFeaturesOptions,
-} from "./AudioAnalysis/AudioAnalysis.types";
+    AmplitudeAlgorithm,
+    AudioAnalysis,
+    AudioFeaturesOptions,
+} from './AudioAnalysis/AudioAnalysis.types'
+import { AudioAnalysisEvent } from './events'
 
 export interface AudioStreamStatus {
-  isRecording: boolean;
-  isPaused: boolean;
-  durationMs: number;
-  size: number;
-  interval: number;
-  mimeType: string;
+    isRecording: boolean
+    isPaused: boolean
+    durationMs: number
+    size: number
+    interval: number
+    mimeType: string
 }
 
 export interface AudioDataEvent {
-  data: string | ArrayBuffer;
-  position: number;
-  fileUri: string;
-  eventDataSize: number;
-  totalSize: number;
+    data: string | Float32Array
+    position: number
+    fileUri: string
+    eventDataSize: number
+    totalSize: number
 }
 
-export interface AudioEventPayload {
-  encoded?: string;
-  buffer?: ArrayBuffer;
-  fileUri: string;
-  lastEmittedSize: number;
-  position: number;
-  deltaSize: number;
-  totalSize: number;
-  mimeType: string;
-  streamUuid: string;
+export type EncodingType = 'pcm_32bit' | 'pcm_16bit' | 'pcm_8bit'
+export type SampleRate = 16000 | 44100 | 48000
+export type BitDepth = 8 | 16 | 32
+
+export interface Chunk {
+    text: string
+    timestamp: [number, number | null]
 }
 
-export type EncodingType = "pcm_32bit" | "pcm_16bit" | "pcm_8bit";
-export type SampleRate = 16000 | 44100 | 48000;
-export type BitDepth = 8 | 16 | 32;
+export interface TranscriberData {
+    isBusy: boolean
+    text: string
+    chunks: Chunk[]
+}
 
-export interface AudioRecordingResult {
-  fileUri: string;
-  webAudioUri?: string;
-  durationMs: number;
-  size: number;
-  mimeType: string;
-  channels: number;
-  bitDepth: BitDepth;
-  sampleRate: SampleRate;
+export interface AudioRecording {
+    fileUri: string
+    filename: string
+    durationMs: number
+    size: number
+    mimeType: string
+    channels: number
+    bitDepth: BitDepth
+    sampleRate: SampleRate
+    transcripts?: TranscriberData[]
+    wavPCMData?: Float32Array // Full PCM data for the recording in WAV format (only on web, for native use the fileUri)
+    analysisData?: AudioAnalysis // Analysis data for the recording depending on enableProcessing flag
 }
 
 export interface StartRecordingResult {
-  fileUri: string;
-  mimeType: string;
-  channels?: number;
-  bitDepth?: BitDepth;
-  sampleRate?: SampleRate;
+    fileUri: string
+    mimeType: string
+    channels?: number
+    bitDepth?: BitDepth
+    sampleRate?: SampleRate
 }
 
 export interface RecordingConfig {
-  sampleRate?: SampleRate; // Sample rate for recording
-  channels?: 1 | 2; // 1 or 2 (MONO or STEREO)
-  encoding?: EncodingType; // Encoding type for the recording
-  interval?: number; // Interval in milliseconds at which to emit recording data
+    sampleRate?: SampleRate // Sample rate for recording
+    channels?: 1 | 2 // 1 or 2 (MONO or STEREO)
+    encoding?: EncodingType // Encoding type for the recording
+    interval?: number // Interval in milliseconds at which to emit recording data
 
-  // Optional parameters for audio processing
-  enableProcessing?: boolean; // Boolean to enable/disable audio processing (default is false)
-  pointsPerSecond?: number; // Number of data points to extract per second of audio (default is 1000)
-  algorithm?: string; // Algorithm to use for extraction (default is "rms")
-  features?: AudioFeaturesOptions; // Feature options to extract (default is empty)
+    // Optional parameters for audio processing
+    enableProcessing?: boolean // Boolean to enable/disable audio processing (default is false)
+    pointsPerSecond?: number // Number of data points to extract per second of audio (default is 1000)
+    algorithm?: AmplitudeAlgorithm // Algorithm to use for amplitude computation (default is "rms")
+    features?: AudioFeaturesOptions // Feature options to extract (default is empty)
 
-  // Optional paramters from web
-  onAudioStream?: (_: AudioDataEvent) => Promise<void>; // Callback function to handle audio stream
-  onProcessingResult?: (_: AudioAnalysisData) => Promise<void>; // Callback function to handle processing results
+    onAudioStream?: (_: AudioDataEvent) => Promise<void> // Callback function to handle audio stream
+    onAudioAnalysis?: (_: AudioAnalysisEvent) => Promise<void> // Callback function to handle audio features extraction results
 }