@lokutor/sdk 1.1.2 → 1.1.8

package/dist/index.d.mts

@@ -62,6 +62,41 @@ interface SynthesizeOptions {
     steps?: number;
     visemes?: boolean;
 }
+/**
+ * Browser audio configuration options
+ */
+interface BrowserAudioOptions {
+    inputSampleRate?: number;
+    outputSampleRate?: number;
+    autoGainControl?: boolean;
+    echoCancellation?: boolean;
+    noiseSuppression?: boolean;
+    analyserEnabled?: boolean;
+    onInputError?: (error: Error) => void;
+}
+/**
+ * Voice agent conversation options
+ */
+interface VoiceAgentOptions {
+    prompt?: string;
+    voice?: VoiceStyle;
+    language?: Language;
+    serverUrl?: string;
+    visemes?: boolean;
+    onTranscription?: (text: string, isUser: boolean) => void;
+    onVisemes?: (visemes: Viseme[]) => void;
+    onStatusChange?: (status: string) => void;
+    onError?: (err: any) => void;
+}
+/**
+ * Viseme data for lip-sync animation
+ * Format: {"v": index, "c": character, "t": timestamp}
+ */
+interface Viseme {
+    v: number;
+    c: string;
+    t: number;
+}
 
 /**
  * Main client for Lokutor Voice Agent SDK
@@ -77,14 +112,19 @@ declare class VoiceAgentClient {
     private onTranscription?;
     private onResponse?;
     private onAudioCallback?;
+    private onVisemesCallback?;
     private onStatus?;
     private onError?;
     private isConnected;
     private messages;
+    private visemeListeners;
+    private wantVisemes;
     constructor(config: LokutorConfig & {
         prompt: string;
         voice?: VoiceStyle;
         language?: Language;
+        visemes?: boolean;
+        onVisemes?: (visemes: Viseme[]) => void;
     });
     /**
      * Connect to the Lokutor Voice Agent server
@@ -110,6 +150,7 @@ declare class VoiceAgentClient {
     private audioListeners;
     private emit;
     onAudio(callback: (data: Uint8Array) => void): void;
+    onVisemes(callback: (visemes: Viseme[]) => void): void;
     /**
      * Disconnect from the server
      */
@@ -171,4 +212,193 @@ declare function simpleTTS(options: SynthesizeOptions & {
     onAudio: (buf: Uint8Array) => void;
 }): Promise<void>;
 
-export { AUDIO_CONFIG, DEFAULT_URLS, Language, type LokutorConfig, type SynthesizeOptions, TTSClient, VoiceAgentClient, VoiceStyle, simpleConversation, simpleTTS };
+/**
+ * Audio utility functions for format conversion, resampling, and PCM processing
+ */
+/**
+ * Convert 16-bit PCM (Int16) to 32-bit Float
+ * @param int16Data Int16Array of PCM audio
+ * @returns Float32Array normalized to [-1, 1]
+ */
+declare function pcm16ToFloat32(int16Data: Int16Array): Float32Array;
+/**
+ * Convert 32-bit Float to 16-bit PCM (Int16)
+ * @param float32Data Float32Array normalized to [-1, 1]
+ * @returns Int16Array of PCM audio
+ */
+declare function float32ToPcm16(float32Data: Float32Array): Int16Array;
+/**
+ * Resample audio data from one sample rate to another using linear interpolation
+ * @param input Float32Array of input audio
+ * @param inputRate Original sample rate in Hz
+ * @param outputRate Target sample rate in Hz
+ * @returns Float32Array of resampled audio
+ */
+declare function resample(input: Float32Array, inputRate: number, outputRate: number): Float32Array;
+/**
+ * Apply a simple low-pass filter for anti-aliasing during downsampling
+ * @param data Float32Array of audio
+ * @param cutoffFreq Cutoff frequency in Hz
+ * @param sampleRate Sample rate in Hz
+ * @returns Filtered Float32Array
+ */
+declare function applyLowPassFilter(data: Float32Array, cutoffFreq: number, sampleRate: number): Float32Array;
+/**
+ * Resample audio with anti-aliasing low-pass filter
+ * Best used when downsampling to prevent aliasing artifacts
+ * @param input Float32Array of input audio
+ * @param inputRate Original sample rate in Hz
+ * @param outputRate Target sample rate in Hz
+ * @returns Float32Array of resampled and filtered audio
+ */
+declare function resampleWithAntiAliasing(input: Float32Array, inputRate: number, outputRate: number): Float32Array;
+/**
+ * Convert raw audio samples to Uint8Array (bytes)
+ * @param data Int16Array of PCM audio
+ * @returns Uint8Array containing PCM bytes
+ */
+declare function pcm16ToBytes(data: Int16Array): Uint8Array;
+/**
+ * Convert bytes to Int16Array
+ * @param bytes Uint8Array of PCM bytes
+ * @returns Int16Array of PCM audio
+ */
+declare function bytesToPcm16(bytes: Uint8Array): Int16Array;
+/**
+ * Normalize audio amplitude to prevent clipping
+ * @param data Float32Array of audio
+ * @param targetPeak Peak level to normalize to (0-1)
+ * @returns Normalized Float32Array
+ */
+declare function normalizeAudio(data: Float32Array, targetPeak?: number): Float32Array;
+/**
+ * Calculate RMS (Root Mean Square) amplitude
+ * @param data Float32Array or Uint8Array of audio
+ * @returns RMS value (0-1 for normalized float, 0-255 for byte data)
+ */
+declare function calculateRMS(data: Float32Array | Uint8Array): number;
+/**
+ * Create a resample function factory for streaming audio
+ * Useful for processing audio in chunks
+ */
+declare class StreamResampler {
+    private inputBuffer;
+    private inputRate;
+    private outputRate;
+    constructor(inputRate: number, outputRate: number);
+    /**
+     * Process a chunk of audio and return resampled data
+     * @param inputChunk Float32Array chunk to process
+     * @param flush If true, output remaining buffered samples
+     * @returns Resampled Float32Array (may be empty if more data needed)
+     */
+    process(inputChunk: Float32Array, flush?: boolean): Float32Array;
+    reset(): void;
+}
+
+/**
+ * Configuration for browser audio handling
+ */
+interface BrowserAudioConfig {
+    inputSampleRate?: number;
+    outputSampleRate?: number;
+    autoGainControl?: boolean;
+    echoCancellation?: boolean;
+    noiseSuppression?: boolean;
+    onInputError?: (error: Error) => void;
+}
+/**
+ * Analyser configuration for audio visualization
+ */
+interface AnalyserConfig {
+    enabled?: boolean;
+    fftSize?: number;
+}
+/**
+ * Browser-based audio manager for Web Audio API operations
+ * Handles microphone input, speaker output, and visualization
+ */
+declare class BrowserAudioManager {
+    private audioContext;
+    private mediaStreamAudioSourceNode;
+    private scriptProcessor;
+    private analyserNode;
+    private mediaStream;
+    private nextPlaybackTime;
+    private activeSources;
+    private playbackQueue;
+    private inputSampleRate;
+    private outputSampleRate;
+    private autoGainControl;
+    private echoCancellation;
+    private noiseSuppression;
+    private onAudioInput?;
+    private onInputError?;
+    private isMuted;
+    private isListening;
+    constructor(config?: BrowserAudioConfig);
+    /**
+     * Initialize the AudioContext and analyser
+     */
+    init(analyserConfig?: AnalyserConfig): Promise<void>;
+    /**
+     * Start capturing audio from the microphone
+     */
+    startMicrophone(onAudioInput: (pcm16Data: Uint8Array) => void): Promise<void>;
+    /**
+     * Internal method to process microphone audio data
+     */
+    private _processAudioInput;
+    /**
+     * Stop capturing microphone input
+     */
+    stopMicrophone(): void;
+    /**
+     * Play back audio received from the server
+     * @param pcm16Data Int16 PCM audio data at SPEAKER_SAMPLE_RATE
+     */
+    playAudio(pcm16Data: Uint8Array): void;
+    /**
+     * Internal method to schedule and play audio with sample-accurate timing
+     */
+    private _schedulePlayback;
+    /**
+     * Stop all currently playing audio and clear the queue
+     */
+    stopPlayback(): void;
+    /**
+     * Toggle mute state
+     */
+    setMuted(muted: boolean): void;
+    /**
+     * Get current mute state
+     */
+    isMicMuted(): boolean;
+    /**
+     * Get current amplitude from analyser (for visualization)
+     * Returns value between 0 and 1
+     */
+    getAmplitude(): number;
+    /**
+     * Get frequency data from analyser for visualization
+     */
+    getFrequencyData(): Uint8Array;
+    /**
+     * Get time-domain data from analyser for waveform visualization
+     */
+    getWaveformData(): Uint8Array;
+    /**
+     * Cleanup and close AudioContext
+     */
+    cleanup(): void;
+    /**
+     * Get current audio context state
+     */
+    getState(): 'running' | 'suspended' | 'closed' | 'interrupted' | null;
+    /**
+     * Check if microphone is currently listening
+     */
+    isRecording(): boolean;
+}
+
+export { AUDIO_CONFIG, type AnalyserConfig, type BrowserAudioConfig, BrowserAudioManager, type BrowserAudioOptions, DEFAULT_URLS, Language, type LokutorConfig, StreamResampler, type SynthesizeOptions, TTSClient, type Viseme, VoiceAgentClient, type VoiceAgentOptions, VoiceStyle, applyLowPassFilter, bytesToPcm16, calculateRMS, float32ToPcm16, normalizeAudio, pcm16ToBytes, pcm16ToFloat32, resample, resampleWithAntiAliasing, simpleConversation, simpleTTS };

package/dist/index.d.ts

@@ -62,6 +62,41 @@ interface SynthesizeOptions {
     steps?: number;
     visemes?: boolean;
 }
+/**
+ * Browser audio configuration options
+ */
+interface BrowserAudioOptions {
+    inputSampleRate?: number;
+    outputSampleRate?: number;
+    autoGainControl?: boolean;
+    echoCancellation?: boolean;
+    noiseSuppression?: boolean;
+    analyserEnabled?: boolean;
+    onInputError?: (error: Error) => void;
+}
+/**
+ * Voice agent conversation options
+ */
+interface VoiceAgentOptions {
+    prompt?: string;
+    voice?: VoiceStyle;
+    language?: Language;
+    serverUrl?: string;
+    visemes?: boolean;
+    onTranscription?: (text: string, isUser: boolean) => void;
+    onVisemes?: (visemes: Viseme[]) => void;
+    onStatusChange?: (status: string) => void;
+    onError?: (err: any) => void;
+}
+/**
+ * Viseme data for lip-sync animation
+ * Format: {"v": index, "c": character, "t": timestamp}
+ */
+interface Viseme {
+    v: number;
+    c: string;
+    t: number;
+}
 
 /**
  * Main client for Lokutor Voice Agent SDK
@@ -77,14 +112,19 @@ declare class VoiceAgentClient {
     private onTranscription?;
     private onResponse?;
     private onAudioCallback?;
+    private onVisemesCallback?;
     private onStatus?;
     private onError?;
     private isConnected;
     private messages;
+    private visemeListeners;
+    private wantVisemes;
     constructor(config: LokutorConfig & {
         prompt: string;
         voice?: VoiceStyle;
         language?: Language;
+        visemes?: boolean;
+        onVisemes?: (visemes: Viseme[]) => void;
     });
     /**
      * Connect to the Lokutor Voice Agent server
@@ -110,6 +150,7 @@ declare class VoiceAgentClient {
     private audioListeners;
     private emit;
     onAudio(callback: (data: Uint8Array) => void): void;
+    onVisemes(callback: (visemes: Viseme[]) => void): void;
     /**
      * Disconnect from the server
      */
@@ -171,4 +212,193 @@ declare function simpleTTS(options: SynthesizeOptions & {
     onAudio: (buf: Uint8Array) => void;
 }): Promise<void>;
 
-export { AUDIO_CONFIG, DEFAULT_URLS, Language, type LokutorConfig, type SynthesizeOptions, TTSClient, VoiceAgentClient, VoiceStyle, simpleConversation, simpleTTS };
+/**
+ * Audio utility functions for format conversion, resampling, and PCM processing
+ */
+/**
+ * Convert 16-bit PCM (Int16) to 32-bit Float
+ * @param int16Data Int16Array of PCM audio
+ * @returns Float32Array normalized to [-1, 1]
+ */
+declare function pcm16ToFloat32(int16Data: Int16Array): Float32Array;
+/**
+ * Convert 32-bit Float to 16-bit PCM (Int16)
+ * @param float32Data Float32Array normalized to [-1, 1]
+ * @returns Int16Array of PCM audio
+ */
+declare function float32ToPcm16(float32Data: Float32Array): Int16Array;
+/**
+ * Resample audio data from one sample rate to another using linear interpolation
+ * @param input Float32Array of input audio
+ * @param inputRate Original sample rate in Hz
+ * @param outputRate Target sample rate in Hz
+ * @returns Float32Array of resampled audio
+ */
+declare function resample(input: Float32Array, inputRate: number, outputRate: number): Float32Array;
+/**
+ * Apply a simple low-pass filter for anti-aliasing during downsampling
+ * @param data Float32Array of audio
+ * @param cutoffFreq Cutoff frequency in Hz
+ * @param sampleRate Sample rate in Hz
+ * @returns Filtered Float32Array
+ */
+declare function applyLowPassFilter(data: Float32Array, cutoffFreq: number, sampleRate: number): Float32Array;
+/**
+ * Resample audio with anti-aliasing low-pass filter
+ * Best used when downsampling to prevent aliasing artifacts
+ * @param input Float32Array of input audio
+ * @param inputRate Original sample rate in Hz
+ * @param outputRate Target sample rate in Hz
+ * @returns Float32Array of resampled and filtered audio
+ */
+declare function resampleWithAntiAliasing(input: Float32Array, inputRate: number, outputRate: number): Float32Array;
+/**
+ * Convert raw audio samples to Uint8Array (bytes)
+ * @param data Int16Array of PCM audio
+ * @returns Uint8Array containing PCM bytes
+ */
+declare function pcm16ToBytes(data: Int16Array): Uint8Array;
+/**
+ * Convert bytes to Int16Array
+ * @param bytes Uint8Array of PCM bytes
+ * @returns Int16Array of PCM audio
+ */
+declare function bytesToPcm16(bytes: Uint8Array): Int16Array;
+/**
+ * Normalize audio amplitude to prevent clipping
+ * @param data Float32Array of audio
+ * @param targetPeak Peak level to normalize to (0-1)
+ * @returns Normalized Float32Array
+ */
+declare function normalizeAudio(data: Float32Array, targetPeak?: number): Float32Array;
+/**
+ * Calculate RMS (Root Mean Square) amplitude
+ * @param data Float32Array or Uint8Array of audio
+ * @returns RMS value (0-1 for normalized float, 0-255 for byte data)
+ */
+declare function calculateRMS(data: Float32Array | Uint8Array): number;
+/**
+ * Create a resample function factory for streaming audio
+ * Useful for processing audio in chunks
+ */
+declare class StreamResampler {
+    private inputBuffer;
+    private inputRate;
+    private outputRate;
+    constructor(inputRate: number, outputRate: number);
+    /**
+     * Process a chunk of audio and return resampled data
+     * @param inputChunk Float32Array chunk to process
+     * @param flush If true, output remaining buffered samples
+     * @returns Resampled Float32Array (may be empty if more data needed)
+     */
+    process(inputChunk: Float32Array, flush?: boolean): Float32Array;
+    reset(): void;
+}
+
+/**
+ * Configuration for browser audio handling
+ */
+interface BrowserAudioConfig {
+    inputSampleRate?: number;
+    outputSampleRate?: number;
+    autoGainControl?: boolean;
+    echoCancellation?: boolean;
+    noiseSuppression?: boolean;
+    onInputError?: (error: Error) => void;
+}
+/**
+ * Analyser configuration for audio visualization
+ */
+interface AnalyserConfig {
+    enabled?: boolean;
+    fftSize?: number;
+}
+/**
+ * Browser-based audio manager for Web Audio API operations
+ * Handles microphone input, speaker output, and visualization
+ */
+declare class BrowserAudioManager {
+    private audioContext;
+    private mediaStreamAudioSourceNode;
+    private scriptProcessor;
+    private analyserNode;
+    private mediaStream;
+    private nextPlaybackTime;
+    private activeSources;
+    private playbackQueue;
+    private inputSampleRate;
+    private outputSampleRate;
+    private autoGainControl;
+    private echoCancellation;
+    private noiseSuppression;
+    private onAudioInput?;
+    private onInputError?;
+    private isMuted;
+    private isListening;
+    constructor(config?: BrowserAudioConfig);
+    /**
+     * Initialize the AudioContext and analyser
+     */
+    init(analyserConfig?: AnalyserConfig): Promise<void>;
+    /**
+     * Start capturing audio from the microphone
+     */
+    startMicrophone(onAudioInput: (pcm16Data: Uint8Array) => void): Promise<void>;
+    /**
+     * Internal method to process microphone audio data
+     */
+    private _processAudioInput;
+    /**
+     * Stop capturing microphone input
+     */
+    stopMicrophone(): void;
+    /**
+     * Play back audio received from the server
+     * @param pcm16Data Int16 PCM audio data at SPEAKER_SAMPLE_RATE
+     */
+    playAudio(pcm16Data: Uint8Array): void;
+    /**
+     * Internal method to schedule and play audio with sample-accurate timing
+     */
+    private _schedulePlayback;
+    /**
+     * Stop all currently playing audio and clear the queue
+     */
+    stopPlayback(): void;
+    /**
+     * Toggle mute state
+     */
+    setMuted(muted: boolean): void;
+    /**
+     * Get current mute state
+     */
+    isMicMuted(): boolean;
+    /**
+     * Get current amplitude from analyser (for visualization)
+     * Returns value between 0 and 1
+     */
+    getAmplitude(): number;
+    /**
+     * Get frequency data from analyser for visualization
+     */
+    getFrequencyData(): Uint8Array;
+    /**
+     * Get time-domain data from analyser for waveform visualization
+     */
+    getWaveformData(): Uint8Array;
+    /**
+     * Cleanup and close AudioContext
+     */
+    cleanup(): void;
+    /**
+     * Get current audio context state
+     */
+    getState(): 'running' | 'suspended' | 'closed' | 'interrupted' | null;
+    /**
+     * Check if microphone is currently listening
+     */
+    isRecording(): boolean;
+}
+
+export { AUDIO_CONFIG, type AnalyserConfig, type BrowserAudioConfig, BrowserAudioManager, type BrowserAudioOptions, DEFAULT_URLS, Language, type LokutorConfig, StreamResampler, type SynthesizeOptions, TTSClient, type Viseme, VoiceAgentClient, type VoiceAgentOptions, VoiceStyle, applyLowPassFilter, bytesToPcm16, calculateRMS, float32ToPcm16, normalizeAudio, pcm16ToBytes, pcm16ToFloat32, resample, resampleWithAntiAliasing, simpleConversation, simpleTTS };