@drawdream/livespeech 0.1.0 → 0.1.2

package/README.md

@@ -26,12 +26,12 @@ const client = new LiveSpeechClient({
 });
 
 // Handle events
-client.setTranscriptHandler((text, isFinal) => {
-  console.log(`Transcript: ${text} (final: ${isFinal})`);
+client.setUserTranscriptHandler((text) => {
+  console.log(`You said: ${text}`);
 });
 
-client.setResponseHandler((text, isFinal) => {
-  console.log(`AI Response: ${text}`);
+client.setTranscriptHandler((text, isFinal) => {
+  console.log(`AI Transcript: ${text}`);
 });
 
 client.setAudioHandler((audioData) => {
@@ -42,10 +42,44 @@ client.setAudioHandler((audioData) => {
 await client.connect();
 await client.startSession({
   prePrompt: 'You are a helpful assistant.',
+  // pipelineMode: 'live' is the default
+});
+
+// Start streaming and send audio
+client.audioStart();
+client.sendAudioChunk(audioBuffer);
+```
+
+## Pipeline Modes
+
+The SDK supports two pipeline modes for audio processing:
+
+### Live Mode (Default)
+
+Uses Gemini 2.5 Flash Live API for end-to-end audio conversation. This provides:
+- **Lower latency** - Direct audio-to-audio processing
+- **Natural conversation** - Built-in voice activity detection and turn-taking
+- **Real-time transcription** - Both user and AI speech are transcribed
+
+```typescript
+await client.startSession({
+  prePrompt: 'You are a helpful assistant.',
+  pipelineMode: 'live',  // Default, can be omitted
 });
+```
+
+### Composed Mode
 
-// Send audio
-client.sendAudio(audioBuffer);
+Uses separate STT + LLM + TTS services for more customization:
+- **More control** - Separate services for each step
+- **Custom voices** - Use different TTS voices
+- **Text responses** - Access to intermediate text responses
+
+```typescript
+await client.startSession({
+  prePrompt: 'You are a helpful assistant.',
+  pipelineMode: 'composed',
+});
 ```
 
 ## API Reference
@@ -87,8 +121,9 @@ The SDK provides built-in region support, so you don't need to remember endpoint
 
 ```typescript
 // Simple handlers
-client.setTranscriptHandler((text, isFinal) => {});
-client.setResponseHandler((text, isFinal) => {});
+client.setUserTranscriptHandler((text) => {}); // User's speech transcription
+client.setTranscriptHandler((text, isFinal) => {}); // AI's speech transcription (live mode)
+client.setResponseHandler((text, isFinal) => {}); // AI text response (composed mode)
 client.setAudioHandler((audioData) => {});
 client.setErrorHandler((error) => {});
 
@@ -97,24 +132,23 @@ client.on('connected', (event) => {});
 client.on('disconnected', (event) => {});
 client.on('sessionStarted', (event) => {});
 client.on('sessionEnded', (event) => {});
-client.on('transcript', (event) => {});
-client.on('response', (event) => {});
+client.on('userTranscript', (event) => {}); // User's speech transcription
+client.on('transcript', (event) => {}); // AI's speech transcription
+client.on('response', (event) => {}); // AI text response
 client.on('audio', (event) => {});
 client.on('error', (event) => {});
 client.on('reconnecting', (event) => {});
+client.on('ready', (event) => {}); // Gemini Live ready (live mode)
+client.on('turnComplete', (event) => {}); // AI finished speaking (live mode)
 ```
 
 ### SessionConfig
 
 | Option | Type | Default | Description |
 |--------|------|---------|-------------|
-| `prePrompt` | `string` | **required** | System prompt for the AI |
-| `voiceId` | `string` | `'en-US-Standard-A'` | TTS voice ID |
-| `languageCode` | `string` | `'en-US'` | Language for STT |
-| `inputFormat` | `AudioFormat` | `'pcm16'` | Input audio format |
-| `outputFormat` | `AudioFormat` | `'pcm16'` | Output audio format |
-| `sampleRate` | `number` | `16000` | Sample rate in Hz |
-| `metadata` | `Record<string,string>` | `{}` | Custom metadata |
+| `prePrompt` | `string` | - | System prompt for the AI |
+| `language` | `string` | `'en-US'` | Language code for speech (e.g., "ko-KR") |
+| `pipelineMode` | `'live' \| 'composed'` | `'live'` | Audio processing mode |
 
 ## Audio Utilities
 

package/dist/index.d.mts

@@ -21,6 +21,12 @@ declare function getEndpointForRegion(region: Region): string;
  */
 declare function isValidRegion(value: string): value is Region;
 
+/**
+ * Pipeline mode for audio processing
+ * - 'live': Uses Gemini Live API for end-to-end audio conversation (default)
+ * - 'composed': Uses separate STT + LLM + TTS services
+ */
+type PipelineMode = 'live' | 'composed';
 /**
  * Configuration options for the LiveSpeech client
  *
@@ -74,41 +80,20 @@ interface SessionConfig {
     /**
      * System prompt for the AI assistant
      */
-    prePrompt: string;
-    /**
-     * Voice ID for text-to-speech output
-     * @default 'en-US-Standard-A'
-     */
-    voiceId?: string;
-    /**
-     * Language code for speech recognition
-     * @default 'en-US'
-     */
-    languageCode?: string;
-    /**
-     * Audio encoding format for input
-     * @default 'pcm16'
-     */
-    inputFormat?: AudioFormat;
-    /**
-     * Audio encoding format for output
-     * @default 'pcm16'
-     */
-    outputFormat?: AudioFormat;
+    prePrompt?: string;
     /**
-     * Sample rate for audio in Hz
-     * @default 16000
+     * Language code for speech recognition (e.g., "en-US", "ko-KR")
+     * @default "en-US"
      */
-    sampleRate?: number;
+    language?: string;
     /**
-     * Custom metadata to attach to the session
+     * Pipeline mode for audio processing
+     * - 'live': Uses Gemini Live API for end-to-end audio conversation (default, lower latency)
+     * - 'composed': Uses separate STT + LLM + TTS services (more customizable)
+     * @default "live"
      */
-    metadata?: Record<string, string>;
+    pipelineMode?: PipelineMode;
 }
-/**
- * Supported audio formats
- */
-type AudioFormat = 'pcm16' | 'opus' | 'wav';
 /**
  * Internal resolved configuration with defaults applied
  */
@@ -125,7 +110,7 @@ interface ResolvedConfig {
 /**
  * Event types emitted by the LiveSpeech client
  */
-type LiveSpeechEventType = 'connected' | 'disconnected' | 'sessionStarted' | 'sessionEnded' | 'transcript' | 'response' | 'audio' | 'error' | 'reconnecting';
+type LiveSpeechEventType = 'connected' | 'disconnected' | 'reconnecting' | 'sessionStarted' | 'sessionEnded' | 'ready' | 'userTranscript' | 'response' | 'audio' | 'turnComplete' | 'error';
 /**
  * Event payload for 'connected' event
  */
@@ -164,13 +149,19 @@ interface SessionEndedEvent {
     timestamp: string;
 }
 /**
- * Event payload for 'transcript' event
+ * Event payload for 'ready' event
+ */
+interface ReadyEvent {
+    type: 'ready';
+    timestamp: string;
+}
+/**
+ * Event payload for 'userTranscript' event
+ * User's speech transcription
  */
-interface TranscriptEvent {
-    type: 'transcript';
+interface UserTranscriptEvent {
+    type: 'userTranscript';
     text: string;
-    isFinal: boolean;
-    confidence?: number;
     timestamp: string;
 }
 /**
@@ -205,7 +196,7 @@ interface ErrorEvent {
 /**
  * Error codes
  */
-type ErrorCode = 'connection_failed' | 'connection_timeout' | 'authentication_failed' | 'session_error' | 'audio_error' | 'stt_error' | 'llm_error' | 'tts_error' | 'rate_limit' | 'internal_error' | 'invalid_message';
+type ErrorCode = 'connection_failed' | 'connection_timeout' | 'authentication_failed' | 'session_error' | 'audio_error' | 'streaming_error' | 'stt_error' | 'llm_error' | 'tts_error' | 'rate_limit' | 'internal_error' | 'invalid_message';
 /**
  * Event payload for 'reconnecting' event
  */
@@ -216,14 +207,22 @@ interface ReconnectingEvent {
     delay: number;
     timestamp: string;
 }
+/**
+ * Event payload for 'turnComplete' event (both modes)
+ * Indicates the AI has finished its response turn
+ */
+interface TurnCompleteEvent {
+    type: 'turnComplete';
+    timestamp: string;
+}
 /**
  * Union type of all event payloads
  */
-type LiveSpeechEvent = ConnectedEvent | DisconnectedEvent | SessionStartedEvent | SessionEndedEvent | TranscriptEvent | ResponseEvent | AudioEvent | ErrorEvent | ReconnectingEvent;
+type LiveSpeechEvent = ConnectedEvent | DisconnectedEvent | ReconnectingEvent | SessionStartedEvent | SessionEndedEvent | ReadyEvent | UserTranscriptEvent | ResponseEvent | AudioEvent | TurnCompleteEvent | ErrorEvent;
 /**
  * Simplified event handlers for common use cases
  */
-type TranscriptHandler = (text: string, isFinal: boolean) => void;
+type UserTranscriptHandler = (text: string) => void;
 type ResponseHandler = (text: string, isFinal: boolean) => void;
 type AudioHandler = (data: Uint8Array) => void;
 type ErrorHandler = (error: ErrorEvent) => void;
@@ -231,30 +230,25 @@ type ErrorHandler = (error: ErrorEvent) => void;
 /**
  * WebSocket message types sent from client to server
  */
-type ClientMessageType = 'startSession' | 'endSession' | 'audio' | 'ping';
+type ClientMessageType = 'startSession' | 'endSession' | 'audioStart' | 'audioChunk' | 'audioEnd' | 'ping';
 /**
  * WebSocket message types received from server
  */
-type ServerMessageType = 'connected' | 'sessionStarted' | 'sessionEnded' | 'transcript' | 'response' | 'audio' | 'error' | 'pong';
+type ServerMessageType = 'sessionStarted' | 'sessionEnded' | 'ready' | 'userTranscript' | 'response' | 'audio' | 'turnComplete' | 'error' | 'pong';
 /**
  * Base interface for client messages
  */
 interface BaseClientMessage {
     action: ClientMessageType;
-    requestId?: string;
 }
 /**
  * Start session message
  */
 interface StartSessionMessage extends BaseClientMessage {
     action: 'startSession';
-    prePrompt: string;
-    voiceId?: string;
-    languageCode?: string;
-    inputFormat?: string;
-    outputFormat?: string;
-    sampleRate?: number;
-    metadata?: Record<string, string>;
+    prePrompt?: string;
+    language?: string;
+    pipelineMode?: 'live' | 'composed';
 }
 /**
  * End session message
@@ -263,14 +257,23 @@ interface EndSessionMessage extends BaseClientMessage {
     action: 'endSession';
 }
 /**
- * Audio data message
+ * Audio start message - begin streaming session
+ */
+interface AudioStartMessage extends BaseClientMessage {
+    action: 'audioStart';
+}
+/**
+ * Audio chunk message - send audio data
  */
-interface AudioMessage extends BaseClientMessage {
-    action: 'audio';
+interface AudioChunkMessage extends BaseClientMessage {
+    action: 'audioChunk';
     data: string;
-    format?: string;
-    sampleRate?: number;
-    isFinal?: boolean;
+}
+/**
+ * Audio end message - end streaming session
+ */
+interface AudioEndMessage extends BaseClientMessage {
+    action: 'audioEnd';
 }
 /**
  * Ping message for keep-alive
@@ -281,22 +284,14 @@ interface PingMessage extends BaseClientMessage {
 /**
  * Union type of all client messages
  */
-type ClientMessage = StartSessionMessage | EndSessionMessage | AudioMessage | PingMessage;
+type ClientMessage = StartSessionMessage | EndSessionMessage | AudioStartMessage | AudioChunkMessage | AudioEndMessage | PingMessage;
 /**
  * Base interface for server messages
  */
 interface BaseServerMessage {
     type: ServerMessageType;
-    requestId?: string;
     timestamp: string;
 }
-/**
- * Connected message from server
- */
-interface ServerConnectedMessage extends BaseServerMessage {
-    type: 'connected';
-    connectionId: string;
-}
 /**
  * Session started message from server
  */
@@ -312,13 +307,11 @@ interface ServerSessionEndedMessage extends BaseServerMessage {
     sessionId: string;
 }
 /**
- * Transcript message from server
+ * User transcript message from server (user's speech transcription)
  */
-interface ServerTranscriptMessage extends BaseServerMessage {
-    type: 'transcript';
+interface ServerUserTranscriptMessage extends BaseServerMessage {
+    type: 'userTranscript';
     text: string;
-    isFinal: boolean;
-    confidence?: number;
 }
 /**
  * Response message from server
@@ -344,7 +337,6 @@ interface ServerErrorMessage extends BaseServerMessage {
     type: 'error';
     code: string;
     message: string;
-    details?: unknown;
 }
 /**
  * Pong message from server
@@ -352,10 +344,24 @@ interface ServerErrorMessage extends BaseServerMessage {
 interface ServerPongMessage extends BaseServerMessage {
     type: 'pong';
 }
+/**
+ * Turn complete message from server
+ * Indicates the AI has finished its response turn
+ */
+interface ServerTurnCompleteMessage extends BaseServerMessage {
+    type: 'turnComplete';
+}
+/**
+ * Ready message from server
+ * Indicates the Gemini Live session is ready for audio input
+ */
+interface ServerReadyMessage extends BaseServerMessage {
+    type: 'ready';
+}
 /**
  * Union type of all server messages
  */
-type ServerMessage = ServerConnectedMessage | ServerSessionStartedMessage | ServerSessionEndedMessage | ServerTranscriptMessage | ServerResponseMessage | ServerAudioMessage | ServerErrorMessage | ServerPongMessage;
+type ServerMessage = ServerSessionStartedMessage | ServerSessionEndedMessage | ServerReadyMessage | ServerUserTranscriptMessage | ServerResponseMessage | ServerAudioMessage | ServerTurnCompleteMessage | ServerErrorMessage | ServerPongMessage;
 
 /**
  * Connection state
@@ -368,13 +374,15 @@ type ConnectionState = 'disconnected' | 'connecting' | 'connected' | 'reconnecti
 type LiveSpeechEventMap = {
     connected: ConnectedEvent;
     disconnected: DisconnectedEvent;
+    reconnecting: ReconnectingEvent;
     sessionStarted: SessionStartedEvent;
     sessionEnded: SessionEndedEvent;
-    transcript: TranscriptEvent;
+    ready: ReadyEvent;
+    userTranscript: UserTranscriptEvent;
     response: ResponseEvent;
     audio: AudioEvent;
+    turnComplete: TurnCompleteEvent;
     error: ErrorEvent;
-    reconnecting: ReconnectingEvent;
 };
 /**
  * LiveSpeech client for real-time speech-to-speech AI conversations
@@ -385,9 +393,9 @@ declare class LiveSpeechClient {
     private readonly audioEncoder;
     private readonly logger;
     private sessionId;
-    private sessionConfig;
+    private isStreaming;
     private readonly eventListeners;
-    private transcriptHandler;
+    private userTranscriptHandler;
     private responseHandler;
     private audioHandler;
     private errorHandler;
@@ -412,6 +420,10 @@ declare class LiveSpeechClient {
      * Check if session is active
      */
     get hasActiveSession(): boolean;
+    /**
+     * Check if audio streaming is active
+     */
+    get isAudioStreaming(): boolean;
     /**
      * Connect to the server
      */
@@ -423,18 +435,23 @@ declare class LiveSpeechClient {
     /**
      * Start a new session
      */
-    startSession(config: SessionConfig): Promise<string>;
+    startSession(config?: SessionConfig): Promise<string>;
     /**
      * End the current session
      */
     endSession(): Promise<void>;
     /**
-     * Send audio data
+     * Start audio streaming session
+     */
+    audioStart(): void;
+    /**
+     * Send audio chunk (PCM16 base64 encoded)
+     */
+    sendAudioChunk(data: Uint8Array): void;
+    /**
+     * End audio streaming session
      */
-    sendAudio(data: Uint8Array, options?: {
-        format?: AudioFormat;
-        isFinal?: boolean;
-    }): void;
+    audioEnd(): void;
     /**
      * Add event listener
      */
@@ -444,17 +461,17 @@ declare class LiveSpeechClient {
      */
     off<K extends keyof LiveSpeechEventMap>(event: K, listener: (event: LiveSpeechEventMap[K]) => void): void;
     /**
-     * Set transcript handler (simplified)
-     */
-    setTranscriptHandler(handler: TranscriptHandler): void;
-    /**
-     * Set response handler (simplified)
+     * Set response handler
      */
     setResponseHandler(handler: ResponseHandler): void;
     /**
      * Set audio handler (simplified)
      */
     setAudioHandler(handler: AudioHandler): void;
+    /**
+     * Set user transcript handler
+     */
+    setUserTranscriptHandler(handler: UserTranscriptHandler): void;
     /**
      * Set error handler (simplified)
      */
@@ -467,6 +484,10 @@ declare class LiveSpeechClient {
     private handleMessage;
 }
 
+/**
+ * Audio format type
+ */
+type AudioFormat = 'pcm16' | 'opus' | 'wav';
 /**
  * Audio encoder options
  */
@@ -552,4 +573,4 @@ declare class AudioEncoder {
     wrapWav(data: Uint8Array): Uint8Array;
 }
 
-export { AudioEncoder, type AudioEncoderOptions, type AudioEvent, type AudioFormat, type AudioHandler, type ClientMessage, type ClientMessageType, type ConnectedEvent, type ConnectionState, type DisconnectReason, type DisconnectedEvent, type ErrorCode, type ErrorEvent, type ErrorHandler, LiveSpeechClient, type LiveSpeechConfig, type LiveSpeechEvent, type LiveSpeechEventMap, type LiveSpeechEventType, type ReconnectingEvent, Region, Region as RegionType, type ResolvedConfig, type ResponseEvent, type ResponseHandler, type ServerMessage, type ServerMessageType, type SessionConfig, type SessionEndedEvent, type SessionStartedEvent, type TranscriptEvent, type TranscriptHandler, createWavHeader, decodeBase64ToAudio, encodeAudioToBase64, extractPcmFromWav, float32ToInt16, getEndpointForRegion, int16ToFloat32, int16ToUint8, isValidRegion, uint8ToInt16, wrapPcmInWav };
+export { AudioEncoder, type AudioEncoderOptions, type AudioEvent, type AudioHandler, type ClientMessage, type ClientMessageType, type ConnectedEvent, type ConnectionState, type DisconnectReason, type DisconnectedEvent, type ErrorCode, type ErrorEvent, type ErrorHandler, LiveSpeechClient, type LiveSpeechConfig, type LiveSpeechEvent, type LiveSpeechEventMap, type LiveSpeechEventType, type PipelineMode, type ReadyEvent, type ReconnectingEvent, Region, Region as RegionType, type ResolvedConfig, type ResponseEvent, type ResponseHandler, type ServerMessage, type ServerMessageType, type SessionConfig, type SessionEndedEvent, type SessionStartedEvent, type TurnCompleteEvent, type UserTranscriptEvent, type UserTranscriptHandler, createWavHeader, decodeBase64ToAudio, encodeAudioToBase64, extractPcmFromWav, float32ToInt16, getEndpointForRegion, int16ToFloat32, int16ToUint8, isValidRegion, uint8ToInt16, wrapPcmInWav };