@drawdream/livespeech 0.1.0 → 0.1.2

package/dist/index.d.ts

@@ -21,6 +21,12 @@ declare function getEndpointForRegion(region: Region): string;
  */
 declare function isValidRegion(value: string): value is Region;
 
+/**
+ * Pipeline mode for audio processing
+ * - 'live': Uses Gemini Live API for end-to-end audio conversation (default)
+ * - 'composed': Uses separate STT + LLM + TTS services
+ */
+type PipelineMode = 'live' | 'composed';
 /**
  * Configuration options for the LiveSpeech client
  *
@@ -74,41 +80,20 @@ interface SessionConfig {
     /**
      * System prompt for the AI assistant
      */
-    prePrompt: string;
-    /**
-     * Voice ID for text-to-speech output
-     * @default 'en-US-Standard-A'
-     */
-    voiceId?: string;
-    /**
-     * Language code for speech recognition
-     * @default 'en-US'
-     */
-    languageCode?: string;
-    /**
-     * Audio encoding format for input
-     * @default 'pcm16'
-     */
-    inputFormat?: AudioFormat;
-    /**
-     * Audio encoding format for output
-     * @default 'pcm16'
-     */
-    outputFormat?: AudioFormat;
+    prePrompt?: string;
     /**
-     * Sample rate for audio in Hz
-     * @default 16000
+     * Language code for speech recognition (e.g., "en-US", "ko-KR")
+     * @default "en-US"
      */
-    sampleRate?: number;
+    language?: string;
     /**
-     * Custom metadata to attach to the session
+     * Pipeline mode for audio processing
+     * - 'live': Uses Gemini Live API for end-to-end audio conversation (default, lower latency)
+     * - 'composed': Uses separate STT + LLM + TTS services (more customizable)
+     * @default "live"
      */
-    metadata?: Record<string, string>;
+    pipelineMode?: PipelineMode;
 }
-/**
- * Supported audio formats
- */
-type AudioFormat = 'pcm16' | 'opus' | 'wav';
 /**
  * Internal resolved configuration with defaults applied
  */
@@ -125,7 +110,7 @@ interface ResolvedConfig {
 /**
  * Event types emitted by the LiveSpeech client
  */
-type LiveSpeechEventType = 'connected' | 'disconnected' | 'sessionStarted' | 'sessionEnded' | 'transcript' | 'response' | 'audio' | 'error' | 'reconnecting';
+type LiveSpeechEventType = 'connected' | 'disconnected' | 'reconnecting' | 'sessionStarted' | 'sessionEnded' | 'ready' | 'userTranscript' | 'response' | 'audio' | 'turnComplete' | 'error';
 /**
  * Event payload for 'connected' event
  */
@@ -164,13 +149,19 @@ interface SessionEndedEvent {
     timestamp: string;
 }
 /**
- * Event payload for 'transcript' event
+ * Event payload for 'ready' event
+ */
+interface ReadyEvent {
+    type: 'ready';
+    timestamp: string;
+}
+/**
+ * Event payload for 'userTranscript' event
+ * User's speech transcription
  */
-interface TranscriptEvent {
-    type: 'transcript';
+interface UserTranscriptEvent {
+    type: 'userTranscript';
     text: string;
-    isFinal: boolean;
-    confidence?: number;
     timestamp: string;
 }
 /**
@@ -205,7 +196,7 @@ interface ErrorEvent {
 /**
  * Error codes
  */
-type ErrorCode = 'connection_failed' | 'connection_timeout' | 'authentication_failed' | 'session_error' | 'audio_error' | 'stt_error' | 'llm_error' | 'tts_error' | 'rate_limit' | 'internal_error' | 'invalid_message';
+type ErrorCode = 'connection_failed' | 'connection_timeout' | 'authentication_failed' | 'session_error' | 'audio_error' | 'streaming_error' | 'stt_error' | 'llm_error' | 'tts_error' | 'rate_limit' | 'internal_error' | 'invalid_message';
 /**
  * Event payload for 'reconnecting' event
  */
@@ -216,14 +207,22 @@ interface ReconnectingEvent {
     delay: number;
     timestamp: string;
 }
+/**
+ * Event payload for 'turnComplete' event (both modes)
+ * Indicates the AI has finished its response turn
+ */
+interface TurnCompleteEvent {
+    type: 'turnComplete';
+    timestamp: string;
+}
 /**
  * Union type of all event payloads
  */
-type LiveSpeechEvent = ConnectedEvent | DisconnectedEvent | SessionStartedEvent | SessionEndedEvent | TranscriptEvent | ResponseEvent | AudioEvent | ErrorEvent | ReconnectingEvent;
+type LiveSpeechEvent = ConnectedEvent | DisconnectedEvent | ReconnectingEvent | SessionStartedEvent | SessionEndedEvent | ReadyEvent | UserTranscriptEvent | ResponseEvent | AudioEvent | TurnCompleteEvent | ErrorEvent;
 /**
  * Simplified event handlers for common use cases
  */
-type TranscriptHandler = (text: string, isFinal: boolean) => void;
+type UserTranscriptHandler = (text: string) => void;
 type ResponseHandler = (text: string, isFinal: boolean) => void;
 type AudioHandler = (data: Uint8Array) => void;
 type ErrorHandler = (error: ErrorEvent) => void;
@@ -231,30 +230,25 @@ type ErrorHandler = (error: ErrorEvent) => void;
 /**
  * WebSocket message types sent from client to server
  */
-type ClientMessageType = 'startSession' | 'endSession' | 'audio' | 'ping';
+type ClientMessageType = 'startSession' | 'endSession' | 'audioStart' | 'audioChunk' | 'audioEnd' | 'ping';
 /**
  * WebSocket message types received from server
  */
-type ServerMessageType = 'connected' | 'sessionStarted' | 'sessionEnded' | 'transcript' | 'response' | 'audio' | 'error' | 'pong';
+type ServerMessageType = 'sessionStarted' | 'sessionEnded' | 'ready' | 'userTranscript' | 'response' | 'audio' | 'turnComplete' | 'error' | 'pong';
 /**
  * Base interface for client messages
  */
 interface BaseClientMessage {
     action: ClientMessageType;
-    requestId?: string;
 }
 /**
  * Start session message
  */
 interface StartSessionMessage extends BaseClientMessage {
     action: 'startSession';
-    prePrompt: string;
-    voiceId?: string;
-    languageCode?: string;
-    inputFormat?: string;
-    outputFormat?: string;
-    sampleRate?: number;
-    metadata?: Record<string, string>;
+    prePrompt?: string;
+    language?: string;
+    pipelineMode?: 'live' | 'composed';
 }
 /**
  * End session message
@@ -263,14 +257,23 @@ interface EndSessionMessage extends BaseClientMessage {
     action: 'endSession';
 }
 /**
- * Audio data message
+ * Audio start message - begin streaming session
+ */
+interface AudioStartMessage extends BaseClientMessage {
+    action: 'audioStart';
+}
+/**
+ * Audio chunk message - send audio data
  */
-interface AudioMessage extends BaseClientMessage {
-    action: 'audio';
+interface AudioChunkMessage extends BaseClientMessage {
+    action: 'audioChunk';
     data: string;
-    format?: string;
-    sampleRate?: number;
-    isFinal?: boolean;
+}
+/**
+ * Audio end message - end streaming session
+ */
+interface AudioEndMessage extends BaseClientMessage {
+    action: 'audioEnd';
 }
 /**
  * Ping message for keep-alive
@@ -281,22 +284,14 @@ interface PingMessage extends BaseClientMessage {
 /**
  * Union type of all client messages
  */
-type ClientMessage = StartSessionMessage | EndSessionMessage | AudioMessage | PingMessage;
+type ClientMessage = StartSessionMessage | EndSessionMessage | AudioStartMessage | AudioChunkMessage | AudioEndMessage | PingMessage;
 /**
  * Base interface for server messages
  */
 interface BaseServerMessage {
     type: ServerMessageType;
-    requestId?: string;
     timestamp: string;
 }
-/**
- * Connected message from server
- */
-interface ServerConnectedMessage extends BaseServerMessage {
-    type: 'connected';
-    connectionId: string;
-}
 /**
  * Session started message from server
  */
@@ -312,13 +307,11 @@ interface ServerSessionEndedMessage extends BaseServerMessage {
     sessionId: string;
 }
 /**
- * Transcript message from server
+ * User transcript message from server (user's speech transcription)
  */
-interface ServerTranscriptMessage extends BaseServerMessage {
-    type: 'transcript';
+interface ServerUserTranscriptMessage extends BaseServerMessage {
+    type: 'userTranscript';
     text: string;
-    isFinal: boolean;
-    confidence?: number;
 }
 /**
  * Response message from server
@@ -344,7 +337,6 @@ interface ServerErrorMessage extends BaseServerMessage {
     type: 'error';
     code: string;
     message: string;
-    details?: unknown;
 }
 /**
  * Pong message from server
@@ -352,10 +344,24 @@ interface ServerErrorMessage extends BaseServerMessage {
 interface ServerPongMessage extends BaseServerMessage {
     type: 'pong';
 }
+/**
+ * Turn complete message from server
+ * Indicates the AI has finished its response turn
+ */
+interface ServerTurnCompleteMessage extends BaseServerMessage {
+    type: 'turnComplete';
+}
+/**
+ * Ready message from server
+ * Indicates the Gemini Live session is ready for audio input
+ */
+interface ServerReadyMessage extends BaseServerMessage {
+    type: 'ready';
+}
 /**
  * Union type of all server messages
  */
-type ServerMessage = ServerConnectedMessage | ServerSessionStartedMessage | ServerSessionEndedMessage | ServerTranscriptMessage | ServerResponseMessage | ServerAudioMessage | ServerErrorMessage | ServerPongMessage;
+type ServerMessage = ServerSessionStartedMessage | ServerSessionEndedMessage | ServerReadyMessage | ServerUserTranscriptMessage | ServerResponseMessage | ServerAudioMessage | ServerTurnCompleteMessage | ServerErrorMessage | ServerPongMessage;
 
 /**
  * Connection state
@@ -368,13 +374,15 @@ type ConnectionState = 'disconnected' | 'connecting' | 'connected' | 'reconnecti
 type LiveSpeechEventMap = {
     connected: ConnectedEvent;
     disconnected: DisconnectedEvent;
+    reconnecting: ReconnectingEvent;
     sessionStarted: SessionStartedEvent;
     sessionEnded: SessionEndedEvent;
-    transcript: TranscriptEvent;
+    ready: ReadyEvent;
+    userTranscript: UserTranscriptEvent;
     response: ResponseEvent;
     audio: AudioEvent;
+    turnComplete: TurnCompleteEvent;
     error: ErrorEvent;
-    reconnecting: ReconnectingEvent;
 };
 /**
  * LiveSpeech client for real-time speech-to-speech AI conversations
@@ -385,9 +393,9 @@ declare class LiveSpeechClient {
     private readonly audioEncoder;
     private readonly logger;
     private sessionId;
-    private sessionConfig;
+    private isStreaming;
     private readonly eventListeners;
-    private transcriptHandler;
+    private userTranscriptHandler;
     private responseHandler;
     private audioHandler;
     private errorHandler;
@@ -412,6 +420,10 @@ declare class LiveSpeechClient {
      * Check if session is active
      */
     get hasActiveSession(): boolean;
+    /**
+     * Check if audio streaming is active
+     */
+    get isAudioStreaming(): boolean;
     /**
      * Connect to the server
      */
@@ -423,18 +435,23 @@ declare class LiveSpeechClient {
     /**
      * Start a new session
      */
-    startSession(config: SessionConfig): Promise<string>;
+    startSession(config?: SessionConfig): Promise<string>;
     /**
      * End the current session
      */
     endSession(): Promise<void>;
     /**
-     * Send audio data
+     * Start audio streaming session
+     */
+    audioStart(): void;
+    /**
+     * Send audio chunk (PCM16 base64 encoded)
+     */
+    sendAudioChunk(data: Uint8Array): void;
+    /**
+     * End audio streaming session
      */
-    sendAudio(data: Uint8Array, options?: {
-        format?: AudioFormat;
-        isFinal?: boolean;
-    }): void;
+    audioEnd(): void;
     /**
      * Add event listener
      */
@@ -444,17 +461,17 @@ declare class LiveSpeechClient {
      */
     off<K extends keyof LiveSpeechEventMap>(event: K, listener: (event: LiveSpeechEventMap[K]) => void): void;
     /**
-     * Set transcript handler (simplified)
-     */
-    setTranscriptHandler(handler: TranscriptHandler): void;
-    /**
-     * Set response handler (simplified)
+     * Set response handler
      */
     setResponseHandler(handler: ResponseHandler): void;
     /**
      * Set audio handler (simplified)
      */
     setAudioHandler(handler: AudioHandler): void;
+    /**
+     * Set user transcript handler
+     */
+    setUserTranscriptHandler(handler: UserTranscriptHandler): void;
     /**
      * Set error handler (simplified)
      */
@@ -467,6 +484,10 @@ declare class LiveSpeechClient {
     private handleMessage;
 }
 
+/**
+ * Audio format type
+ */
+type AudioFormat = 'pcm16' | 'opus' | 'wav';
 /**
  * Audio encoder options
  */
@@ -552,4 +573,4 @@ declare class AudioEncoder {
     wrapWav(data: Uint8Array): Uint8Array;
 }
 
-export { AudioEncoder, type AudioEncoderOptions, type AudioEvent, type AudioFormat, type AudioHandler, type ClientMessage, type ClientMessageType, type ConnectedEvent, type ConnectionState, type DisconnectReason, type DisconnectedEvent, type ErrorCode, type ErrorEvent, type ErrorHandler, LiveSpeechClient, type LiveSpeechConfig, type LiveSpeechEvent, type LiveSpeechEventMap, type LiveSpeechEventType, type ReconnectingEvent, Region, Region as RegionType, type ResolvedConfig, type ResponseEvent, type ResponseHandler, type ServerMessage, type ServerMessageType, type SessionConfig, type SessionEndedEvent, type SessionStartedEvent, type TranscriptEvent, type TranscriptHandler, createWavHeader, decodeBase64ToAudio, encodeAudioToBase64, extractPcmFromWav, float32ToInt16, getEndpointForRegion, int16ToFloat32, int16ToUint8, isValidRegion, uint8ToInt16, wrapPcmInWav };
+export { AudioEncoder, type AudioEncoderOptions, type AudioEvent, type AudioHandler, type ClientMessage, type ClientMessageType, type ConnectedEvent, type ConnectionState, type DisconnectReason, type DisconnectedEvent, type ErrorCode, type ErrorEvent, type ErrorHandler, LiveSpeechClient, type LiveSpeechConfig, type LiveSpeechEvent, type LiveSpeechEventMap, type LiveSpeechEventType, type PipelineMode, type ReadyEvent, type ReconnectingEvent, Region, Region as RegionType, type ResolvedConfig, type ResponseEvent, type ResponseHandler, type ServerMessage, type ServerMessageType, type SessionConfig, type SessionEndedEvent, type SessionStartedEvent, type TurnCompleteEvent, type UserTranscriptEvent, type UserTranscriptHandler, createWavHeader, decodeBase64ToAudio, encodeAudioToBase64, extractPcmFromWav, float32ToInt16, getEndpointForRegion, int16ToFloat32, int16ToUint8, isValidRegion, uint8ToInt16, wrapPcmInWav };

package/dist/index.js

@@ -46,7 +46,7 @@ var Region = {
 };
 var REGION_ENDPOINTS = {
   "ap-northeast-2": "wss://talk.drawdream.co.kr",
-  "us-west-2": "wss://talk..drawdream.ca"
+  "us-west-2": "wss://talk.drawdream.ca"
   // Coming soon
 };
 function getEndpointForRegion(region) {
@@ -332,22 +332,13 @@ var WebSocketConnection = class {
   /**
    * Handle incoming message
    */
-  handleMessage(data, onFirstConnect) {
+  handleMessage(data) {
     const message = parseServerMessage(data);
     if (!message) {
       this.logger.warn("Invalid message received:", data);
       return;
     }
     this.logger.debug("Received message:", message.type);
-    if (message.type === "connected") {
-      this.connectionId = message.connectionId;
-      this.state = "connected";
-      this.retryController.reset();
-      this.startPingInterval();
-      this.events.onOpen?.(message.connectionId);
-      onFirstConnect?.();
-      return;
-    }
     if (message.type === "pong") {
       this.logger.debug("Pong received");
       return;
@@ -613,24 +604,17 @@ var CONFIG_DEFAULTS = {
   reconnectDelay: 1e3,
   debug: false
 };
-var SESSION_DEFAULTS = {
-  voiceId: "en-US-Standard-A",
-  languageCode: "en-US",
-  inputFormat: "pcm16",
-  outputFormat: "pcm16",
-  sampleRate: 16e3
-};
 var LiveSpeechClient = class {
   config;
   connection;
   audioEncoder;
   logger;
   sessionId = null;
-  sessionConfig = null;
+  isStreaming = false;
   // Event listeners using a simple map
   eventListeners = /* @__PURE__ */ new Map();
   // Simplified handlers
-  transcriptHandler = null;
+  userTranscriptHandler = null;
   responseHandler = null;
   audioHandler = null;
   errorHandler = null;
@@ -692,6 +676,12 @@ var LiveSpeechClient = class {
   get hasActiveSession() {
     return this.sessionId !== null;
   }
+  /**
+   * Check if audio streaming is active
+   */
+  get isAudioStreaming() {
+    return this.isStreaming;
+  }
   /**
    * Connect to the server
    */
@@ -705,7 +695,7 @@ var LiveSpeechClient = class {
   disconnect() {
     this.logger.info("Disconnecting...");
     this.sessionId = null;
-    this.sessionConfig = null;
+    this.isStreaming = false;
     this.connection.disconnect();
   }
   /**
@@ -718,16 +708,6 @@ var LiveSpeechClient = class {
     if (this.sessionId) {
       throw new Error("Session already active. Call endSession() first.");
     }
-    const resolvedConfig = {
-      prePrompt: config.prePrompt,
-      voiceId: config.voiceId ?? SESSION_DEFAULTS.voiceId,
-      languageCode: config.languageCode ?? SESSION_DEFAULTS.languageCode,
-      inputFormat: config.inputFormat ?? SESSION_DEFAULTS.inputFormat,
-      outputFormat: config.outputFormat ?? SESSION_DEFAULTS.outputFormat,
-      sampleRate: config.sampleRate ?? SESSION_DEFAULTS.sampleRate,
-      metadata: config.metadata ?? {}
-    };
-    this.sessionConfig = resolvedConfig;
     this.logger.info("Starting session...");
     return new Promise((resolve, reject) => {
       const onSessionStarted = (event) => {
@@ -744,16 +724,17 @@ var LiveSpeechClient = class {
       };
       this.on("sessionStarted", onSessionStarted);
       this.on("error", onError);
-      this.connection.send({
-        action: "startSession",
-        prePrompt: resolvedConfig.prePrompt,
-        voiceId: resolvedConfig.voiceId,
-        languageCode: resolvedConfig.languageCode,
-        inputFormat: resolvedConfig.inputFormat,
-        outputFormat: resolvedConfig.outputFormat,
-        sampleRate: resolvedConfig.sampleRate,
-        metadata: resolvedConfig.metadata
-      });
+      const startMessage = {
+        action: "startSession"
+      };
+      if (config?.prePrompt) {
+        startMessage.prePrompt = config.prePrompt;
+      }
+      if (config?.language) {
+        startMessage.language = config.language;
+      }
+      startMessage.pipelineMode = config?.pipelineMode ?? "live";
+      this.connection.send(startMessage);
     });
   }
   /**
@@ -765,6 +746,9 @@ var LiveSpeechClient = class {
       return;
     }
     this.logger.info("Ending session...");
+    if (this.isStreaming) {
+      this.audioEnd();
+    }
     return new Promise((resolve) => {
       const onSessionEnded = () => {
         this.off("sessionEnded", onSessionEnded);
@@ -775,28 +759,49 @@ var LiveSpeechClient = class {
     });
   }
   /**
-   * Send audio data
+   * Start audio streaming session
    */
-  sendAudio(data, options) {
+  audioStart() {
     if (!this.isConnected) {
       throw new Error("Not connected");
     }
     if (!this.sessionId) {
       throw new Error("No active session. Call startSession() first.");
     }
+    if (this.isStreaming) {
+      throw new Error("Already streaming. Call audioEnd() first.");
+    }
+    this.logger.info("Starting audio stream...");
+    this.connection.send({ action: "audioStart" });
+    this.isStreaming = true;
+  }
+  /**
+   * Send audio chunk (PCM16 base64 encoded)
+   */
+  sendAudioChunk(data) {
+    if (!this.isConnected) {
+      throw new Error("Not connected");
+    }
+    if (!this.isStreaming) {
+      throw new Error("Not streaming. Call audioStart() first.");
+    }
     const base64Data = this.audioEncoder.encode(data);
-    const format = options?.format ?? this.sessionConfig?.inputFormat ?? SESSION_DEFAULTS.inputFormat;
-    const sampleRate = this.sessionConfig?.sampleRate ?? SESSION_DEFAULTS.sampleRate;
-    const audioMessage = {
-      action: "audio",
-      data: base64Data,
-      format,
-      sampleRate
-    };
-    if (options?.isFinal !== void 0) {
-      audioMessage.isFinal = options.isFinal;
+    this.connection.send({
+      action: "audioChunk",
+      data: base64Data
+    });
+  }
+  /**
+   * End audio streaming session
+   */
+  audioEnd() {
+    if (!this.isStreaming) {
+      this.logger.warn("Not streaming");
+      return;
     }
-    this.connection.send(audioMessage);
+    this.logger.info("Ending audio stream...");
+    this.connection.send({ action: "audioEnd" });
+    this.isStreaming = false;
   }
   // ==================== Event System ====================
   /**
@@ -818,13 +823,7 @@ var LiveSpeechClient = class {
     }
   }
   /**
-   * Set transcript handler (simplified)
-   */
-  setTranscriptHandler(handler) {
-    this.transcriptHandler = handler;
-  }
-  /**
-   * Set response handler (simplified)
+   * Set response handler
    */
   setResponseHandler(handler) {
     this.responseHandler = handler;
@@ -835,6 +834,12 @@ var LiveSpeechClient = class {
   setAudioHandler(handler) {
     this.audioHandler = handler;
   }
+  /**
+   * Set user transcript handler
+   */
+  setUserTranscriptHandler(handler) {
+    this.userTranscriptHandler = handler;
+  }
   /**
    * Set error handler (simplified)
    */
@@ -864,7 +869,7 @@ var LiveSpeechClient = class {
   }
   handleDisconnected(code, _reason) {
     this.sessionId = null;
-    this.sessionConfig = null;
+    this.isStreaming = false;
     const event = {
       type: "disconnected",
       reason: code === 1e3 ? "normal" : "error",
@@ -906,25 +911,19 @@ var LiveSpeechClient = class {
         break;
       case "sessionEnded":
         this.sessionId = null;
-        this.sessionConfig = null;
+        this.isStreaming = false;
         this.emit("sessionEnded", {
           type: "sessionEnded",
           sessionId: message.sessionId,
           timestamp: message.timestamp
         });
         break;
-      case "transcript": {
-        const transcriptEvent = {
-          type: "transcript",
-          text: message.text,
-          isFinal: message.isFinal,
+      case "ready": {
+        const readyEvent = {
+          type: "ready",
           timestamp: message.timestamp
         };
-        if (message.confidence !== void 0) {
-          transcriptEvent.confidence = message.confidence;
-        }
-        this.emit("transcript", transcriptEvent);
-        this.transcriptHandler?.(message.text, message.isFinal);
+        this.emit("ready", readyEvent);
         break;
       }
       case "response": {
@@ -951,8 +950,26 @@ var LiveSpeechClient = class {
         this.audioHandler?.(audioData);
         break;
       }
+      case "userTranscript": {
+        const userTranscriptEvent = {
+          type: "userTranscript",
+          text: message.text,
+          timestamp: message.timestamp
+        };
+        this.emit("userTranscript", userTranscriptEvent);
+        this.userTranscriptHandler?.(message.text);
+        break;
+      }
+      case "turnComplete": {
+        const turnCompleteEvent = {
+          type: "turnComplete",
+          timestamp: message.timestamp
+        };
+        this.emit("turnComplete", turnCompleteEvent);
+        break;
+      }
       case "error":
-        this.handleError(message.code, message.message, message.details);
+        this.handleError(message.code, message.message);
         break;
       default:
         this.logger.warn("Unknown message type:", message.type);