@pipecat-ai/gemini-live-websocket-transport 0.3.3

package/LICENSE

@@ -0,0 +1,24 @@
+BSD 2-Clause License
+
+Copyright (c) 2024, Daily
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package/README.md

@@ -0,0 +1,124 @@
+# Gemini Live Websocket Transport
+
+A real-time websocket transport implementation for interacting with Google's Gemini Multimodal Live API, supporting bidirectional audio and unidirectional text communication.
+
+## Installation
+
+```bash
+npm install @pipecat-ai/client-js @pipecat-ai/real-time-websocket-transport @pipecat-ai/gemini-live-websocket-transport
+```
+
+## Overview
+
+The `GeminiLiveWebsocketTransport` class extends the `RealTimeWebsocketTransport` to implement a fully functional [RTVI `Transport`](https://docs.pipecat.ai/client/reference/js/transports/transport). It provides a framework for implementing real-time communication directly with the [Gemini Multimodal Live](https://ai.google.dev/api/multimodal-live) voice-to-voice service. It handles media device management, audio/video streams, and state management for the connection.
+
+## Features
+
+- Real-time bidirectional communication with Gemini Multimodal Live
+- Audio streaming support
+- Text message support
+- Automatic reconnection handling
+- Configurable generation parameters
+- Support for initial conversation context
+
+## Usage
+
+### Basic Setup
+
+```javascript
+import { GeminiLiveWebsocketTransport, GeminiLLMServiceOptions } from '@pipecat-ai/gemini-live-websocket-transport';
+
+const options: GeminiLLMServiceOptions = {
+  api_key: 'YOUR_API_KEY',
+  generation_config: {
+    temperature: 0.7,
+    maxOutput_tokens: 1000
+  }
+};
+
+const transport = new GeminiLiveWebsocketTransport(options);
+let RTVIConfig: RTVIClientOptions = {
+  transport,
+  ...
+};
+
+```
+
+### Configuration Options
+
+```typescript
+interface GeminiLLMServiceOptions {
+  api_key: string;                    // Required: Your Gemini API key
+  initial_messages?: Array<{          // Optional: Initial conversation context
+    content: string;
+    role: string;
+  }>;
+  generation_config?: {               // Optional: Generation parameters
+    candidate_count?: number;
+    maxOutput_tokens?: number;
+    temperature?: number;
+    top_p?: number;
+    top_k?: number;
+    presence_penalty?: number;
+    frequency_penalty?: number;
+    response_modalities?: string;
+    speech_config?: {
+      voice_config?: {
+        prebuilt_voice_config?: {
+          voice_name: "Puck" | "Charon" | "Kore" | "Fenrir" | "Aoede";
+        };
+      };
+    };
+  };
+}
+```
+
+### Sending Messages
+
+```javascript
+// Send text prompt message
+rtviClient.sendMessage({
+  type: 'send-text',
+  data: 'Hello, Gemini!'
+});
+```
+
+### Handling Events
+
+The transport implements the various [RTVI event handlers](https://docs.pipecat.ai/client/reference/js/callbacks). Check out the docs or samples for more info.
+
+## API Reference
+
+### Methods
+
+- `initialize()`: Set up the transport and establish connection
+- `sendMessage(message)`: Send a text message
+- `handleUserAudioStream(data)`: Stream audio data to the model
+- `disconnectLLM()`: Close the connection
+- `sendReadyMessage()`: Signal ready state
+
+### States
+
+The transport can be in one of the following states:
+- "disconnected"
+- "initializing"
+- "initialized"
+- "connecting"
+- "connected"
+- "ready"
+- "disconnecting
+- "error"
+
+## Error Handling
+
+The transport includes comprehensive error handling for:
+- Connection failures
+- Websocket errors
+- API key validation
+- Message transmission errors
+
+## License
+BSD-2 Clause
+
+## Contributing
+Feel free to submit issues and pull requests for improvements or bug fixes. Be nice :)

package/dist/index.d.ts

@@ -0,0 +1,41 @@
+import { MediaManager, RealTimeWebsocketTransport, LLMServiceOptions } from "@pipecat-ai/realtime-websocket-transport";
+import { RTVIMessage } from "@pipecat-ai/client-js";
+export interface GeminiLLMServiceOptions extends LLMServiceOptions {
+    initial_messages?: Array<{
+        content: string;
+        role: string;
+    }>;
+    api_key: string;
+    generation_config?: {
+        candidate_count?: number;
+        maxOutput_tokens?: number;
+        temperature?: number;
+        top_p?: number;
+        top_k?: number;
+        presence_penalty?: number;
+        frequency_penalty?: number;
+        response_modalities?: string;
+        speech_config?: {
+            voice_config?: {
+                prebuilt_voice_config?: {
+                    voice_name: "Puck" | "Charon" | "Kore" | "Fenrir" | "Aoede";
+                };
+            };
+        };
+    };
+}
+export class GeminiLiveWebsocketTransport extends RealTimeWebsocketTransport {
+    constructor(service_options: GeminiLLMServiceOptions, manager?: MediaManager);
+    initializeLLM(): void;
+    attachLLMListeners(): void;
+    connectLLM(): Promise<void>;
+    disconnectLLM(): Promise<void>;
+    sendReadyMessage(): Promise<void>;
+    handleUserAudioStream(data: ArrayBuffer): void;
+    sendMessage(message: RTVIMessage): void;
+    _sendAudioInput(data: ArrayBuffer): Promise<void>;
+    _sendTextInput(text: string, role: string): Promise<void>;
+    _sendMsg(msg: unknown): Promise<void>;
+}
+
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"mappings":";;ACkBA,wCAAyC,SAAQ,iBAAiB;IAChE,gBAAgB,CAAC,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5D,OAAO,EAAE,MAAM,CAAC;IAChB,iBAAiB,CAAC,EAAE;QAClB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,aAAa,CAAC,EAAE;YACd,YAAY,CAAC,EAAE;gBACb,qBAAqB,CAAC,EAAE;oBACtB,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;iBAE7D,CAAC;aACH,CAAC;SACH,CAAC;KACH,CAAC;CACH;AAED,yCAA0C,SAAQ,0BAA0B;gBAQxE,eAAe,EAAE,uBAAuB,EACxC,OAAO,CAAC,EAAE,YAAY;IAYxB,aAAa,IAAI,IAAI;IAcrB,kBAAkB,IAAI,IAAI;IA8DpB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAiC3B,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IAI9B,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IAevC,qBAAqB,CAAC,IAAI,EAAE,WAAW,GAAG,IAAI;IAa9C,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,IAAI;IAMjC,eAAe,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBjD,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAezD,QAAQ,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;CAmB5C","sources":["transports/lib/websocket-utils/reconnectingWebSocket.ts","transports/gemini-live-websocket-transport/src/src/geminiLiveWebSocketTransport.ts","transports/gemini-live-websocket-transport/src/src/index.ts","transports/gemini-live-websocket-transport/src/index.ts"],"sourcesContent":[null,null,null,"export * from \"./geminiLiveWebSocketTransport\";\n"],"names":[],"version":3,"file":"index.d.ts.map"}