@firebase/ai 2.4.0-canary.91c218db2 → 2.4.0-canary.bc5a7c4a7

package/dist/esm/src/methods/live-session.d.ts

@@ -53,32 +53,65 @@ export declare class LiveSession {
      */
     send(request: string | Array<string | Part>, turnComplete?: boolean): Promise<void>;
     /**
-     * Sends realtime input to the server.
+     * Sends text to the server in realtime.
      *
-     * @param mediaChunks - The media chunks to send.
+     * @example
+     * ```javascript
+     * liveSession.sendTextRealtime("Hello, how are you?");
+     * ```
+     *
+     * @param text - The text data to send.
      * @throws If this session has been closed.
      *
      * @beta
      */
-    sendMediaChunks(mediaChunks: GenerativeContentBlob[]): Promise<void>;
+    sendTextRealtime(text: string): Promise<void>;
     /**
-     * Sends function responses to the server.
+     * Sends audio data to the server in realtime.
      *
-     * @param functionResponses - The function responses to send.
+     * @remarks The server requires that the audio data is base64-encoded 16-bit PCM at 16kHz
+     * little-endian.
+     *
+     * @example
+     * ```javascript
+     * // const pcmData = ... base64-encoded 16-bit PCM at 16kHz little-endian.
+     * const blob = { mimeType: "audio/pcm", data: pcmData };
+     * liveSession.sendAudioRealtime(blob);
+     * ```
+     *
+     * @param blob - The base64-encoded PCM data to send to the server in realtime.
      * @throws If this session has been closed.
      *
      * @beta
      */
-    sendFunctionResponses(functionResponses: FunctionResponse[]): Promise<void>;
+    sendAudioRealtime(blob: GenerativeContentBlob): Promise<void>;
     /**
-     * Sends a stream of {@link GenerativeContentBlob}.
+     * Sends video data to the server in realtime.
      *
-     * @param mediaChunkStream - The stream of {@link GenerativeContentBlob} to send.
+     * @remarks The server requires that the video is sent as individual video frames at 1 FPS. It
+     * is recommended to set `mimeType` to `image/jpeg`.
+     *
+     * @example
+     * ```javascript
+     * // const videoFrame = ... base64-encoded JPEG data
+     * const blob = { mimeType: "image/jpeg", data: videoFrame };
+     * liveSession.sendVideoRealtime(blob);
+     * ```
+     * @param blob - The base64-encoded video data to send to the server in realtime.
      * @throws If this session has been closed.
      *
      * @beta
      */
-    sendMediaStream(mediaChunkStream: ReadableStream<GenerativeContentBlob>): Promise<void>;
+    sendVideoRealtime(blob: GenerativeContentBlob): Promise<void>;
+    /**
+     * Sends function responses to the server.
+     *
+     * @param functionResponses - The function responses to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    sendFunctionResponses(functionResponses: FunctionResponse[]): Promise<void>;
     /**
      * Yields messages received from the server.
      * This can only be used by one consumer at a time.
@@ -96,4 +129,26 @@ export declare class LiveSession {
      * @beta
      */
     close(): Promise<void>;
+    /**
+     * Sends realtime input to the server.
+     *
+     * @deprecated Use `sendTextRealtime()`, `sendAudioRealtime()`, and `sendVideoRealtime()` instead.
+     *
+     * @param mediaChunks - The media chunks to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    sendMediaChunks(mediaChunks: GenerativeContentBlob[]): Promise<void>;
+    /**
+     * @deprecated Use `sendTextRealtime()`, `sendAudioRealtime()`, and `sendVideoRealtime()` instead.
+     *
+     * Sends a stream of {@link GenerativeContentBlob}.
+     *
+     * @param mediaChunkStream - The stream of {@link GenerativeContentBlob} to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    sendMediaStream(mediaChunkStream: ReadableStream<GenerativeContentBlob>): Promise<void>;
 }

package/dist/esm/src/types/live-responses.d.ts

@@ -15,7 +15,8 @@
  * limitations under the License.
  */
 import { Content, FunctionResponse, GenerativeContentBlob, Part } from './content';
-import { LiveGenerationConfig, Tool, ToolConfig } from './requests';
+import { AudioTranscriptionConfig, LiveGenerationConfig, Tool, ToolConfig } from './requests';
+import { Transcription } from './responses';
 /**
  * User input that is sent to the model.
  *
@@ -25,6 +26,8 @@ export interface _LiveClientContent {
     clientContent: {
         turns: [Content];
         turnComplete: boolean;
+        inputTranscription?: Transcription;
+        outputTranscription?: Transcription;
     };
 }
 /**
@@ -34,7 +37,13 @@ export interface _LiveClientContent {
  */
 export interface _LiveClientRealtimeInput {
     realtimeInput: {
-        mediaChunks: GenerativeContentBlob[];
+        text?: string;
+        audio?: GenerativeContentBlob;
+        video?: GenerativeContentBlob;
+        /**
+         * @deprecated Use `text`, `audio`, and `video` instead.
+         */
+        mediaChunks?: GenerativeContentBlob[];
     };
 }
 /**
@@ -53,9 +62,18 @@ export interface _LiveClientToolResponse {
 export interface _LiveClientSetup {
     setup: {
         model: string;
-        generationConfig?: LiveGenerationConfig;
+        generationConfig?: _LiveGenerationConfig;
         tools?: Tool[];
         toolConfig?: ToolConfig;
         systemInstruction?: string | Part | Content;
+        inputAudioTranscription?: AudioTranscriptionConfig;
+        outputAudioTranscription?: AudioTranscriptionConfig;
     };
 }
+/**
+ * The Live Generation Config.
+ *
+ * The public API ({@link LiveGenerationConfig}) has `inputAudioTranscription` and `outputAudioTranscription`,
+ * but the server expects these fields to be in the top-level `setup` message. This was a conscious API decision.
+ */
+export type _LiveGenerationConfig = Omit<LiveGenerationConfig, 'inputAudioTranscription' | 'outputAudioTranscription'>;

package/dist/esm/src/types/requests.d.ts

@@ -167,6 +167,24 @@ export interface LiveGenerationConfig {
      * The modalities of the response.
      */
     responseModalities?: ResponseModality[];
+    /**
+     * Enables transcription of audio input.
+     *
+     * When enabled, the model will respond with transcriptions of your audio input in the `inputTranscriptions` property
+     * in {@link LiveServerContent} messages. Note that the transcriptions are broken up across
+     * messages, so you may only receive small amounts of text per message. For example, if you ask the model
+     * "How are you today?", the model may transcribe that input across three messages, broken up as "How a", "re yo", "u today?".
+     */
+    inputAudioTranscription?: AudioTranscriptionConfig;
+    /**
+     * Enables transcription of audio input.
+     *
+     * When enabled, the model will respond with transcriptions of its audio output in the `outputTranscription` property
+     * in {@link LiveServerContent} messages. Note that the transcriptions are broken up across
+     * messages, so you may only receive small amounts of text per message. For example, if the model says
+     * "How are you today?", the model may transcribe that output across three messages, broken up as "How a", "re yo", "u today?".
+     */
+    outputAudioTranscription?: AudioTranscriptionConfig;
 }
 /**
  * Params for {@link GenerativeModel.startChat}.
@@ -439,3 +457,8 @@ export interface SpeechConfig {
      */
     voiceConfig?: VoiceConfig;
 }
+/**
+ * The audio transcription configuration.
+ */
+export interface AudioTranscriptionConfig {
+}

package/dist/esm/src/types/responses.d.ts

@@ -516,6 +516,27 @@ export interface LiveServerContent {
      * model was not interrupted.
      */
     interrupted?: boolean;
+    /**
+     * Transcription of the audio that was input to the model.
+     */
+    inputTranscription?: Transcription;
+    /**
+     * Transcription of the audio output from the model.
+     */
+    outputTranscription?: Transcription;
+}
+/**
+ * Transcription of audio. This can be returned from a {@link LiveGenerativeModel} if transcription
+ * is enabled with the `inputAudioTranscription` or `outputAudioTranscription` properties on
+ * the {@link LiveGenerationConfig}.
+ *
+ * @beta
+ */
+export interface Transcription {
+    /**
+     * The text transcription of the audio.
+     */
+    text?: string;
 }
 /**
  * A request from the model for the client to execute one or more functions.

package/dist/index.cjs.js

@@ -8,7 +8,7 @@ var util = require('@firebase/util');
 var logger$1 = require('@firebase/logger');
 
 var name = "@firebase/ai";
-var version = "2.4.0-canary.91c218db2";
+var version = "2.4.0-canary.bc5a7c4a7";
 
 /**
  * @license
@@ -2874,75 +2874,104 @@ class LiveSession {
         this.webSocketHandler.send(JSON.stringify(message));
     }
     /**
-     * Sends realtime input to the server.
+     * Sends text to the server in realtime.
      *
-     * @param mediaChunks - The media chunks to send.
+     * @example
+     * ```javascript
+     * liveSession.sendTextRealtime("Hello, how are you?");
+     * ```
+     *
+     * @param text - The text data to send.
      * @throws If this session has been closed.
      *
      * @beta
      */
-    async sendMediaChunks(mediaChunks) {
+    async sendTextRealtime(text) {
         if (this.isClosed) {
             throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
         }
-        // The backend does not support sending more than one mediaChunk in one message.
-        // Work around this limitation by sending mediaChunks in separate messages.
-        mediaChunks.forEach(mediaChunk => {
-            const message = {
-                realtimeInput: { mediaChunks: [mediaChunk] }
-            };
-            this.webSocketHandler.send(JSON.stringify(message));
-        });
+        const message = {
+            realtimeInput: {
+                text
+            }
+        };
+        this.webSocketHandler.send(JSON.stringify(message));
     }
     /**
-     * Sends function responses to the server.
+     * Sends audio data to the server in realtime.
      *
-     * @param functionResponses - The function responses to send.
+     * @remarks The server requires that the audio data is base64-encoded 16-bit PCM at 16kHz
+     * little-endian.
+     *
+     * @example
+     * ```javascript
+     * // const pcmData = ... base64-encoded 16-bit PCM at 16kHz little-endian.
+     * const blob = { mimeType: "audio/pcm", data: pcmData };
+     * liveSession.sendAudioRealtime(blob);
+     * ```
+     *
+     * @param blob - The base64-encoded PCM data to send to the server in realtime.
      * @throws If this session has been closed.
      *
      * @beta
      */
-    async sendFunctionResponses(functionResponses) {
+    async sendAudioRealtime(blob) {
         if (this.isClosed) {
             throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
         }
         const message = {
-            toolResponse: {
-                functionResponses
+            realtimeInput: {
+                audio: blob
             }
         };
         this.webSocketHandler.send(JSON.stringify(message));
     }
     /**
-     * Sends a stream of {@link GenerativeContentBlob}.
+     * Sends video data to the server in realtime.
      *
-     * @param mediaChunkStream - The stream of {@link GenerativeContentBlob} to send.
+     * @remarks The server requires that the video is sent as individual video frames at 1 FPS. It
+     * is recommended to set `mimeType` to `image/jpeg`.
+     *
+     * @example
+     * ```javascript
+     * // const videoFrame = ... base64-encoded JPEG data
+     * const blob = { mimeType: "image/jpeg", data: videoFrame };
+     * liveSession.sendVideoRealtime(blob);
+     * ```
+     * @param blob - The base64-encoded video data to send to the server in realtime.
      * @throws If this session has been closed.
      *
      * @beta
      */
-    async sendMediaStream(mediaChunkStream) {
+    async sendVideoRealtime(blob) {
         if (this.isClosed) {
             throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
         }
-        const reader = mediaChunkStream.getReader();
-        while (true) {
-            try {
-                const { done, value } = await reader.read();
-                if (done) {
-                    break;
-                }
-                else if (!value) {
-                    throw new Error('Missing chunk in reader, but reader is not done.');
-                }
-                await this.sendMediaChunks([value]);
-            }
-            catch (e) {
-                // Re-throw any errors that occur during stream consumption or sending.
-                const message = e instanceof Error ? e.message : 'Error processing media stream.';
-                throw new AIError(AIErrorCode.REQUEST_ERROR, message);
+        const message = {
+            realtimeInput: {
+                video: blob
             }
+        };
+        this.webSocketHandler.send(JSON.stringify(message));
+    }
+    /**
+     * Sends function responses to the server.
+     *
+     * @param functionResponses - The function responses to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    async sendFunctionResponses(functionResponses) {
+        if (this.isClosed) {
+            throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
         }
+        const message = {
+            toolResponse: {
+                functionResponses
+            }
+        };
+        this.webSocketHandler.send(JSON.stringify(message));
     }
     /**
      * Yields messages received from the server.
@@ -3000,6 +3029,62 @@ class LiveSession {
             await this.webSocketHandler.close(1000, 'Client closed session.');
         }
     }
+    /**
+     * Sends realtime input to the server.
+     *
+     * @deprecated Use `sendTextRealtime()`, `sendAudioRealtime()`, and `sendVideoRealtime()` instead.
+     *
+     * @param mediaChunks - The media chunks to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    async sendMediaChunks(mediaChunks) {
+        if (this.isClosed) {
+            throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
+        }
+        // The backend does not support sending more than one mediaChunk in one message.
+        // Work around this limitation by sending mediaChunks in separate messages.
+        mediaChunks.forEach(mediaChunk => {
+            const message = {
+                realtimeInput: { mediaChunks: [mediaChunk] }
+            };
+            this.webSocketHandler.send(JSON.stringify(message));
+        });
+    }
+    /**
+     * @deprecated Use `sendTextRealtime()`, `sendAudioRealtime()`, and `sendVideoRealtime()` instead.
+     *
+     * Sends a stream of {@link GenerativeContentBlob}.
+     *
+     * @param mediaChunkStream - The stream of {@link GenerativeContentBlob} to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    async sendMediaStream(mediaChunkStream) {
+        if (this.isClosed) {
+            throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
+        }
+        const reader = mediaChunkStream.getReader();
+        while (true) {
+            try {
+                const { done, value } = await reader.read();
+                if (done) {
+                    break;
+                }
+                else if (!value) {
+                    throw new Error('Missing chunk in reader, but reader is not done.');
+                }
+                await this.sendMediaChunks([value]);
+            }
+            catch (e) {
+                // Re-throw any errors that occur during stream consumption or sending.
+                const message = e instanceof Error ? e.message : 'Error processing media stream.';
+                throw new AIError(AIErrorCode.REQUEST_ERROR, message);
+            }
+        }
+    }
 }
 
 /**
@@ -3060,13 +3145,18 @@ class LiveGenerativeModel extends AIModel {
         else {
             fullModelPath = `projects/${this._apiSettings.project}/locations/${this._apiSettings.location}/${this.model}`;
         }
+        // inputAudioTranscription and outputAudioTranscription are on the generation config in the public API,
+        // but the backend expects them to be in the `setup` message.
+        const { inputAudioTranscription, outputAudioTranscription, ...generationConfig } = this.generationConfig;
         const setupMessage = {
             setup: {
                 model: fullModelPath,
-                generationConfig: this.generationConfig,
+                generationConfig,
                 tools: this.tools,
                 toolConfig: this.toolConfig,
-                systemInstruction: this.systemInstruction
+                systemInstruction: this.systemInstruction,
+                inputAudioTranscription,
+                outputAudioTranscription
             }
         };
         try {
@@ -3772,7 +3862,7 @@ class AudioConversationRunner {
                 mimeType: 'audio/pcm',
                 data: base64
             };
-            void this.liveSession.sendMediaChunks([chunk]);
+            void this.liveSession.sendAudioRealtime(chunk);
         };
     }
     /**