@firebase/ai 2.4.0-canary.44d9891f9 → 2.4.0-canary.6e0e30317

package/dist/index.node.mjs

@@ -4,7 +4,7 @@ import { FirebaseError, Deferred, getModularInstance } from '@firebase/util';
 import { Logger } from '@firebase/logger';
 
 var name = "@firebase/ai";
-var version = "2.4.0-canary.44d9891f9";
+var version = "2.4.0-canary.6e0e30317";
 
 /**
  * @license
@@ -34,6 +34,62 @@ const DEFAULT_FETCH_TIMEOUT_MS = 180 * 1000;
  */
 const DEFAULT_HYBRID_IN_CLOUD_MODEL = 'gemini-2.0-flash-lite';
 
+/**
+ * @license
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Error class for the Firebase AI SDK.
+ *
+ * @public
+ */
+class AIError extends FirebaseError {
+    /**
+     * Constructs a new instance of the `AIError` class.
+     *
+     * @param code - The error code from {@link (AIErrorCode:type)}.
+     * @param message - A human-readable message describing the error.
+     * @param customErrorData - Optional error data.
+     */
+    constructor(code, message, customErrorData) {
+        // Match error format used by FirebaseError from ErrorFactory
+        const service = AI_TYPE;
+        const fullCode = `${service}/${code}`;
+        const fullMessage = `${service}: ${message} (${fullCode})`;
+        super(code, fullMessage);
+        this.code = code;
+        this.customErrorData = customErrorData;
+        // FirebaseError initializes a stack trace, but it assumes the error is created from the error
+        // factory. Since we break this assumption, we set the stack trace to be originating from this
+        // constructor.
+        // This is only supported in V8.
+        if (Error.captureStackTrace) {
+            // Allows us to initialize the stack trace without including the constructor itself at the
+            // top level of the stack trace.
+            Error.captureStackTrace(this, AIError);
+        }
+        // Allows instanceof AIError in ES5/ES6
+        // https://github.com/Microsoft/TypeScript-wiki/blob/master/Breaking-Changes.md#extending-built-ins-like-error-array-and-map-may-no-longer-work
+        // TODO(dlarocque): Replace this with `new.target`: https://www.typescriptlang.org/docs/handbook/release-notes/typescript-2-2.html#support-for-newtarget
+        //                   which we can now use since we no longer target ES5.
+        Object.setPrototypeOf(this, AIError.prototype);
+        // Since Error is an interface, we don't inherit toString and so we define it ourselves.
+        this.toString = () => fullMessage;
+    }
+}
+
 /**
  * @license
  * Copyright 2024 Google LLC
@@ -738,6 +794,64 @@ class VertexAIBackend extends Backend {
     }
 }
 
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Encodes a {@link Backend} into a string that will be used to uniquely identify {@link AI}
+ * instances by backend type.
+ *
+ * @internal
+ */
+function encodeInstanceIdentifier(backend) {
+    if (backend instanceof GoogleAIBackend) {
+        return `${AI_TYPE}/googleai`;
+    }
+    else if (backend instanceof VertexAIBackend) {
+        return `${AI_TYPE}/vertexai/${backend.location}`;
+    }
+    else {
+        throw new AIError(AIErrorCode.ERROR, `Invalid backend: ${JSON.stringify(backend.backendType)}`);
+    }
+}
+/**
+ * Decodes an instance identifier string into a {@link Backend}.
+ *
+ * @internal
+ */
+function decodeInstanceIdentifier(instanceIdentifier) {
+    const identifierParts = instanceIdentifier.split('/');
+    if (identifierParts[0] !== AI_TYPE) {
+        throw new AIError(AIErrorCode.ERROR, `Invalid instance identifier, unknown prefix '${identifierParts[0]}'`);
+    }
+    const backendType = identifierParts[1];
+    switch (backendType) {
+        case 'vertexai':
+            const location = identifierParts[2];
+            if (!location) {
+                throw new AIError(AIErrorCode.ERROR, `Invalid instance identifier, unknown location '${instanceIdentifier}'`);
+            }
+            return new VertexAIBackend(location);
+        case 'googleai':
+            return new GoogleAIBackend();
+        default:
+            throw new AIError(AIErrorCode.ERROR, `Invalid instance identifier string: '${instanceIdentifier}'`);
+    }
+}
+
 /**
  * @license
  * Copyright 2024 Google LLC
@@ -781,62 +895,6 @@ class AIService {
     }
 }
 
-/**
- * @license
- * Copyright 2024 Google LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * Error class for the Firebase AI SDK.
- *
- * @public
- */
-class AIError extends FirebaseError {
-    /**
-     * Constructs a new instance of the `AIError` class.
-     *
-     * @param code - The error code from {@link (AIErrorCode:type)}.
-     * @param message - A human-readable message describing the error.
-     * @param customErrorData - Optional error data.
-     */
-    constructor(code, message, customErrorData) {
-        // Match error format used by FirebaseError from ErrorFactory
-        const service = AI_TYPE;
-        const fullCode = `${service}/${code}`;
-        const fullMessage = `${service}: ${message} (${fullCode})`;
-        super(code, fullMessage);
-        this.code = code;
-        this.customErrorData = customErrorData;
-        // FirebaseError initializes a stack trace, but it assumes the error is created from the error
-        // factory. Since we break this assumption, we set the stack trace to be originating from this
-        // constructor.
-        // This is only supported in V8.
-        if (Error.captureStackTrace) {
-            // Allows us to initialize the stack trace without including the constructor itself at the
-            // top level of the stack trace.
-            Error.captureStackTrace(this, AIError);
-        }
-        // Allows instanceof AIError in ES5/ES6
-        // https://github.com/Microsoft/TypeScript-wiki/blob/master/Breaking-Changes.md#extending-built-ins-like-error-array-and-map-may-no-longer-work
-        // TODO(dlarocque): Replace this with `new.target`: https://www.typescriptlang.org/docs/handbook/release-notes/typescript-2-2.html#support-for-newtarget
-        //                   which we can now use since we no longer target ES5.
-        Object.setPrototypeOf(this, AIError.prototype);
-        // Since Error is an interface, we don't inherit toString and so we define it ourselves.
-        this.toString = () => fullMessage;
-    }
-}
-
 /**
  * @license
  * Copyright 2025 Google LLC
@@ -853,46 +911,16 @@ class AIError extends FirebaseError {
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-/**
- * Encodes a {@link Backend} into a string that will be used to uniquely identify {@link AI}
- * instances by backend type.
- *
- * @internal
- */
-function encodeInstanceIdentifier(backend) {
-    if (backend instanceof GoogleAIBackend) {
-        return `${AI_TYPE}/googleai`;
-    }
-    else if (backend instanceof VertexAIBackend) {
-        return `${AI_TYPE}/vertexai/${backend.location}`;
-    }
-    else {
-        throw new AIError(AIErrorCode.ERROR, `Invalid backend: ${JSON.stringify(backend.backendType)}`);
-    }
-}
-/**
- * Decodes an instance identifier string into a {@link Backend}.
- *
- * @internal
- */
-function decodeInstanceIdentifier(instanceIdentifier) {
-    const identifierParts = instanceIdentifier.split('/');
-    if (identifierParts[0] !== AI_TYPE) {
-        throw new AIError(AIErrorCode.ERROR, `Invalid instance identifier, unknown prefix '${identifierParts[0]}'`);
-    }
-    const backendType = identifierParts[1];
-    switch (backendType) {
-        case 'vertexai':
-            const location = identifierParts[2];
-            if (!location) {
-                throw new AIError(AIErrorCode.ERROR, `Invalid instance identifier, unknown location '${instanceIdentifier}'`);
-            }
-            return new VertexAIBackend(location);
-        case 'googleai':
-            return new GoogleAIBackend();
-        default:
-            throw new AIError(AIErrorCode.ERROR, `Invalid instance identifier string: '${instanceIdentifier}'`);
+function factory(container, { instanceIdentifier }) {
+    if (!instanceIdentifier) {
+        throw new AIError(AIErrorCode.ERROR, 'AIService instance identifier is undefined.');
     }
+    const backend = decodeInstanceIdentifier(instanceIdentifier);
+    // getImmediate for FirebaseApp will always succeed
+    const app = container.getProvider('app').getImmediate();
+    const auth = container.getProvider('auth-internal');
+    const appCheckProvider = container.getProvider('app-check-internal');
+    return new AIService(app, backend, auth, appCheckProvider);
 }
 
 /**
@@ -2534,75 +2562,104 @@ class LiveSession {
         this.webSocketHandler.send(JSON.stringify(message));
     }
     /**
-     * Sends realtime input to the server.
+     * Sends text to the server in realtime.
      *
-     * @param mediaChunks - The media chunks to send.
+     * @example
+     * ```javascript
+     * liveSession.sendTextRealtime("Hello, how are you?");
+     * ```
+     *
+     * @param text - The text data to send.
      * @throws If this session has been closed.
      *
      * @beta
      */
-    async sendMediaChunks(mediaChunks) {
+    async sendTextRealtime(text) {
         if (this.isClosed) {
             throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
         }
-        // The backend does not support sending more than one mediaChunk in one message.
-        // Work around this limitation by sending mediaChunks in separate messages.
-        mediaChunks.forEach(mediaChunk => {
-            const message = {
-                realtimeInput: { mediaChunks: [mediaChunk] }
-            };
-            this.webSocketHandler.send(JSON.stringify(message));
-        });
+        const message = {
+            realtimeInput: {
+                text
+            }
+        };
+        this.webSocketHandler.send(JSON.stringify(message));
     }
     /**
-     * Sends function responses to the server.
+     * Sends audio data to the server in realtime.
      *
-     * @param functionResponses - The function responses to send.
+     * @remarks The server requires that the audio data is base64-encoded 16-bit PCM at 16kHz
+     * little-endian.
+     *
+     * @example
+     * ```javascript
+     * // const pcmData = ... base64-encoded 16-bit PCM at 16kHz little-endian.
+     * const blob = { mimeType: "audio/pcm", data: pcmData };
+     * liveSession.sendAudioRealtime(blob);
+     * ```
+     *
+     * @param blob - The base64-encoded PCM data to send to the server in realtime.
      * @throws If this session has been closed.
      *
      * @beta
      */
-    async sendFunctionResponses(functionResponses) {
+    async sendAudioRealtime(blob) {
         if (this.isClosed) {
             throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
         }
         const message = {
-            toolResponse: {
-                functionResponses
+            realtimeInput: {
+                audio: blob
             }
         };
         this.webSocketHandler.send(JSON.stringify(message));
     }
     /**
-     * Sends a stream of {@link GenerativeContentBlob}.
+     * Sends video data to the server in realtime.
      *
-     * @param mediaChunkStream - The stream of {@link GenerativeContentBlob} to send.
+     * @remarks The server requires that the video is sent as individual video frames at 1 FPS. It
+     * is recommended to set `mimeType` to `image/jpeg`.
+     *
+     * @example
+     * ```javascript
+     * // const videoFrame = ... base64-encoded JPEG data
+     * const blob = { mimeType: "image/jpeg", data: videoFrame };
+     * liveSession.sendVideoRealtime(blob);
+     * ```
+     * @param blob - The base64-encoded video data to send to the server in realtime.
      * @throws If this session has been closed.
      *
      * @beta
      */
-    async sendMediaStream(mediaChunkStream) {
+    async sendVideoRealtime(blob) {
         if (this.isClosed) {
             throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
         }
-        const reader = mediaChunkStream.getReader();
-        while (true) {
-            try {
-                const { done, value } = await reader.read();
-                if (done) {
-                    break;
-                }
-                else if (!value) {
-                    throw new Error('Missing chunk in reader, but reader is not done.');
-                }
-                await this.sendMediaChunks([value]);
-            }
-            catch (e) {
-                // Re-throw any errors that occur during stream consumption or sending.
-                const message = e instanceof Error ? e.message : 'Error processing media stream.';
-                throw new AIError(AIErrorCode.REQUEST_ERROR, message);
+        const message = {
+            realtimeInput: {
+                video: blob
             }
+        };
+        this.webSocketHandler.send(JSON.stringify(message));
+    }
+    /**
+     * Sends function responses to the server.
+     *
+     * @param functionResponses - The function responses to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    async sendFunctionResponses(functionResponses) {
+        if (this.isClosed) {
+            throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
         }
+        const message = {
+            toolResponse: {
+                functionResponses
+            }
+        };
+        this.webSocketHandler.send(JSON.stringify(message));
     }
     /**
      * Yields messages received from the server.
@@ -2660,6 +2717,62 @@ class LiveSession {
             await this.webSocketHandler.close(1000, 'Client closed session.');
         }
     }
+    /**
+     * Sends realtime input to the server.
+     *
+     * @deprecated Use `sendTextRealtime()`, `sendAudioRealtime()`, and `sendVideoRealtime()` instead.
+     *
+     * @param mediaChunks - The media chunks to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    async sendMediaChunks(mediaChunks) {
+        if (this.isClosed) {
+            throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
+        }
+        // The backend does not support sending more than one mediaChunk in one message.
+        // Work around this limitation by sending mediaChunks in separate messages.
+        mediaChunks.forEach(mediaChunk => {
+            const message = {
+                realtimeInput: { mediaChunks: [mediaChunk] }
+            };
+            this.webSocketHandler.send(JSON.stringify(message));
+        });
+    }
+    /**
+     * @deprecated Use `sendTextRealtime()`, `sendAudioRealtime()`, and `sendVideoRealtime()` instead.
+     *
+     * Sends a stream of {@link GenerativeContentBlob}.
+     *
+     * @param mediaChunkStream - The stream of {@link GenerativeContentBlob} to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    async sendMediaStream(mediaChunkStream) {
+        if (this.isClosed) {
+            throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
+        }
+        const reader = mediaChunkStream.getReader();
+        while (true) {
+            try {
+                const { done, value } = await reader.read();
+                if (done) {
+                    break;
+                }
+                else if (!value) {
+                    throw new Error('Missing chunk in reader, but reader is not done.');
+                }
+                await this.sendMediaChunks([value]);
+            }
+            catch (e) {
+                // Re-throw any errors that occur during stream consumption or sending.
+                const message = e instanceof Error ? e.message : 'Error processing media stream.';
+                throw new AIError(AIErrorCode.REQUEST_ERROR, message);
+            }
+        }
+    }
 }
 
 /**
@@ -2720,13 +2833,18 @@ class LiveGenerativeModel extends AIModel {
         else {
             fullModelPath = `projects/${this._apiSettings.project}/locations/${this._apiSettings.location}/${this.model}`;
         }
+        // inputAudioTranscription and outputAudioTranscription are on the generation config in the public API,
+        // but the backend expects them to be in the `setup` message.
+        const { inputAudioTranscription, outputAudioTranscription, ...generationConfig } = this.generationConfig;
         const setupMessage = {
             setup: {
                 model: fullModelPath,
-                generationConfig: this.generationConfig,
+                generationConfig,
                 tools: this.tools,
                 toolConfig: this.toolConfig,
-                systemInstruction: this.systemInstruction
+                systemInstruction: this.systemInstruction,
+                inputAudioTranscription,
+                outputAudioTranscription
             }
         };
         try {
@@ -3432,7 +3550,7 @@ class AudioConversationRunner {
                 mimeType: 'audio/pcm',
                 data: base64
             };
-            void this.liveSession.sendMediaChunks([chunk]);
+            void this.liveSession.sendAudioRealtime(chunk);
         };
     }
     /**
@@ -3810,17 +3928,7 @@ function getLiveGenerativeModel(ai, modelParams) {
  * @packageDocumentation
  */
 function registerAI() {
-    _registerComponent(new Component(AI_TYPE, (container, { instanceIdentifier }) => {
-        if (!instanceIdentifier) {
-            throw new AIError(AIErrorCode.ERROR, 'AIService instance identifier is undefined.');
-        }
-        const backend = decodeInstanceIdentifier(instanceIdentifier);
-        // getImmediate for FirebaseApp will always succeed
-        const app = container.getProvider('app').getImmediate();
-        const auth = container.getProvider('auth-internal');
-        const appCheckProvider = container.getProvider('app-check-internal');
-        return new AIService(app, backend, auth, appCheckProvider);
-    }, "PUBLIC" /* ComponentType.PUBLIC */).setMultipleInstances(true));
+    _registerComponent(new Component(AI_TYPE, factory, "PUBLIC" /* ComponentType.PUBLIC */).setMultipleInstances(true));
     registerVersion(name, version, 'node');
     // BUILD_TARGET will be replaced by values like esm, cjs, etc during the compilation
     registerVersion(name, version, 'esm2020');