@firebase/ai 2.4.0 → 2.5.0-canary.0800a8bed

package/dist/esm/src/factory-node.d.ts

@@ -0,0 +1,19 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { ComponentContainer, InstanceFactoryOptions } from '@firebase/component';
+import { AIService } from './service';
+export declare function factory(container: ComponentContainer, { instanceIdentifier }: InstanceFactoryOptions): AIService;

package/dist/esm/src/methods/chrome-adapter.d.ts

@@ -25,11 +25,11 @@ import { LanguageModel } from '../types/language-model';
 export declare class ChromeAdapterImpl implements ChromeAdapter {
     languageModelProvider: LanguageModel;
     mode: InferenceMode;
-    onDeviceParams: OnDeviceParams;
     static SUPPORTED_MIME_TYPES: string[];
     private isDownloading;
     private downloadPromise;
     private oldSession;
+    onDeviceParams: OnDeviceParams;
     constructor(languageModelProvider: LanguageModel, mode: InferenceMode, onDeviceParams?: OnDeviceParams);
     /**
      * Checks if a given request can be made on-device.

package/dist/esm/src/methods/live-session.d.ts

@@ -53,32 +53,65 @@ export declare class LiveSession {
      */
     send(request: string | Array<string | Part>, turnComplete?: boolean): Promise<void>;
     /**
-     * Sends realtime input to the server.
+     * Sends text to the server in realtime.
      *
-     * @param mediaChunks - The media chunks to send.
+     * @example
+     * ```javascript
+     * liveSession.sendTextRealtime("Hello, how are you?");
+     * ```
+     *
+     * @param text - The text data to send.
      * @throws If this session has been closed.
      *
      * @beta
      */
-    sendMediaChunks(mediaChunks: GenerativeContentBlob[]): Promise<void>;
+    sendTextRealtime(text: string): Promise<void>;
     /**
-     * Sends function responses to the server.
+     * Sends audio data to the server in realtime.
      *
-     * @param functionResponses - The function responses to send.
+     * @remarks The server requires that the audio data is base64-encoded 16-bit PCM at 16kHz
+     * little-endian.
+     *
+     * @example
+     * ```javascript
+     * // const pcmData = ... base64-encoded 16-bit PCM at 16kHz little-endian.
+     * const blob = { mimeType: "audio/pcm", data: pcmData };
+     * liveSession.sendAudioRealtime(blob);
+     * ```
+     *
+     * @param blob - The base64-encoded PCM data to send to the server in realtime.
      * @throws If this session has been closed.
      *
      * @beta
      */
-    sendFunctionResponses(functionResponses: FunctionResponse[]): Promise<void>;
+    sendAudioRealtime(blob: GenerativeContentBlob): Promise<void>;
     /**
-     * Sends a stream of {@link GenerativeContentBlob}.
+     * Sends video data to the server in realtime.
      *
-     * @param mediaChunkStream - The stream of {@link GenerativeContentBlob} to send.
+     * @remarks The server requires that the video is sent as individual video frames at 1 FPS. It
+     * is recommended to set `mimeType` to `image/jpeg`.
+     *
+     * @example
+     * ```javascript
+     * // const videoFrame = ... base64-encoded JPEG data
+     * const blob = { mimeType: "image/jpeg", data: videoFrame };
+     * liveSession.sendVideoRealtime(blob);
+     * ```
+     * @param blob - The base64-encoded video data to send to the server in realtime.
      * @throws If this session has been closed.
      *
      * @beta
      */
-    sendMediaStream(mediaChunkStream: ReadableStream<GenerativeContentBlob>): Promise<void>;
+    sendVideoRealtime(blob: GenerativeContentBlob): Promise<void>;
+    /**
+     * Sends function responses to the server.
+     *
+     * @param functionResponses - The function responses to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    sendFunctionResponses(functionResponses: FunctionResponse[]): Promise<void>;
     /**
      * Yields messages received from the server.
      * This can only be used by one consumer at a time.
@@ -96,4 +129,26 @@ export declare class LiveSession {
      * @beta
      */
     close(): Promise<void>;
+    /**
+     * Sends realtime input to the server.
+     *
+     * @deprecated Use `sendTextRealtime()`, `sendAudioRealtime()`, and `sendVideoRealtime()` instead.
+     *
+     * @param mediaChunks - The media chunks to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    sendMediaChunks(mediaChunks: GenerativeContentBlob[]): Promise<void>;
+    /**
+     * @deprecated Use `sendTextRealtime()`, `sendAudioRealtime()`, and `sendVideoRealtime()` instead.
+     *
+     * Sends a stream of {@link GenerativeContentBlob}.
+     *
+     * @param mediaChunkStream - The stream of {@link GenerativeContentBlob} to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    sendMediaStream(mediaChunkStream: ReadableStream<GenerativeContentBlob>): Promise<void>;
 }

package/dist/esm/src/requests/hybrid-helpers.d.ts

@@ -14,7 +14,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { GenerateContentRequest, ChromeAdapter } from '../types';
+import { GenerateContentRequest, ChromeAdapter, InferenceSource } from '../types';
+interface CallResult<Response> {
+    response: Response;
+    inferenceSource: InferenceSource;
+}
 /**
  * Dispatches a request to the appropriate backend (on-device or in-cloud)
  * based on the inference mode.
@@ -25,4 +29,5 @@ import { GenerateContentRequest, ChromeAdapter } from '../types';
  * @param inCloudCall - The function to call for in-cloud inference.
  * @returns The response from the backend.
  */
-export declare function callCloudOrDevice<Response>(request: GenerateContentRequest, chromeAdapter: ChromeAdapter | undefined, onDeviceCall: () => Promise<Response>, inCloudCall: () => Promise<Response>): Promise<Response>;
+export declare function callCloudOrDevice<Response>(request: GenerateContentRequest, chromeAdapter: ChromeAdapter | undefined, onDeviceCall: () => Promise<Response>, inCloudCall: () => Promise<Response>): Promise<CallResult<Response>>;
+export {};

package/dist/esm/src/requests/response-helpers.d.ts

@@ -14,12 +14,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { EnhancedGenerateContentResponse, FunctionCall, GenerateContentResponse, ImagenGCSImage, ImagenInlineImage, InlineDataPart, Part } from '../types';
+import { EnhancedGenerateContentResponse, FunctionCall, GenerateContentResponse, ImagenGCSImage, ImagenInlineImage, InlineDataPart, Part, InferenceSource } from '../types';
 /**
  * Creates an EnhancedGenerateContentResponse object that has helper functions and
  * other modifications that improve usability.
  */
-export declare function createEnhancedContentResponse(response: GenerateContentResponse): EnhancedGenerateContentResponse;
+export declare function createEnhancedContentResponse(response: GenerateContentResponse, inferenceSource?: InferenceSource): EnhancedGenerateContentResponse;
 /**
  * Adds convenience helper methods to a response object, including stream
  * chunks (as long as each chunk is a complete GenerateContentResponse JSON).

package/dist/esm/src/requests/stream-reader.d.ts

@@ -16,6 +16,7 @@
  */
 import { GenerateContentResponse, GenerateContentStreamResult } from '../types';
 import { ApiSettings } from '../types/internal';
+import { InferenceSource } from '../public-types';
 /**
  * Process a response.body stream from the backend and return an
  * iterator that provides one complete GenerateContentResponse at a time
@@ -24,7 +25,7 @@ import { ApiSettings } from '../types/internal';
  *
  * @param response - Response from a fetch call
  */
-export declare function processStream(response: Response, apiSettings: ApiSettings): GenerateContentStreamResult;
+export declare function processStream(response: Response, apiSettings: ApiSettings, inferenceSource?: InferenceSource): GenerateContentStreamResult;
 /**
  * Reads a raw stream from the fetch response and join incomplete
  * chunks, returning a new stream that provides a single complete

package/dist/esm/src/service.d.ts

@@ -15,21 +15,20 @@
  * limitations under the License.
  */
 import { FirebaseApp, _FirebaseService } from '@firebase/app';
-import { AI, AIOptions, InferenceMode, OnDeviceParams } from './public-types';
+import { AI, AIOptions, ChromeAdapter, InferenceMode, OnDeviceParams } from './public-types';
 import { AppCheckInternalComponentName, FirebaseAppCheckInternal } from '@firebase/app-check-interop-types';
 import { Provider } from '@firebase/component';
 import { FirebaseAuthInternal, FirebaseAuthInternalName } from '@firebase/auth-interop-types';
 import { Backend } from './backend';
-import { ChromeAdapterImpl } from './methods/chrome-adapter';
 export declare class AIService implements AI, _FirebaseService {
     app: FirebaseApp;
     backend: Backend;
-    chromeAdapterFactory?: ((mode: InferenceMode, window?: Window, params?: OnDeviceParams) => ChromeAdapterImpl | undefined) | undefined;
+    chromeAdapterFactory?: ((mode: InferenceMode, window?: Window, params?: OnDeviceParams) => ChromeAdapter | undefined) | undefined;
     auth: FirebaseAuthInternal | null;
     appCheck: FirebaseAppCheckInternal | null;
     _options?: Omit<AIOptions, 'backend'>;
     location: string;
-    constructor(app: FirebaseApp, backend: Backend, authProvider?: Provider<FirebaseAuthInternalName>, appCheckProvider?: Provider<AppCheckInternalComponentName>, chromeAdapterFactory?: ((mode: InferenceMode, window?: Window, params?: OnDeviceParams) => ChromeAdapterImpl | undefined) | undefined);
+    constructor(app: FirebaseApp, backend: Backend, authProvider?: Provider<FirebaseAuthInternalName>, appCheckProvider?: Provider<AppCheckInternalComponentName>, chromeAdapterFactory?: ((mode: InferenceMode, window?: Window, params?: OnDeviceParams) => ChromeAdapter | undefined) | undefined);
     _delete(): Promise<void>;
     set options(optionsToSet: AIOptions);
     get options(): AIOptions | undefined;

package/dist/esm/src/types/chrome-adapter.d.ts

@@ -14,6 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import { InferenceMode } from './enums';
 import { CountTokensRequest, GenerateContentRequest } from './requests';
 /**
  * Defines an inference "backend" that uses Chrome's on-device model,
@@ -25,6 +26,10 @@ import { CountTokensRequest, GenerateContentRequest } from './requests';
  * @beta
  */
 export interface ChromeAdapter {
+    /**
+     * @internal
+     */
+    mode: InferenceMode;
     /**
      * Checks if the on-device model is capable of handling a given
      * request.

package/dist/esm/src/types/enums.d.ts

@@ -349,6 +349,21 @@ export declare const InferenceMode: {
  * @beta
  */
 export type InferenceMode = (typeof InferenceMode)[keyof typeof InferenceMode];
+/**
+ * Indicates whether inference happened on-device or in-cloud.
+ *
+ * @beta
+ */
+export declare const InferenceSource: {
+    readonly ON_DEVICE: "on_device";
+    readonly IN_CLOUD: "in_cloud";
+};
+/**
+ * Indicates whether inference happened on-device or in-cloud.
+ *
+ * @beta
+ */
+export type InferenceSource = (typeof InferenceSource)[keyof typeof InferenceSource];
 /**
  * Represents the result of the code execution.
  *

package/dist/esm/src/types/live-responses.d.ts

@@ -15,7 +15,8 @@
  * limitations under the License.
  */
 import { Content, FunctionResponse, GenerativeContentBlob, Part } from './content';
-import { LiveGenerationConfig, Tool, ToolConfig } from './requests';
+import { AudioTranscriptionConfig, LiveGenerationConfig, Tool, ToolConfig } from './requests';
+import { Transcription } from './responses';
 /**
  * User input that is sent to the model.
  *
@@ -25,6 +26,8 @@ export interface _LiveClientContent {
     clientContent: {
         turns: [Content];
         turnComplete: boolean;
+        inputTranscription?: Transcription;
+        outputTranscription?: Transcription;
     };
 }
 /**
@@ -34,7 +37,13 @@ export interface _LiveClientContent {
  */
 export interface _LiveClientRealtimeInput {
     realtimeInput: {
-        mediaChunks: GenerativeContentBlob[];
+        text?: string;
+        audio?: GenerativeContentBlob;
+        video?: GenerativeContentBlob;
+        /**
+         * @deprecated Use `text`, `audio`, and `video` instead.
+         */
+        mediaChunks?: GenerativeContentBlob[];
     };
 }
 /**
@@ -53,9 +62,18 @@ export interface _LiveClientToolResponse {
 export interface _LiveClientSetup {
     setup: {
         model: string;
-        generationConfig?: LiveGenerationConfig;
+        generationConfig?: _LiveGenerationConfig;
         tools?: Tool[];
         toolConfig?: ToolConfig;
         systemInstruction?: string | Part | Content;
+        inputAudioTranscription?: AudioTranscriptionConfig;
+        outputAudioTranscription?: AudioTranscriptionConfig;
     };
 }
+/**
+ * The Live Generation Config.
+ *
+ * The public API ({@link LiveGenerationConfig}) has `inputAudioTranscription` and `outputAudioTranscription`,
+ * but the server expects these fields to be in the top-level `setup` message. This was a conscious API decision.
+ */
+export type _LiveGenerationConfig = Omit<LiveGenerationConfig, 'inputAudioTranscription' | 'outputAudioTranscription'>;

package/dist/esm/src/types/requests.d.ts

@@ -167,6 +167,24 @@ export interface LiveGenerationConfig {
      * The modalities of the response.
      */
     responseModalities?: ResponseModality[];
+    /**
+     * Enables transcription of audio input.
+     *
+     * When enabled, the model will respond with transcriptions of your audio input in the `inputTranscriptions` property
+     * in {@link LiveServerContent} messages. Note that the transcriptions are broken up across
+     * messages, so you may only receive small amounts of text per message. For example, if you ask the model
+     * "How are you today?", the model may transcribe that input across three messages, broken up as "How a", "re yo", "u today?".
+     */
+    inputAudioTranscription?: AudioTranscriptionConfig;
+    /**
+     * Enables transcription of audio input.
+     *
+     * When enabled, the model will respond with transcriptions of its audio output in the `outputTranscription` property
+     * in {@link LiveServerContent} messages. Note that the transcriptions are broken up across
+     * messages, so you may only receive small amounts of text per message. For example, if the model says
+     * "How are you today?", the model may transcribe that output across three messages, broken up as "How a", "re yo", "u today?".
+     */
+    outputAudioTranscription?: AudioTranscriptionConfig;
 }
 /**
  * Params for {@link GenerativeModel.startChat}.
@@ -439,3 +457,8 @@ export interface SpeechConfig {
      */
     voiceConfig?: VoiceConfig;
 }
+/**
+ * The audio transcription configuration.
+ */
+export interface AudioTranscriptionConfig {
+}

package/dist/esm/src/types/responses.d.ts

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 import { Content, FunctionCall, InlineDataPart } from './content';
-import { BlockReason, FinishReason, HarmCategory, HarmProbability, HarmSeverity, Modality } from './enums';
+import { BlockReason, FinishReason, HarmCategory, HarmProbability, HarmSeverity, InferenceSource, Modality } from './enums';
 /**
  * Result object returned from {@link GenerativeModel.generateContent} call.
  *
@@ -76,6 +76,12 @@ export interface EnhancedGenerateContentResponse extends GenerateContentResponse
      * set to `true`.
      */
     thoughtSummary: () => string | undefined;
+    /**
+     * Indicates whether inference happened on-device or in-cloud.
+     *
+     * @beta
+     */
+    inferenceSource?: InferenceSource;
 }
 /**
  * Individual response from {@link GenerativeModel.generateContent} and
@@ -510,6 +516,27 @@ export interface LiveServerContent {
      * model was not interrupted.
      */
     interrupted?: boolean;
+    /**
+     * Transcription of the audio that was input to the model.
+     */
+    inputTranscription?: Transcription;
+    /**
+     * Transcription of the audio output from the model.
+     */
+    outputTranscription?: Transcription;
+}
+/**
+ * Transcription of audio. This can be returned from a {@link LiveGenerativeModel} if transcription
+ * is enabled with the `inputAudioTranscription` or `outputAudioTranscription` properties on
+ * the {@link LiveGenerationConfig}.
+ *
+ * @beta
+ */
+export interface Transcription {
+    /**
+     * The text transcription of the audio.
+     */
+    text?: string;
 }
 /**
  * A request from the model for the client to execute one or more functions.