@firebase/ai 2.4.0 → 2.5.0-canary.0800a8bed

package/dist/ai-public.d.ts

@@ -192,6 +192,12 @@ export declare interface AudioConversationController {
     stop: () => Promise<void>;
 }
 
+/**
+ * The audio transcription configuration.
+ */
+export declare interface AudioTranscriptionConfig {
+}
+
 /**
  * Abstract base class representing the configuration for an AI service backend.
  * This class should not be instantiated directly. Use its subclasses; {@link GoogleAIBackend} for
@@ -335,6 +341,7 @@ export declare class ChatSession {
  * @beta
  */
 export declare interface ChromeAdapter {
+    /* Excluded from this release type: mode */
     /**
      * Checks if the on-device model is capable of handling a given
      * request.
@@ -558,6 +565,12 @@ export declare interface EnhancedGenerateContentResponse extends GenerateContent
      * set to `true`.
      */
     thoughtSummary: () => string | undefined;
+    /**
+     * Indicates whether inference happened on-device or in-cloud.
+     *
+     * @beta
+     */
+    inferenceSource?: InferenceSource;
 }
 
 /**
@@ -1833,6 +1846,23 @@ export declare const InferenceMode: {
  */
 export declare type InferenceMode = (typeof InferenceMode)[keyof typeof InferenceMode];
 
+/**
+ * Indicates whether inference happened on-device or in-cloud.
+ *
+ * @beta
+ */
+export declare const InferenceSource: {
+    readonly ON_DEVICE: "on_device";
+    readonly IN_CLOUD: "in_cloud";
+};
+
+/**
+ * Indicates whether inference happened on-device or in-cloud.
+ *
+ * @beta
+ */
+export declare type InferenceSource = (typeof InferenceSource)[keyof typeof InferenceSource];
+
 /**
  * Content part interface if the part represents an image.
  * @public
@@ -1997,6 +2027,24 @@ export declare interface LiveGenerationConfig {
      * The modalities of the response.
      */
     responseModalities?: ResponseModality[];
+    /**
+     * Enables transcription of audio input.
+     *
+     * When enabled, the model will respond with transcriptions of your audio input in the `inputTranscriptions` property
+     * in {@link LiveServerContent} messages. Note that the transcriptions are broken up across
+     * messages, so you may only receive small amounts of text per message. For example, if you ask the model
+     * "How are you today?", the model may transcribe that input across three messages, broken up as "How a", "re yo", "u today?".
+     */
+    inputAudioTranscription?: AudioTranscriptionConfig;
+    /**
+     * Enables transcription of audio input.
+     *
+     * When enabled, the model will respond with transcriptions of its audio output in the `outputTranscription` property
+     * in {@link LiveServerContent} messages. Note that the transcriptions are broken up across
+     * messages, so you may only receive small amounts of text per message. For example, if the model says
+     * "How are you today?", the model may transcribe that output across three messages, broken up as "How a", "re yo", "u today?".
+     */
+    outputAudioTranscription?: AudioTranscriptionConfig;
 }
 
 /**
@@ -2078,6 +2126,14 @@ export declare interface LiveServerContent {
      * model was not interrupted.
      */
     interrupted?: boolean;
+    /**
+     * Transcription of the audio that was input to the model.
+     */
+    inputTranscription?: Transcription;
+    /**
+     * Transcription of the audio output from the model.
+     */
+    outputTranscription?: Transcription;
 }
 
 /**
@@ -2140,32 +2196,65 @@ export declare class LiveSession {
      */
     send(request: string | Array<string | Part>, turnComplete?: boolean): Promise<void>;
     /**
-     * Sends realtime input to the server.
+     * Sends text to the server in realtime.
      *
-     * @param mediaChunks - The media chunks to send.
+     * @example
+     * ```javascript
+     * liveSession.sendTextRealtime("Hello, how are you?");
+     * ```
+     *
+     * @param text - The text data to send.
      * @throws If this session has been closed.
      *
      * @beta
      */
-    sendMediaChunks(mediaChunks: GenerativeContentBlob[]): Promise<void>;
+    sendTextRealtime(text: string): Promise<void>;
     /**
-     * Sends function responses to the server.
+     * Sends audio data to the server in realtime.
      *
-     * @param functionResponses - The function responses to send.
+     * @remarks The server requires that the audio data is base64-encoded 16-bit PCM at 16kHz
+     * little-endian.
+     *
+     * @example
+     * ```javascript
+     * // const pcmData = ... base64-encoded 16-bit PCM at 16kHz little-endian.
+     * const blob = { mimeType: "audio/pcm", data: pcmData };
+     * liveSession.sendAudioRealtime(blob);
+     * ```
+     *
+     * @param blob - The base64-encoded PCM data to send to the server in realtime.
      * @throws If this session has been closed.
      *
      * @beta
      */
-    sendFunctionResponses(functionResponses: FunctionResponse[]): Promise<void>;
+    sendAudioRealtime(blob: GenerativeContentBlob): Promise<void>;
     /**
-     * Sends a stream of {@link GenerativeContentBlob}.
+     * Sends video data to the server in realtime.
      *
-     * @param mediaChunkStream - The stream of {@link GenerativeContentBlob} to send.
+     * @remarks The server requires that the video is sent as individual video frames at 1 FPS. It
+     * is recommended to set `mimeType` to `image/jpeg`.
+     *
+     * @example
+     * ```javascript
+     * // const videoFrame = ... base64-encoded JPEG data
+     * const blob = { mimeType: "image/jpeg", data: videoFrame };
+     * liveSession.sendVideoRealtime(blob);
+     * ```
+     * @param blob - The base64-encoded video data to send to the server in realtime.
      * @throws If this session has been closed.
      *
      * @beta
      */
-    sendMediaStream(mediaChunkStream: ReadableStream<GenerativeContentBlob>): Promise<void>;
+    sendVideoRealtime(blob: GenerativeContentBlob): Promise<void>;
+    /**
+     * Sends function responses to the server.
+     *
+     * @param functionResponses - The function responses to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    sendFunctionResponses(functionResponses: FunctionResponse[]): Promise<void>;
     /**
      * Yields messages received from the server.
      * This can only be used by one consumer at a time.
@@ -2183,6 +2272,28 @@ export declare class LiveSession {
      * @beta
      */
     close(): Promise<void>;
+    /**
+     * Sends realtime input to the server.
+     *
+     * @deprecated Use `sendTextRealtime()`, `sendAudioRealtime()`, and `sendVideoRealtime()` instead.
+     *
+     * @param mediaChunks - The media chunks to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    sendMediaChunks(mediaChunks: GenerativeContentBlob[]): Promise<void>;
+    /**
+     * @deprecated Use `sendTextRealtime()`, `sendAudioRealtime()`, and `sendVideoRealtime()` instead.
+     *
+     * Sends a stream of {@link GenerativeContentBlob}.
+     *
+     * @param mediaChunkStream - The stream of {@link GenerativeContentBlob} to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    sendMediaStream(mediaChunkStream: ReadableStream<GenerativeContentBlob>): Promise<void>;
 }
 
 /**
@@ -2874,6 +2985,20 @@ export declare interface ToolConfig {
     functionCallingConfig?: FunctionCallingConfig;
 }
 
+/**
+ * Transcription of audio. This can be returned from a {@link LiveGenerativeModel} if transcription
+ * is enabled with the `inputAudioTranscription` or `outputAudioTranscription` properties on
+ * the {@link LiveGenerationConfig}.
+ *
+ * @beta
+ */
+export declare interface Transcription {
+    /**
+     * The text transcription of the audio.
+     */
+    text?: string;
+}
+
 /**
  * A type that includes all specific Schema types.
  * @public

package/dist/ai.d.ts

@@ -232,6 +232,12 @@ export declare interface AudioConversationController {
     stop: () => Promise<void>;
 }
 
+/**
+ * The audio transcription configuration.
+ */
+export declare interface AudioTranscriptionConfig {
+}
+
 /**
  * Abstract base class representing the configuration for an AI service backend.
  * This class should not be instantiated directly. Use its subclasses; {@link GoogleAIBackend} for
@@ -375,6 +381,10 @@ export declare class ChatSession {
  * @beta
  */
 export declare interface ChromeAdapter {
+    /**
+     * @internal
+     */
+    mode: InferenceMode;
     /**
      * Checks if the on-device model is capable of handling a given
      * request.
@@ -604,6 +614,12 @@ export declare interface EnhancedGenerateContentResponse extends GenerateContent
      * set to `true`.
      */
     thoughtSummary: () => string | undefined;
+    /**
+     * Indicates whether inference happened on-device or in-cloud.
+     *
+     * @beta
+     */
+    inferenceSource?: InferenceSource;
 }
 
 /**
@@ -1945,6 +1961,23 @@ export declare const InferenceMode: {
  */
 export declare type InferenceMode = (typeof InferenceMode)[keyof typeof InferenceMode];
 
+/**
+ * Indicates whether inference happened on-device or in-cloud.
+ *
+ * @beta
+ */
+export declare const InferenceSource: {
+    readonly ON_DEVICE: "on_device";
+    readonly IN_CLOUD: "in_cloud";
+};
+
+/**
+ * Indicates whether inference happened on-device or in-cloud.
+ *
+ * @beta
+ */
+export declare type InferenceSource = (typeof InferenceSource)[keyof typeof InferenceSource];
+
 /**
  * Content part interface if the part represents an image.
  * @public
@@ -2112,6 +2145,24 @@ export declare interface LiveGenerationConfig {
      * The modalities of the response.
      */
     responseModalities?: ResponseModality[];
+    /**
+     * Enables transcription of audio input.
+     *
+     * When enabled, the model will respond with transcriptions of your audio input in the `inputTranscriptions` property
+     * in {@link LiveServerContent} messages. Note that the transcriptions are broken up across
+     * messages, so you may only receive small amounts of text per message. For example, if you ask the model
+     * "How are you today?", the model may transcribe that input across three messages, broken up as "How a", "re yo", "u today?".
+     */
+    inputAudioTranscription?: AudioTranscriptionConfig;
+    /**
+     * Enables transcription of audio input.
+     *
+     * When enabled, the model will respond with transcriptions of its audio output in the `outputTranscription` property
+     * in {@link LiveServerContent} messages. Note that the transcriptions are broken up across
+     * messages, so you may only receive small amounts of text per message. For example, if the model says
+     * "How are you today?", the model may transcribe that output across three messages, broken up as "How a", "re yo", "u today?".
+     */
+    outputAudioTranscription?: AudioTranscriptionConfig;
 }
 
 /**
@@ -2203,6 +2254,14 @@ export declare interface LiveServerContent {
      * model was not interrupted.
      */
     interrupted?: boolean;
+    /**
+     * Transcription of the audio that was input to the model.
+     */
+    inputTranscription?: Transcription;
+    /**
+     * Transcription of the audio output from the model.
+     */
+    outputTranscription?: Transcription;
 }
 
 /**
@@ -2268,32 +2327,65 @@ export declare class LiveSession {
      */
     send(request: string | Array<string | Part>, turnComplete?: boolean): Promise<void>;
     /**
-     * Sends realtime input to the server.
+     * Sends text to the server in realtime.
      *
-     * @param mediaChunks - The media chunks to send.
+     * @example
+     * ```javascript
+     * liveSession.sendTextRealtime("Hello, how are you?");
+     * ```
+     *
+     * @param text - The text data to send.
      * @throws If this session has been closed.
      *
      * @beta
      */
-    sendMediaChunks(mediaChunks: GenerativeContentBlob[]): Promise<void>;
+    sendTextRealtime(text: string): Promise<void>;
     /**
-     * Sends function responses to the server.
+     * Sends audio data to the server in realtime.
      *
-     * @param functionResponses - The function responses to send.
+     * @remarks The server requires that the audio data is base64-encoded 16-bit PCM at 16kHz
+     * little-endian.
+     *
+     * @example
+     * ```javascript
+     * // const pcmData = ... base64-encoded 16-bit PCM at 16kHz little-endian.
+     * const blob = { mimeType: "audio/pcm", data: pcmData };
+     * liveSession.sendAudioRealtime(blob);
+     * ```
+     *
+     * @param blob - The base64-encoded PCM data to send to the server in realtime.
      * @throws If this session has been closed.
      *
      * @beta
      */
-    sendFunctionResponses(functionResponses: FunctionResponse[]): Promise<void>;
+    sendAudioRealtime(blob: GenerativeContentBlob): Promise<void>;
     /**
-     * Sends a stream of {@link GenerativeContentBlob}.
+     * Sends video data to the server in realtime.
      *
-     * @param mediaChunkStream - The stream of {@link GenerativeContentBlob} to send.
+     * @remarks The server requires that the video is sent as individual video frames at 1 FPS. It
+     * is recommended to set `mimeType` to `image/jpeg`.
+     *
+     * @example
+     * ```javascript
+     * // const videoFrame = ... base64-encoded JPEG data
+     * const blob = { mimeType: "image/jpeg", data: videoFrame };
+     * liveSession.sendVideoRealtime(blob);
+     * ```
+     * @param blob - The base64-encoded video data to send to the server in realtime.
      * @throws If this session has been closed.
      *
      * @beta
      */
-    sendMediaStream(mediaChunkStream: ReadableStream<GenerativeContentBlob>): Promise<void>;
+    sendVideoRealtime(blob: GenerativeContentBlob): Promise<void>;
+    /**
+     * Sends function responses to the server.
+     *
+     * @param functionResponses - The function responses to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    sendFunctionResponses(functionResponses: FunctionResponse[]): Promise<void>;
     /**
      * Yields messages received from the server.
      * This can only be used by one consumer at a time.
@@ -2311,6 +2403,28 @@ export declare class LiveSession {
      * @beta
      */
     close(): Promise<void>;
+    /**
+     * Sends realtime input to the server.
+     *
+     * @deprecated Use `sendTextRealtime()`, `sendAudioRealtime()`, and `sendVideoRealtime()` instead.
+     *
+     * @param mediaChunks - The media chunks to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    sendMediaChunks(mediaChunks: GenerativeContentBlob[]): Promise<void>;
+    /**
+     * @deprecated Use `sendTextRealtime()`, `sendAudioRealtime()`, and `sendVideoRealtime()` instead.
+     *
+     * Sends a stream of {@link GenerativeContentBlob}.
+     *
+     * @param mediaChunkStream - The stream of {@link GenerativeContentBlob} to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    sendMediaStream(mediaChunkStream: ReadableStream<GenerativeContentBlob>): Promise<void>;
 }
 
 /**
@@ -3016,6 +3130,20 @@ export declare interface ToolConfig {
     functionCallingConfig?: FunctionCallingConfig;
 }
 
+/**
+ * Transcription of audio. This can be returned from a {@link LiveGenerativeModel} if transcription
+ * is enabled with the `inputAudioTranscription` or `outputAudioTranscription` properties on
+ * the {@link LiveGenerationConfig}.
+ *
+ * @beta
+ */
+export declare interface Transcription {
+    /**
+     * The text transcription of the audio.
+     */
+    text?: string;
+}
+
 /**
  * A type that includes all specific Schema types.
  * @public