@firebase/ai 2.1.0 → 2.2.0-canary.095c098de

package/dist/ai.d.ts

@@ -27,6 +27,10 @@ export declare interface AI {
      * Vertex AI Gemini API (using {@link VertexAIBackend}).
      */
     backend: Backend;
+    /**
+     * Options applied to this {@link AI} instance.
+     */
+    options?: AIOptions;
     /**
      * @deprecated use `AI.backend.location` instead.
      *
@@ -67,6 +71,8 @@ export declare const AIErrorCode: {
     readonly RESPONSE_ERROR: "response-error";
     /** An error occurred while performing a fetch. */
     readonly FETCH_ERROR: "fetch-error";
+    /** An error occurred because an operation was attempted on a closed session. */
+    readonly SESSION_CLOSED: "session-closed";
     /** An error associated with a Content object.  */
     readonly INVALID_CONTENT: "invalid-content";
     /** An error due to the Firebase API not being enabled in the Console. */
@@ -111,7 +117,7 @@ export declare abstract class AIModel {
     /**
      * @internal
      */
-    protected _apiSettings: ApiSettings;
+    _apiSettings: ApiSettings;
     /**
      * Constructs a new instance of the {@link AIModel} class.
      *
@@ -159,8 +165,13 @@ export declare abstract class AIModel {
 export declare interface AIOptions {
     /**
      * The backend configuration to use for the AI service instance.
+     * Defaults to the Gemini Developer API backend ({@link GoogleAIBackend}).
      */
-    backend: Backend;
+    backend?: Backend;
+    /**
+     * Whether to use App Check limited use tokens. Defaults to false.
+     */
+    useLimitedUseAppCheckTokens?: boolean;
 }
 
 /**
@@ -208,6 +219,19 @@ export declare class ArraySchema extends Schema {
     toJSON(): SchemaRequest;
 }
 
+/**
+ * A controller for managing an active audio conversation.
+ *
+ * @beta
+ */
+export declare interface AudioConversationController {
+    /**
+     * Stops the audio conversation, closes the microphone connection, and
+     * cleans up resources. Returns a promise that resolves when cleanup is complete.
+     */
+    stop: () => Promise<void>;
+}
+
 /**
  * Abstract base class representing the configuration for an AI service backend.
  * This class should not be instantiated directly. Use its subclasses; {@link GoogleAIBackend} for
@@ -360,16 +384,18 @@ export declare interface ChromeAdapter {
     /**
      * Generates content using on-device inference.
      *
-     * <p>This is comparable to {@link GenerativeModel.generateContent} for generating
-     * content using in-cloud inference.</p>
+     * @remarks
+     * This is comparable to {@link GenerativeModel.generateContent} for generating
+     * content using in-cloud inference.
      * @param request - a standard Firebase AI {@link GenerateContentRequest}
      */
     generateContent(request: GenerateContentRequest): Promise<Response>;
     /**
      * Generates a content stream using on-device inference.
      *
-     * <p>This is comparable to {@link GenerativeModel.generateContentStream} for generating
-     * a content stream using in-cloud inference.</p>
+     * @remarks
+     * This is comparable to {@link GenerativeModel.generateContentStream} for generating
+     * a content stream using in-cloud inference.
      * @param request - a standard Firebase AI {@link GenerateContentRequest}
      */
     generateContentStream(request: GenerateContentRequest): Promise<Response>;
@@ -500,15 +526,34 @@ export declare interface EnhancedGenerateContentResponse extends GenerateContent
      */
     text: () => string;
     /**
-     * Aggregates and returns all {@link InlineDataPart}s from the {@link GenerateContentResponse}'s
-     * first candidate.
-     *
-     * @returns An array of {@link InlineDataPart}s containing data from the response, if available.
+     * Aggregates and returns every {@link InlineDataPart} from the first candidate of
+     * {@link GenerateContentResponse}.
      *
      * @throws If the prompt or candidate was blocked.
      */
     inlineDataParts: () => InlineDataPart[] | undefined;
+    /**
+     * Aggregates and returns every {@link FunctionCall} from the first candidate of
+     * {@link GenerateContentResponse}.
+     *
+     * @throws If the prompt or candidate was blocked.
+     */
     functionCalls: () => FunctionCall[] | undefined;
+    /**
+     * Aggregates and returns every {@link TextPart} with their `thought` property set
+     * to `true` from the first candidate of {@link GenerateContentResponse}.
+     *
+     * @throws If the prompt or candidate was blocked.
+     *
+     * @remarks
+     * Thought summaries provide a brief overview of the model's internal thinking process,
+     * offering insight into how it arrived at the final answer. This can be useful for
+     * debugging, understanding the model's reasoning, and verifying its accuracy.
+     *
+     * Thoughts will only be included if {@link ThinkingConfig.includeThoughts} is
+     * set to `true`.
+     */
+    thoughtSummary: () => string | undefined;
 }
 
 /**
@@ -547,6 +592,11 @@ export declare interface FileDataPart {
     functionCall?: never;
     functionResponse?: never;
     fileData: FileData;
+    thought?: boolean;
+    /**
+     * @internal
+     */
+    thoughtSignature?: never;
 }
 
 /**
@@ -605,6 +655,15 @@ export declare type FinishReason = (typeof FinishReason)[keyof typeof FinishReas
  * @public
  */
 export declare interface FunctionCall {
+    /**
+     * The id of the function call. This must be sent back in the associated {@link FunctionResponse}.
+     *
+     *
+     * @remarks This property is only supported in the Gemini Developer API ({@link GoogleAIBackend}).
+     * When using the Gemini Developer API ({@link GoogleAIBackend}), this property will be
+     * `undefined`.
+     */
+    id?: string;
     name: string;
     args: object;
 }
@@ -654,6 +713,11 @@ export declare interface FunctionCallPart {
     inlineData?: never;
     functionCall: FunctionCall;
     functionResponse?: never;
+    thought?: boolean;
+    /**
+     * @internal
+     */
+    thoughtSignature?: never;
 }
 
 /**
@@ -715,6 +779,14 @@ export declare interface FunctionDeclarationsTool {
  * @public
  */
 export declare interface FunctionResponse {
+    /**
+     * The id of the {@link FunctionCall}.
+     *
+     * @remarks This property is only supported in the Gemini Developer API ({@link GoogleAIBackend}).
+     * When using the Gemini Developer API ({@link GoogleAIBackend}), this property will be
+     * `undefined`.
+     */
+    id?: string;
     name: string;
     response: object;
 }
@@ -728,6 +800,11 @@ export declare interface FunctionResponsePart {
     inlineData?: never;
     functionCall?: never;
     functionResponse: FunctionResponse;
+    thought?: boolean;
+    /**
+     * @internal
+     */
+    thoughtSignature?: never;
 }
 
 /**
@@ -937,6 +1014,20 @@ export declare function getGenerativeModel(ai: AI, modelParams: ModelParams | Hy
  */
 export declare function getImagenModel(ai: AI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel;
 
+/**
+ * Returns a {@link LiveGenerativeModel} class for real-time, bidirectional communication.
+ *
+ * The Live API is only supported in modern browser windows and Node >= 22.
+ *
+ * @param ai - An {@link AI} instance.
+ * @param modelParams - Parameters to use when setting up a {@link LiveSession}.
+ * @throws If the `apiKey` or `projectId` fields are missing in your
+ * Firebase config.
+ *
+ * @beta
+ */
+export declare function getLiveGenerativeModel(ai: AI, modelParams: LiveModelParams): LiveGenerativeModel;
+
 /**
  * Configuration class for the Gemini Developer API.
  *
@@ -1759,6 +1850,11 @@ export declare interface InlineDataPart {
      * Applicable if `inlineData` is a video.
      */
     videoMetadata?: VideoMetadata;
+    thought?: boolean;
+    /**
+     * @internal
+     */
+    thoughtSignature?: never;
 }
 
 /**
@@ -1849,6 +1945,247 @@ export declare interface LanguageModelPromptOptions {
     responseConstraint?: object;
 }
 
+/**
+ * Configuration parameters used by {@link LiveGenerativeModel} to control live content generation.
+ *
+ * @beta
+ */
+export declare interface LiveGenerationConfig {
+    /**
+     * Configuration for speech synthesis.
+     */
+    speechConfig?: SpeechConfig;
+    /**
+     * Specifies the maximum number of tokens that can be generated in the response. The number of
+     * tokens per word varies depending on the language outputted. Is unbounded by default.
+     */
+    maxOutputTokens?: number;
+    /**
+     * Controls the degree of randomness in token selection. A `temperature` value of 0 means that the highest
+     * probability tokens are always selected. In this case, responses for a given prompt are mostly
+     * deterministic, but a small amount of variation is still possible.
+     */
+    temperature?: number;
+    /**
+     * Changes how the model selects tokens for output. Tokens are
+     * selected from the most to least probable until the sum of their probabilities equals the `topP`
+     * value. For example, if tokens A, B, and C have probabilities of 0.3, 0.2, and 0.1 respectively
+     * and the `topP` value is 0.5, then the model will select either A or B as the next token by using
+     * the `temperature` and exclude C as a candidate. Defaults to 0.95 if unset.
+     */
+    topP?: number;
+    /**
+     * Changes how the model selects token for output. A `topK` value of 1 means the select token is
+     * the most probable among all tokens in the model's vocabulary, while a `topK` value 3 means that
+     * the next token is selected from among the 3 most probably using probabilities sampled. Tokens
+     * are then further filtered with the highest selected `temperature` sampling. Defaults to 40
+     * if unspecified.
+     */
+    topK?: number;
+    /**
+     * Positive penalties.
+     */
+    presencePenalty?: number;
+    /**
+     * Frequency penalties.
+     */
+    frequencyPenalty?: number;
+    /**
+     * The modalities of the response.
+     */
+    responseModalities?: ResponseModality[];
+}
+
+/**
+ * Class for Live generative model APIs. The Live API enables low-latency, two-way multimodal
+ * interactions with Gemini.
+ *
+ * This class should only be instantiated with {@link getLiveGenerativeModel}.
+ *
+ * @beta
+ */
+export declare class LiveGenerativeModel extends AIModel {
+    /**
+     * @internal
+     */
+    private _webSocketHandler;
+    generationConfig: LiveGenerationConfig;
+    tools?: Tool[];
+    toolConfig?: ToolConfig;
+    systemInstruction?: Content;
+    /**
+     * @internal
+     */
+    constructor(ai: AI, modelParams: LiveModelParams, 
+    /**
+     * @internal
+     */
+    _webSocketHandler: WebSocketHandler);
+    /**
+     * Starts a {@link LiveSession}.
+     *
+     * @returns A {@link LiveSession}.
+     * @throws If the connection failed to be established with the server.
+     *
+     * @beta
+     */
+    connect(): Promise<LiveSession>;
+}
+
+/**
+ * Params passed to {@link getLiveGenerativeModel}.
+ * @beta
+ */
+export declare interface LiveModelParams {
+    model: string;
+    generationConfig?: LiveGenerationConfig;
+    tools?: Tool[];
+    toolConfig?: ToolConfig;
+    systemInstruction?: string | Part | Content;
+}
+
+/**
+ * The types of responses that can be returned by {@link LiveSession.receive}.
+ *
+ * @beta
+ */
+export declare const LiveResponseType: {
+    SERVER_CONTENT: string;
+    TOOL_CALL: string;
+    TOOL_CALL_CANCELLATION: string;
+};
+
+/**
+ * The types of responses that can be returned by {@link LiveSession.receive}.
+ * This is a property on all messages that can be used for type narrowing. This property is not
+ * returned by the server, it is assigned to a server message object once it's parsed.
+ *
+ * @beta
+ */
+export declare type LiveResponseType = (typeof LiveResponseType)[keyof typeof LiveResponseType];
+
+/**
+ * An incremental content update from the model.
+ *
+ * @beta
+ */
+export declare interface LiveServerContent {
+    type: 'serverContent';
+    /**
+     * The content that the model has generated as part of the current conversation with the user.
+     */
+    modelTurn?: Content;
+    /**
+     * Indicates whether the turn is complete. This is `undefined` if the turn is not complete.
+     */
+    turnComplete?: boolean;
+    /**
+     * Indicates whether the model was interrupted by the client. An interruption occurs when
+     * the client sends a message before the model finishes it's turn. This is `undefined` if the
+     * model was not interrupted.
+     */
+    interrupted?: boolean;
+}
+
+/**
+ * A request from the model for the client to execute one or more functions.
+ *
+ * @beta
+ */
+export declare interface LiveServerToolCall {
+    type: 'toolCall';
+    /**
+     * An array of function calls to run.
+     */
+    functionCalls: FunctionCall[];
+}
+
+/**
+ * Notification to cancel a previous function call triggered by {@link LiveServerToolCall}.
+ *
+ * @beta
+ */
+export declare interface LiveServerToolCallCancellation {
+    type: 'toolCallCancellation';
+    /**
+     * IDs of function calls that were cancelled. These refer to the `id` property of a {@link FunctionCall}.
+     */
+    functionIds: string[];
+}
+
+/**
+ * Represents an active, real-time, bidirectional conversation with the model.
+ *
+ * This class should only be instantiated by calling {@link LiveGenerativeModel.connect}.
+ *
+ * @beta
+ */
+export declare class LiveSession {
+    private webSocketHandler;
+    private serverMessages;
+    /**
+     * Indicates whether this Live session is closed.
+     *
+     * @beta
+     */
+    isClosed: boolean;
+    /**
+     * Indicates whether this Live session is being controlled by an `AudioConversationController`.
+     *
+     * @beta
+     */
+    inConversation: boolean;
+    /**
+     * @internal
+     */
+    constructor(webSocketHandler: WebSocketHandler, serverMessages: AsyncGenerator<unknown>);
+    /**
+     * Sends content to the server.
+     *
+     * @param request - The message to send to the model.
+     * @param turnComplete - Indicates if the turn is complete. Defaults to false.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    send(request: string | Array<string | Part>, turnComplete?: boolean): Promise<void>;
+    /**
+     * Sends realtime input to the server.
+     *
+     * @param mediaChunks - The media chunks to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    sendMediaChunks(mediaChunks: GenerativeContentBlob[]): Promise<void>;
+    /**
+     * Sends a stream of {@link GenerativeContentBlob}.
+     *
+     * @param mediaChunkStream - The stream of {@link GenerativeContentBlob} to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    sendMediaStream(mediaChunkStream: ReadableStream<GenerativeContentBlob>): Promise<void>;
+    /**
+     * Yields messages received from the server.
+     * This can only be used by one consumer at a time.
+     *
+     * @returns An `AsyncGenerator` that yields server messages as they arrive.
+     * @throws If the session is already closed, or if we receive a response that we don't support.
+     *
+     * @beta
+     */
+    receive(): AsyncGenerator<LiveServerContent | LiveServerToolCall | LiveServerToolCallCancellation>;
+    /**
+     * Closes this session.
+     * All methods on this session will throw an error once this resolves.
+     *
+     * @beta
+     */
+    close(): Promise<void>;
+}
+
 /**
  * Content part modality.
  * @public
@@ -1977,6 +2314,20 @@ export declare type Part = TextPart | InlineDataPart | FunctionCallPart | Functi
  */
 export declare const POSSIBLE_ROLES: readonly ["user", "model", "function", "system"];
 
+/**
+ * Configuration for a pre-built voice.
+ *
+ * @beta
+ */
+export declare interface PrebuiltVoiceConfig {
+    /**
+     * The voice name to use for speech synthesis.
+     *
+     * For a full list of names and demos of what each voice sounds like, see {@link https://cloud.google.com/text-to-speech/docs/chirp3-hd | Chirp 3: HD Voices}.
+     */
+    voiceName?: string;
+}
+
 /**
  * If the prompt was blocked, this will be populated with `blockReason` and
  * the relevant `safetyRatings`.
@@ -2003,7 +2354,10 @@ export declare interface RequestOptions {
      */
     timeout?: number;
     /**
-     * Base url for endpoint. Defaults to https://firebasevertexai.googleapis.com
+     * Base url for endpoint. Defaults to
+     * https://firebasevertexai.googleapis.com, which is the
+     * {@link https://console.cloud.google.com/apis/library/firebasevertexai.googleapis.com?project=_ | Firebase AI Logic API}
+     * (used regardless of your chosen Gemini API provider).
      */
     baseUrl?: string;
 }
@@ -2024,6 +2378,11 @@ export declare const ResponseModality: {
      * @beta
      */
     readonly IMAGE: "IMAGE";
+    /**
+     * Audio.
+     * @beta
+     */
+    readonly AUDIO: "AUDIO";
 };
 
 /**
@@ -2347,6 +2706,80 @@ export declare interface Segment {
     text: string;
 }
 
+/**
+ * Configures speech synthesis.
+ *
+ * @beta
+ */
+export declare interface SpeechConfig {
+    /**
+     * Configures the voice to be used in speech synthesis.
+     */
+    voiceConfig?: VoiceConfig;
+}
+
+/**
+ * Starts a real-time, bidirectional audio conversation with the model. This helper function manages
+ * the complexities of microphone access, audio recording, playback, and interruptions.
+ *
+ * @remarks Important: This function must be called in response to a user gesture
+ * (for example, a button click) to comply with {@link https://developer.mozilla.org/en-US/docs/Web/API/Web_Audio_API/Best_practices#autoplay_policy | browser autoplay policies}.
+ *
+ * @example
+ * ```javascript
+ * const liveSession = await model.connect();
+ * let conversationController;
+ *
+ * // This function must be called from within a click handler.
+ * async function startConversation() {
+ *   try {
+ *     conversationController = await startAudioConversation(liveSession);
+ *   } catch (e) {
+ *     // Handle AI-specific errors
+ *     if (e instanceof AIError) {
+ *       console.error("AI Error:", e.message);
+ *     }
+ *     // Handle microphone permission and hardware errors
+ *     else if (e instanceof DOMException) {
+ *       console.error("Microphone Error:", e.message);
+ *     }
+ *     // Handle other unexpected errors
+ *     else {
+ *       console.error("An unexpected error occurred:", e);
+ *     }
+ *   }
+ * }
+ *
+ * // Later, to stop the conversation:
+ * // if (conversationController) {
+ * //   await conversationController.stop();
+ * // }
+ * ```
+ *
+ * @param liveSession - An active {@link LiveSession} instance.
+ * @param options - Configuration options for the audio conversation.
+ * @returns A `Promise` that resolves with an {@link AudioConversationController}.
+ * @throws `AIError` if the environment does not support required Web APIs (`UNSUPPORTED`), if a conversation is already active (`REQUEST_ERROR`), the session is closed (`SESSION_CLOSED`), or if an unexpected initialization error occurs (`ERROR`).
+ * @throws `DOMException` Thrown by `navigator.mediaDevices.getUserMedia()` if issues occur with microphone access, such as permissions being denied (`NotAllowedError`) or no compatible hardware being found (`NotFoundError`). See the {@link https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia#exceptions | MDN documentation} for a full list of exceptions.
+ *
+ * @beta
+ */
+export declare function startAudioConversation(liveSession: LiveSession, options?: StartAudioConversationOptions): Promise<AudioConversationController>;
+
+/**
+ * Options for {@link startAudioConversation}.
+ *
+ * @beta
+ */
+export declare interface StartAudioConversationOptions {
+    /**
+     * An async handler that is called when the model requests a function to be executed.
+     * The handler should perform the function call and return the result as a `Part`,
+     * which will then be sent back to the model.
+     */
+    functionCallingHandler?: (functionCalls: LiveServerToolCall['functionCalls']) => Promise<Part>;
+}
+
 /**
  * Params for {@link GenerativeModel.startChat}.
  * @public
@@ -2381,6 +2814,11 @@ export declare interface TextPart {
     inlineData?: never;
     functionCall?: never;
     functionResponse?: never;
+    thought?: boolean;
+    /**
+     * @internal
+     */
+    thoughtSignature?: string;
 }
 
 /**
@@ -2406,6 +2844,15 @@ export declare interface ThinkingConfig {
      * feature or if the specified budget is not within the model's supported range.
      */
     thinkingBudget?: number;
+    /**
+     * Whether to include "thought summaries" in the model's response.
+     *
+     * @remarks
+     * Thought summaries provide a brief overview of the model's internal thinking process,
+     * offering insight into how it arrived at the final answer. This can be useful for
+     * debugging, understanding the model's reasoning, and verifying its accuracy.
+     */
+    includeThoughts?: boolean;
 }
 
 /**
@@ -2487,6 +2934,18 @@ export declare interface VideoMetadata {
     endOffset: string;
 }
 
+/**
+ * Configuration for the voice to used in speech synthesis.
+ *
+ * @beta
+ */
+export declare interface VoiceConfig {
+    /**
+     * Configures the voice using a pre-built voice configuration.
+     */
+    prebuiltVoiceConfig?: PrebuiltVoiceConfig;
+}
+
 /**
  * @public
  */
@@ -2522,4 +2981,59 @@ export declare interface WebGroundingChunk {
     domain?: string;
 }
 
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * A standardized interface for interacting with a WebSocket connection.
+ * This abstraction allows the SDK to use the appropriate WebSocket implementation
+ * for the current JS environment (Browser vs. Node) without
+ * changing the core logic of the `LiveSession`.
+ * @internal
+ */
+declare interface WebSocketHandler {
+    /**
+     * Establishes a connection to the given URL.
+     *
+     * @param url The WebSocket URL (e.g., wss://...).
+     * @returns A promise that resolves on successful connection or rejects on failure.
+     */
+    connect(url: string): Promise<void>;
+    /**
+     * Sends data over the WebSocket.
+     *
+     * @param data The string or binary data to send.
+     */
+    send(data: string | ArrayBuffer): void;
+    /**
+     * Returns an async generator that yields parsed JSON objects from the server.
+     * The yielded type is `unknown` because the handler cannot guarantee the shape of the data.
+     * The consumer is responsible for type validation.
+     * The generator terminates when the connection is closed.
+     *
+     * @returns A generator that allows consumers to pull messages using a `for await...of` loop.
+     */
+    listen(): AsyncGenerator<unknown>;
+    /**
+     * Closes the WebSocket connection.
+     *
+     * @param code - A numeric status code explaining why the connection is closing.
+     * @param reason - A human-readable string explaining why the connection is closing.
+     */
+    close(code?: number, reason?: string): Promise<void>;
+}
+
 export { }