@firebase/ai 2.1.0-canary.84b8bed35 → 2.1.0-canary.984086b0b

package/dist/ai.d.ts

@@ -79,6 +79,8 @@ export declare const AIErrorCode: {
     readonly RESPONSE_ERROR: "response-error";
     /** An error occurred while performing a fetch. */
     readonly FETCH_ERROR: "fetch-error";
+    /** An error occurred because an operation was attempted on a closed session. */
+    readonly SESSION_CLOSED: "session-closed";
     /** An error associated with a Content object.  */
     readonly INVALID_CONTENT: "invalid-content";
     /** An error due to the Firebase API not being enabled in the Console. */
@@ -183,11 +185,12 @@ export declare interface AIOptions {
 declare class AIService implements AI, _FirebaseService {
     app: FirebaseApp;
     backend: Backend;
+    chromeAdapterFactory?: ((mode: InferenceMode, window?: Window, params?: OnDeviceParams) => ChromeAdapterImpl | undefined) | undefined;
     auth: FirebaseAuthInternal | null;
     appCheck: FirebaseAppCheckInternal | null;
     _options?: Omit<AIOptions, 'backend'>;
     location: string;
-    constructor(app: FirebaseApp, backend: Backend, authProvider?: Provider<FirebaseAuthInternalName>, appCheckProvider?: Provider<AppCheckInternalComponentName>);
+    constructor(app: FirebaseApp, backend: Backend, authProvider?: Provider<FirebaseAuthInternalName>, appCheckProvider?: Provider<AppCheckInternalComponentName>, chromeAdapterFactory?: ((mode: InferenceMode, window?: Window, params?: OnDeviceParams) => ChromeAdapterImpl | undefined) | undefined);
     _delete(): Promise<void>;
     set options(optionsToSet: AIOptions);
     get options(): AIOptions | undefined;
@@ -238,6 +241,29 @@ export declare class ArraySchema extends Schema {
     toJSON(): SchemaRequest;
 }
 
+/**
+ * A controller for managing an active audio conversation.
+ *
+ * @beta
+ */
+export declare interface AudioConversationController {
+    /**
+     * Stops the audio conversation, closes the microphone connection, and
+     * cleans up resources. Returns a promise that resolves when cleanup is complete.
+     */
+    stop: () => Promise<void>;
+}
+
+/**
+ * @internal
+ */
+declare enum Availability {
+    'UNAVAILABLE' = "unavailable",
+    'DOWNLOADABLE' = "downloadable",
+    'DOWNLOADING' = "downloading",
+    'AVAILABLE' = "available"
+}
+
 /**
  * Abstract base class representing the configuration for an AI service backend.
  * This class should not be instantiated directly. Use its subclasses; {@link GoogleAIBackend} for
@@ -411,6 +437,108 @@ export declare interface ChromeAdapter {
     countTokens(request: CountTokensRequest): Promise<Response>;
 }
 
+/**
+ * Defines an inference "backend" that uses Chrome's on-device model,
+ * and encapsulates logic for detecting when on-device inference is
+ * possible.
+ */
+declare class ChromeAdapterImpl implements ChromeAdapter {
+    languageModelProvider: LanguageModel;
+    mode: InferenceMode;
+    onDeviceParams: OnDeviceParams;
+    static SUPPORTED_MIME_TYPES: string[];
+    private isDownloading;
+    private downloadPromise;
+    private oldSession;
+    constructor(languageModelProvider: LanguageModel, mode: InferenceMode, onDeviceParams?: OnDeviceParams);
+    /**
+     * Checks if a given request can be made on-device.
+     *
+     * Encapsulates a few concerns:
+     *   the mode
+     *   API existence
+     *   prompt formatting
+     *   model availability, including triggering download if necessary
+     *
+     *
+     * Pros: callers needn't be concerned with details of on-device availability.</p>
+     * Cons: this method spans a few concerns and splits request validation from usage.
+     * If instance variables weren't already part of the API, we could consider a better
+     * separation of concerns.
+     */
+    isAvailable(request: GenerateContentRequest): Promise<boolean>;
+    /**
+     * Generates content on device.
+     *
+     * @remarks
+     * This is comparable to {@link GenerativeModel.generateContent} for generating content in
+     * Cloud.
+     * @param request - a standard Firebase AI {@link GenerateContentRequest}
+     * @returns {@link Response}, so we can reuse common response formatting.
+     */
+    generateContent(request: GenerateContentRequest): Promise<Response>;
+    /**
+     * Generates content stream on device.
+     *
+     * @remarks
+     * This is comparable to {@link GenerativeModel.generateContentStream} for generating content in
+     * Cloud.
+     * @param request - a standard Firebase AI {@link GenerateContentRequest}
+     * @returns {@link Response}, so we can reuse common response formatting.
+     */
+    generateContentStream(request: GenerateContentRequest): Promise<Response>;
+    countTokens(_request: CountTokensRequest): Promise<Response>;
+    /**
+     * Asserts inference for the given request can be performed by an on-device model.
+     */
+    private static isOnDeviceRequest;
+    /**
+     * Encapsulates logic to get availability and download a model if one is downloadable.
+     */
+    private downloadIfAvailable;
+    /**
+     * Triggers out-of-band download of an on-device model.
+     *
+     * Chrome only downloads models as needed. Chrome knows a model is needed when code calls
+     * LanguageModel.create.
+     *
+     * Since Chrome manages the download, the SDK can only avoid redundant download requests by
+     * tracking if a download has previously been requested.
+     */
+    private download;
+    /**
+     * Converts Firebase AI {@link Content} object to a Chrome {@link LanguageModelMessage} object.
+     */
+    private static toLanguageModelMessage;
+    /**
+     * Converts a Firebase AI Part object to a Chrome LanguageModelMessageContent object.
+     */
+    private static toLanguageModelMessageContent;
+    /**
+     * Converts a Firebase AI {@link Role} string to a {@link LanguageModelMessageRole} string.
+     */
+    private static toLanguageModelMessageRole;
+    /**
+     * Abstracts Chrome session creation.
+     *
+     * Chrome uses a multi-turn session for all inference. Firebase AI uses single-turn for all
+     * inference. To map the Firebase AI API to Chrome's API, the SDK creates a new session for all
+     * inference.
+     *
+     * Chrome will remove a model from memory if it's no longer in use, so this method ensures a
+     * new session is created before an old session is destroyed.
+     */
+    private createSession;
+    /**
+     * Formats string returned by Chrome as a {@link Response} returned by Firebase AI.
+     */
+    private static toResponse;
+    /**
+     * Formats string stream returned by Chrome as SSE returned by Firebase AI.
+     */
+    private static toStreamResponse;
+}
+
 /**
  * A single citation.
  * @public
@@ -663,6 +791,15 @@ export declare type FinishReason = (typeof FinishReason)[keyof typeof FinishReas
  * @public
  */
 export declare interface FunctionCall {
+    /**
+     * The id of the function call. This must be sent back in the associated {@link FunctionResponse}.
+     *
+     *
+     * @remarks This property is only supported in the Gemini Developer API ({@link GoogleAIBackend}).
+     * When using the Gemini Developer API ({@link GoogleAIBackend}), this property will be
+     * `undefined`.
+     */
+    id?: string;
     name: string;
     args: object;
 }
@@ -778,6 +915,14 @@ export declare interface FunctionDeclarationsTool {
  * @public
  */
 export declare interface FunctionResponse {
+    /**
+     * The id of the {@link FunctionCall}.
+     *
+     * @remarks This property is only supported in the Gemini Developer API ({@link GoogleAIBackend}).
+     * When using the Gemini Developer API ({@link GoogleAIBackend}), this property will be
+     * `undefined`.
+     */
+    id?: string;
     name: string;
     response: object;
 }
@@ -1005,6 +1150,20 @@ export declare function getGenerativeModel(ai: AI, modelParams: ModelParams | Hy
  */
 export declare function getImagenModel(ai: AI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel;
 
+/**
+ * Returns a {@link LiveGenerativeModel} class for real-time, bidirectional communication.
+ *
+ * The Live API is only supported in modern browser windows and Node >= 22.
+ *
+ * @param ai - An {@link AI} instance.
+ * @param modelParams - Parameters to use when setting up a {@link LiveSession}.
+ * @throws If the `apiKey` or `projectId` fields are missing in your
+ * Firebase config.
+ *
+ * @beta
+ */
+export declare function getLiveGenerativeModel(ai: AI, modelParams: LiveModelParams): LiveGenerativeModel;
+
 /**
  * Configuration class for the Gemini Developer API.
  *
@@ -1842,6 +2001,38 @@ export declare class IntegerSchema extends Schema {
     constructor(schemaParams?: SchemaParams);
 }
 
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * The subset of the Prompt API
+ * (see {@link https://github.com/webmachinelearning/prompt-api#full-api-surface-in-web-idl }
+ * required for hybrid functionality.
+ *
+ * @internal
+ */
+declare interface LanguageModel extends EventTarget {
+    create(options?: LanguageModelCreateOptions): Promise<LanguageModel>;
+    availability(options?: LanguageModelCreateCoreOptions): Promise<Availability>;
+    prompt(input: LanguageModelPrompt, options?: LanguageModelPromptOptions): Promise<string>;
+    promptStreaming(input: LanguageModelPrompt, options?: LanguageModelPromptOptions): ReadableStream;
+    measureInputUsage(input: LanguageModelPrompt, options?: LanguageModelPromptOptions): Promise<number>;
+    destroy(): undefined;
+}
+
 /**
  * <b>(EXPERIMENTAL)</b>
  * Configures the creation of an on-device language model session.
@@ -1913,6 +2104,13 @@ export declare type LanguageModelMessageRole = 'system' | 'user' | 'assistant';
  */
 export declare type LanguageModelMessageType = 'text' | 'image' | 'audio';
 
+/**
+ * <b>(EXPERIMENTAL)</b>
+ * An on-device language model prompt.
+ * @public
+ */
+declare type LanguageModelPrompt = LanguageModelMessage[];
+
 /**
  * <b>(EXPERIMENTAL)</b>
  * Options for an on-device language model prompt.
@@ -1922,6 +2120,247 @@ export declare interface LanguageModelPromptOptions {
     responseConstraint?: object;
 }
 
+/**
+ * Configuration parameters used by {@link LiveGenerativeModel} to control live content generation.
+ *
+ * @beta
+ */
+export declare interface LiveGenerationConfig {
+    /**
+     * Configuration for speech synthesis.
+     */
+    speechConfig?: SpeechConfig;
+    /**
+     * Specifies the maximum number of tokens that can be generated in the response. The number of
+     * tokens per word varies depending on the language outputted. Is unbounded by default.
+     */
+    maxOutputTokens?: number;
+    /**
+     * Controls the degree of randomness in token selection. A `temperature` value of 0 means that the highest
+     * probability tokens are always selected. In this case, responses for a given prompt are mostly
+     * deterministic, but a small amount of variation is still possible.
+     */
+    temperature?: number;
+    /**
+     * Changes how the model selects tokens for output. Tokens are
+     * selected from the most to least probable until the sum of their probabilities equals the `topP`
+     * value. For example, if tokens A, B, and C have probabilities of 0.3, 0.2, and 0.1 respectively
+     * and the `topP` value is 0.5, then the model will select either A or B as the next token by using
+     * the `temperature` and exclude C as a candidate. Defaults to 0.95 if unset.
+     */
+    topP?: number;
+    /**
+     * Changes how the model selects token for output. A `topK` value of 1 means the select token is
+     * the most probable among all tokens in the model's vocabulary, while a `topK` value 3 means that
+     * the next token is selected from among the 3 most probably using probabilities sampled. Tokens
+     * are then further filtered with the highest selected `temperature` sampling. Defaults to 40
+     * if unspecified.
+     */
+    topK?: number;
+    /**
+     * Positive penalties.
+     */
+    presencePenalty?: number;
+    /**
+     * Frequency penalties.
+     */
+    frequencyPenalty?: number;
+    /**
+     * The modalities of the response.
+     */
+    responseModalities?: ResponseModality[];
+}
+
+/**
+ * Class for Live generative model APIs. The Live API enables low-latency, two-way multimodal
+ * interactions with Gemini.
+ *
+ * This class should only be instantiated with {@link getLiveGenerativeModel}.
+ *
+ * @beta
+ */
+export declare class LiveGenerativeModel extends AIModel {
+    /**
+     * @internal
+     */
+    private _webSocketHandler;
+    generationConfig: LiveGenerationConfig;
+    tools?: Tool[];
+    toolConfig?: ToolConfig;
+    systemInstruction?: Content;
+    /**
+     * @internal
+     */
+    constructor(ai: AI, modelParams: LiveModelParams, 
+    /**
+     * @internal
+     */
+    _webSocketHandler: WebSocketHandler);
+    /**
+     * Starts a {@link LiveSession}.
+     *
+     * @returns A {@link LiveSession}.
+     * @throws If the connection failed to be established with the server.
+     *
+     * @beta
+     */
+    connect(): Promise<LiveSession>;
+}
+
+/**
+ * Params passed to {@link getLiveGenerativeModel}.
+ * @beta
+ */
+export declare interface LiveModelParams {
+    model: string;
+    generationConfig?: LiveGenerationConfig;
+    tools?: Tool[];
+    toolConfig?: ToolConfig;
+    systemInstruction?: string | Part | Content;
+}
+
+/**
+ * The types of responses that can be returned by {@link LiveSession.receive}.
+ *
+ * @beta
+ */
+export declare const LiveResponseType: {
+    SERVER_CONTENT: string;
+    TOOL_CALL: string;
+    TOOL_CALL_CANCELLATION: string;
+};
+
+/**
+ * The types of responses that can be returned by {@link LiveSession.receive}.
+ * This is a property on all messages that can be used for type narrowing. This property is not
+ * returned by the server, it is assigned to a server message object once it's parsed.
+ *
+ * @beta
+ */
+export declare type LiveResponseType = (typeof LiveResponseType)[keyof typeof LiveResponseType];
+
+/**
+ * An incremental content update from the model.
+ *
+ * @beta
+ */
+export declare interface LiveServerContent {
+    type: 'serverContent';
+    /**
+     * The content that the model has generated as part of the current conversation with the user.
+     */
+    modelTurn?: Content;
+    /**
+     * Indicates whether the turn is complete. This is `undefined` if the turn is not complete.
+     */
+    turnComplete?: boolean;
+    /**
+     * Indicates whether the model was interrupted by the client. An interruption occurs when
+     * the client sends a message before the model finishes it's turn. This is `undefined` if the
+     * model was not interrupted.
+     */
+    interrupted?: boolean;
+}
+
+/**
+ * A request from the model for the client to execute one or more functions.
+ *
+ * @beta
+ */
+export declare interface LiveServerToolCall {
+    type: 'toolCall';
+    /**
+     * An array of function calls to run.
+     */
+    functionCalls: FunctionCall[];
+}
+
+/**
+ * Notification to cancel a previous function call triggered by {@link LiveServerToolCall}.
+ *
+ * @beta
+ */
+export declare interface LiveServerToolCallCancellation {
+    type: 'toolCallCancellation';
+    /**
+     * IDs of function calls that were cancelled. These refer to the `id` property of a {@link FunctionCall}.
+     */
+    functionIds: string[];
+}
+
+/**
+ * Represents an active, real-time, bidirectional conversation with the model.
+ *
+ * This class should only be instantiated by calling {@link LiveGenerativeModel.connect}.
+ *
+ * @beta
+ */
+export declare class LiveSession {
+    private webSocketHandler;
+    private serverMessages;
+    /**
+     * Indicates whether this Live session is closed.
+     *
+     * @beta
+     */
+    isClosed: boolean;
+    /**
+     * Indicates whether this Live session is being controlled by an `AudioConversationController`.
+     *
+     * @beta
+     */
+    inConversation: boolean;
+    /**
+     * @internal
+     */
+    constructor(webSocketHandler: WebSocketHandler, serverMessages: AsyncGenerator<unknown>);
+    /**
+     * Sends content to the server.
+     *
+     * @param request - The message to send to the model.
+     * @param turnComplete - Indicates if the turn is complete. Defaults to false.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    send(request: string | Array<string | Part>, turnComplete?: boolean): Promise<void>;
+    /**
+     * Sends realtime input to the server.
+     *
+     * @param mediaChunks - The media chunks to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    sendMediaChunks(mediaChunks: GenerativeContentBlob[]): Promise<void>;
+    /**
+     * Sends a stream of {@link GenerativeContentBlob}.
+     *
+     * @param mediaChunkStream - The stream of {@link GenerativeContentBlob} to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    sendMediaStream(mediaChunkStream: ReadableStream<GenerativeContentBlob>): Promise<void>;
+    /**
+     * Yields messages received from the server.
+     * This can only be used by one consumer at a time.
+     *
+     * @returns An `AsyncGenerator` that yields server messages as they arrive.
+     * @throws If the session is already closed, or if we receive a response that we don't support.
+     *
+     * @beta
+     */
+    receive(): AsyncGenerator<LiveServerContent | LiveServerToolCall | LiveServerToolCallCancellation>;
+    /**
+     * Closes this session.
+     * All methods on this session will throw an error once this resolves.
+     *
+     * @beta
+     */
+    close(): Promise<void>;
+}
+
 /**
  * Content part modality.
  * @public
@@ -2050,6 +2489,20 @@ export declare type Part = TextPart | InlineDataPart | FunctionCallPart | Functi
  */
 export declare const POSSIBLE_ROLES: readonly ["user", "model", "function", "system"];
 
+/**
+ * Configuration for a pre-built voice.
+ *
+ * @beta
+ */
+export declare interface PrebuiltVoiceConfig {
+    /**
+     * The voice name to use for speech synthesis.
+     *
+     * For a full list of names and demos of what each voice sounds like, see {@link https://cloud.google.com/text-to-speech/docs/chirp3-hd | Chirp 3: HD Voices}.
+     */
+    voiceName?: string;
+}
+
 /**
  * If the prompt was blocked, this will be populated with `blockReason` and
  * the relevant `safetyRatings`.
@@ -2100,6 +2553,11 @@ export declare const ResponseModality: {
      * @beta
      */
     readonly IMAGE: "IMAGE";
+    /**
+     * Audio.
+     * @beta
+     */
+    readonly AUDIO: "AUDIO";
 };
 
 /**
@@ -2423,6 +2881,80 @@ export declare interface Segment {
     text: string;
 }
 
+/**
+ * Configures speech synthesis.
+ *
+ * @beta
+ */
+export declare interface SpeechConfig {
+    /**
+     * Configures the voice to be used in speech synthesis.
+     */
+    voiceConfig?: VoiceConfig;
+}
+
+/**
+ * Starts a real-time, bidirectional audio conversation with the model. This helper function manages
+ * the complexities of microphone access, audio recording, playback, and interruptions.
+ *
+ * @remarks Important: This function must be called in response to a user gesture
+ * (for example, a button click) to comply with {@link https://developer.mozilla.org/en-US/docs/Web/API/Web_Audio_API/Best_practices#autoplay_policy | browser autoplay policies}.
+ *
+ * @example
+ * ```javascript
+ * const liveSession = await model.connect();
+ * let conversationController;
+ *
+ * // This function must be called from within a click handler.
+ * async function startConversation() {
+ *   try {
+ *     conversationController = await startAudioConversation(liveSession);
+ *   } catch (e) {
+ *     // Handle AI-specific errors
+ *     if (e instanceof AIError) {
+ *       console.error("AI Error:", e.message);
+ *     }
+ *     // Handle microphone permission and hardware errors
+ *     else if (e instanceof DOMException) {
+ *       console.error("Microphone Error:", e.message);
+ *     }
+ *     // Handle other unexpected errors
+ *     else {
+ *       console.error("An unexpected error occurred:", e);
+ *     }
+ *   }
+ * }
+ *
+ * // Later, to stop the conversation:
+ * // if (conversationController) {
+ * //   await conversationController.stop();
+ * // }
+ * ```
+ *
+ * @param liveSession - An active {@link LiveSession} instance.
+ * @param options - Configuration options for the audio conversation.
+ * @returns A `Promise` that resolves with an {@link AudioConversationController}.
+ * @throws `AIError` if the environment does not support required Web APIs (`UNSUPPORTED`), if a conversation is already active (`REQUEST_ERROR`), the session is closed (`SESSION_CLOSED`), or if an unexpected initialization error occurs (`ERROR`).
+ * @throws `DOMException` Thrown by `navigator.mediaDevices.getUserMedia()` if issues occur with microphone access, such as permissions being denied (`NotAllowedError`) or no compatible hardware being found (`NotFoundError`). See the {@link https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia#exceptions | MDN documentation} for a full list of exceptions.
+ *
+ * @beta
+ */
+export declare function startAudioConversation(liveSession: LiveSession, options?: StartAudioConversationOptions): Promise<AudioConversationController>;
+
+/**
+ * Options for {@link startAudioConversation}.
+ *
+ * @beta
+ */
+export declare interface StartAudioConversationOptions {
+    /**
+     * An async handler that is called when the model requests a function to be executed.
+     * The handler should perform the function call and return the result as a `Part`,
+     * which will then be sent back to the model.
+     */
+    functionCallingHandler?: (functionCalls: LiveServerToolCall['functionCalls']) => Promise<Part>;
+}
+
 /**
  * Params for {@link GenerativeModel.startChat}.
  * @public
@@ -2577,6 +3109,18 @@ export declare interface VideoMetadata {
     endOffset: string;
 }
 
+/**
+ * Configuration for the voice to used in speech synthesis.
+ *
+ * @beta
+ */
+export declare interface VoiceConfig {
+    /**
+     * Configures the voice using a pre-built voice configuration.
+     */
+    prebuiltVoiceConfig?: PrebuiltVoiceConfig;
+}
+
 /**
  * @public
  */
@@ -2612,4 +3156,59 @@ export declare interface WebGroundingChunk {
     domain?: string;
 }
 
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * A standardized interface for interacting with a WebSocket connection.
+ * This abstraction allows the SDK to use the appropriate WebSocket implementation
+ * for the current JS environment (Browser vs. Node) without
+ * changing the core logic of the `LiveSession`.
+ * @internal
+ */
+declare interface WebSocketHandler {
+    /**
+     * Establishes a connection to the given URL.
+     *
+     * @param url The WebSocket URL (e.g., wss://...).
+     * @returns A promise that resolves on successful connection or rejects on failure.
+     */
+    connect(url: string): Promise<void>;
+    /**
+     * Sends data over the WebSocket.
+     *
+     * @param data The string or binary data to send.
+     */
+    send(data: string | ArrayBuffer): void;
+    /**
+     * Returns an async generator that yields parsed JSON objects from the server.
+     * The yielded type is `unknown` because the handler cannot guarantee the shape of the data.
+     * The consumer is responsible for type validation.
+     * The generator terminates when the connection is closed.
+     *
+     * @returns A generator that allows consumers to pull messages using a `for await...of` loop.
+     */
+    listen(): AsyncGenerator<unknown>;
+    /**
+     * Closes the WebSocket connection.
+     *
+     * @param code - A numeric status code explaining why the connection is closing.
+     * @param reason - A human-readable string explaining why the connection is closing.
+     */
+    close(code?: number, reason?: string): Promise<void>;
+}
+
 export { }