@firebase/ai 2.3.0 → 2.4.0-canary.22e0a1adb

package/dist/ai-public.d.ts

@@ -396,7 +396,7 @@ export declare interface CitationMetadata {
 /**
  * The results of code execution run by the model.
  *
- * @public
+ * @beta
  */
 export declare interface CodeExecutionResult {
     /**
@@ -413,7 +413,7 @@ export declare interface CodeExecutionResult {
 /**
  * Represents the code execution result from the model.
  *
- * @public
+ * @beta
  */
 export declare interface CodeExecutionResultPart {
     text?: never;
@@ -430,7 +430,7 @@ export declare interface CodeExecutionResultPart {
 /**
  * A tool that enables the model to use code execution.
  *
- * @public
+ * @beta
  */
 export declare interface CodeExecutionTool {
     /**
@@ -558,6 +558,12 @@ export declare interface EnhancedGenerateContentResponse extends GenerateContent
      * set to `true`.
      */
     thoughtSummary: () => string | undefined;
+    /**
+     * Indicates whether inference happened on-device or in-cloud.
+     *
+     * @beta
+     */
+    inferenceSource?: InferenceSource;
 }
 
 /**
@@ -580,7 +586,7 @@ export declare interface ErrorDetails {
 /**
  * An interface for executable code returned by the model.
  *
- * @public
+ * @beta
  */
 export declare interface ExecutableCode {
     /**
@@ -596,7 +602,7 @@ export declare interface ExecutableCode {
 /**
  * Represents the code that is executed by the model.
  *
- * @public
+ * @beta
  */
 export declare interface ExecutableCodePart {
     text?: never;
@@ -853,6 +859,7 @@ export declare interface GenerateContentCandidate {
     safetyRatings?: SafetyRating[];
     citationMetadata?: CitationMetadata;
     groundingMetadata?: GroundingMetadata;
+    urlContextMetadata?: URLContextMetadata;
 }
 
 /**
@@ -1044,7 +1051,7 @@ export declare function getGenerativeModel(ai: AI, modelParams: ModelParams | Hy
  * @throws If the `apiKey` or `projectId` fields are missing in your
  * Firebase config.
  *
- * @beta
+ * @public
  */
 export declare function getImagenModel(ai: AI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel;
 
@@ -1360,7 +1367,7 @@ export declare interface HybridParams {
  * See the {@link http://firebase.google.com/docs/vertex-ai/generate-images | documentation }
  * for more details and examples of the supported aspect ratios.
  *
- * @beta
+ * @public
  */
 export declare const ImagenAspectRatio: {
     /**
@@ -1394,7 +1401,7 @@ export declare const ImagenAspectRatio: {
  * See the {@link http://firebase.google.com/docs/vertex-ai/generate-images | documentation }
  * for more details and examples of the supported aspect ratios.
  *
- * @beta
+ * @public
  */
 export declare type ImagenAspectRatio = (typeof ImagenAspectRatio)[keyof typeof ImagenAspectRatio];
 
@@ -1402,7 +1409,7 @@ export declare type ImagenAspectRatio = (typeof ImagenAspectRatio)[keyof typeof
  * An image generated by Imagen, stored in a Cloud Storage for Firebase bucket.
  *
  * This feature is not available yet.
- * @beta
+ * @public
  */
 export declare interface ImagenGCSImage {
     /**
@@ -1425,7 +1432,7 @@ export declare interface ImagenGCSImage {
  * See the {@link http://firebase.google.com/docs/vertex-ai/generate-images-imagen | documentation} for
  * more details.
  *
- * @beta
+ * @public
  */
 export declare interface ImagenGenerationConfig {
     /**
@@ -1477,7 +1484,7 @@ export declare interface ImagenGenerationConfig {
 /**
  * The response from a request to generate images with Imagen.
  *
- * @beta
+ * @public
  */
 export declare interface ImagenGenerationResponse<T extends ImagenInlineImage | ImagenGCSImage> {
     /**
@@ -1532,7 +1539,7 @@ export declare interface ImagenGenerationResponse<T extends ImagenInlineImage |
  * }
  * ```
  *
- * @beta
+ * @public
  */
 export declare class ImagenImageFormat {
     /**
@@ -1550,7 +1557,7 @@ export declare class ImagenImageFormat {
      * @param compressionQuality - The level of compression (a number between 0 and 100).
      * @returns An {@link ImagenImageFormat} object for a JPEG image.
      *
-     * @beta
+     * @public
      */
     static jpeg(compressionQuality?: number): ImagenImageFormat;
     /**
@@ -1558,7 +1565,7 @@ export declare class ImagenImageFormat {
      *
      * @returns An {@link ImagenImageFormat} object for a PNG image.
      *
-     * @beta
+     * @public
      */
     static png(): ImagenImageFormat;
 }
@@ -1582,7 +1589,7 @@ export declare class ImagenImageFormat {
 /**
  * An image generated by Imagen, represented as inline data.
  *
- * @beta
+ * @public
  */
 export declare interface ImagenInlineImage {
     /**
@@ -1617,7 +1624,7 @@ export declare interface ImagenInlineImage {
  * }
  * ```
  *
- * @beta
+ * @public
  */
 export declare class ImagenModel extends AIModel {
     requestOptions?: RequestOptions | undefined;
@@ -1656,7 +1663,7 @@ export declare class ImagenModel extends AIModel {
      * returned object will have a `filteredReason` property.
      * If all images are filtered, the `images` array will be empty.
      *
-     * @beta
+     * @public
      */
     generateImages(prompt: string): Promise<ImagenGenerationResponse<ImagenInlineImage>>;
     /* Excluded from this release type: generateImagesGCS */
@@ -1665,7 +1672,7 @@ export declare class ImagenModel extends AIModel {
 /**
  * Parameters for configuring an {@link ImagenModel}.
  *
- * @beta
+ * @public
  */
 export declare interface ImagenModelParams {
     /**
@@ -1694,7 +1701,7 @@ export declare interface ImagenModelParams {
  * See the <a href="http://firebase.google.com/docs/vertex-ai/generate-images">personGeneration</a>
  * documentation for more details.
  *
- * @beta
+ * @public
  */
 export declare const ImagenPersonFilterLevel: {
     /**
@@ -1725,7 +1732,7 @@ export declare const ImagenPersonFilterLevel: {
  * See the <a href="http://firebase.google.com/docs/vertex-ai/generate-images">personGeneration</a>
  * documentation for more details.
  *
- * @beta
+ * @public
  */
 export declare type ImagenPersonFilterLevel = (typeof ImagenPersonFilterLevel)[keyof typeof ImagenPersonFilterLevel];
 
@@ -1739,7 +1746,7 @@ export declare type ImagenPersonFilterLevel = (typeof ImagenPersonFilterLevel)[k
  * and the {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters | Responsible AI and usage guidelines}
  * for more details.
  *
- * @beta
+ * @public
  */
 export declare const ImagenSafetyFilterLevel: {
     /**
@@ -1773,7 +1780,7 @@ export declare const ImagenSafetyFilterLevel: {
  * and the {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters | Responsible AI and usage guidelines}
  * for more details.
  *
- * @beta
+ * @public
  */
 export declare type ImagenSafetyFilterLevel = (typeof ImagenSafetyFilterLevel)[keyof typeof ImagenSafetyFilterLevel];
 
@@ -1783,7 +1790,7 @@ export declare type ImagenSafetyFilterLevel = (typeof ImagenSafetyFilterLevel)[k
  * See the {@link http://firebase.google.com/docs/vertex-ai/generate-images | documentation }
  * for more details.
  *
- * @beta
+ * @public
  */
 export declare interface ImagenSafetySettings {
     /**
@@ -1832,6 +1839,23 @@ export declare const InferenceMode: {
  */
 export declare type InferenceMode = (typeof InferenceMode)[keyof typeof InferenceMode];
 
+/**
+ * Indicates whether inference happened on-device or in-cloud.
+ *
+ * @beta
+ */
+export declare const InferenceSource: {
+    readonly ON_DEVICE: "on_device";
+    readonly IN_CLOUD: "in_cloud";
+};
+
+/**
+ * Indicates whether inference happened on-device or in-cloud.
+ *
+ * @beta
+ */
+export declare type InferenceSource = (typeof InferenceSource)[keyof typeof InferenceSource];
+
 /**
  * Content part interface if the part represents an image.
  * @public
@@ -1862,7 +1886,7 @@ export declare class IntegerSchema extends Schema {
 /**
  * The programming language of the code.
  *
- * @public
+ * @beta
  */
 export declare const Language: {
     UNSPECIFIED: string;
@@ -1872,7 +1896,7 @@ export declare const Language: {
 /**
  * The programming language of the code.
  *
- * @public
+ * @beta
  */
 export declare type Language = (typeof Language)[keyof typeof Language];
 
@@ -2139,23 +2163,65 @@ export declare class LiveSession {
      */
     send(request: string | Array<string | Part>, turnComplete?: boolean): Promise<void>;
     /**
-     * Sends realtime input to the server.
+     * Sends text to the server in realtime.
      *
-     * @param mediaChunks - The media chunks to send.
+     * @example
+     * ```javascript
+     * liveSession.sendTextRealtime("Hello, how are you?");
+     * ```
+     *
+     * @param text - The text data to send.
      * @throws If this session has been closed.
      *
      * @beta
      */
-    sendMediaChunks(mediaChunks: GenerativeContentBlob[]): Promise<void>;
+    sendTextRealtime(text: string): Promise<void>;
     /**
-     * Sends a stream of {@link GenerativeContentBlob}.
+     * Sends audio data to the server in realtime.
      *
-     * @param mediaChunkStream - The stream of {@link GenerativeContentBlob} to send.
+     * @remarks The server requires that the audio data is base64-encoded 16-bit PCM at 16kHz
+     * little-endian.
+     *
+     * @example
+     * ```javascript
+     * // const pcmData = ... base64-encoded 16-bit PCM at 16kHz little-endian.
+     * const blob = { mimeType: "audio/pcm", data: pcmData };
+     * liveSession.sendAudioRealtime(blob);
+     * ```
+     *
+     * @param blob - The base64-encoded PCM data to send to the server in realtime.
      * @throws If this session has been closed.
      *
      * @beta
      */
-    sendMediaStream(mediaChunkStream: ReadableStream<GenerativeContentBlob>): Promise<void>;
+    sendAudioRealtime(blob: GenerativeContentBlob): Promise<void>;
+    /**
+     * Sends video data to the server in realtime.
+     *
+     * @remarks The server requires that the video is sent as individual video frames at 1 FPS. It
+     * is recommended to set `mimeType` to `image/jpeg`.
+     *
+     * @example
+     * ```javascript
+     * // const videoFrame = ... base64-encoded JPEG data
+     * const blob = { mimeType: "image/jpeg", data: videoFrame };
+     * liveSession.sendVideoRealtime(blob);
+     * ```
+     * @param blob - The base64-encoded video data to send to the server in realtime.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    sendVideoRealtime(blob: GenerativeContentBlob): Promise<void>;
+    /**
+     * Sends function responses to the server.
+     *
+     * @param functionResponses - The function responses to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    sendFunctionResponses(functionResponses: FunctionResponse[]): Promise<void>;
     /**
      * Yields messages received from the server.
      * This can only be used by one consumer at a time.
@@ -2173,6 +2239,28 @@ export declare class LiveSession {
      * @beta
      */
     close(): Promise<void>;
+    /**
+     * Sends realtime input to the server.
+     *
+     * @deprecated Use `sendTextRealtime()`, `sendAudioRealtime()`, and `sendVideoRealtime()` instead.
+     *
+     * @param mediaChunks - The media chunks to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    sendMediaChunks(mediaChunks: GenerativeContentBlob[]): Promise<void>;
+    /**
+     * @deprecated Use `sendTextRealtime()`, `sendAudioRealtime()`, and `sendVideoRealtime()` instead.
+     *
+     * Sends a stream of {@link GenerativeContentBlob}.
+     *
+     * @param mediaChunkStream - The stream of {@link GenerativeContentBlob} to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    sendMediaStream(mediaChunkStream: ReadableStream<GenerativeContentBlob>): Promise<void>;
 }
 
 /**
@@ -2289,7 +2377,7 @@ export declare interface OnDeviceParams {
 /**
  * Represents the result of the code execution.
  *
- * @public
+ * @beta
  */
 export declare const Outcome: {
     UNSPECIFIED: string;
@@ -2301,7 +2389,7 @@ export declare const Outcome: {
 /**
  * Represents the result of the code execution.
  *
- * @public
+ * @beta
  */
 export declare type Outcome = (typeof Outcome)[keyof typeof Outcome];
 
@@ -2776,7 +2864,7 @@ export declare interface StartAudioConversationOptions {
      * The handler should perform the function call and return the result as a `Part`,
      * which will then be sent back to the model.
      */
-    functionCallingHandler?: (functionCalls: LiveServerToolCall['functionCalls']) => Promise<Part>;
+    functionCallingHandler?: (functionCalls: FunctionCall[]) => Promise<FunctionResponse>;
 }
 
 /**
@@ -2854,7 +2942,7 @@ export declare interface ThinkingConfig {
  * Defines a tool that model can call to access external knowledge.
  * @public
  */
-export declare type Tool = FunctionDeclarationsTool | GoogleSearchTool | CodeExecutionTool;
+export declare type Tool = FunctionDeclarationsTool | GoogleSearchTool | CodeExecutionTool | URLContextTool;
 
 /**
  * Tool config. This config is shared for all tools provided in the request.
@@ -2870,6 +2958,115 @@ export declare interface ToolConfig {
  */
 export declare type TypedSchema = IntegerSchema | NumberSchema | StringSchema | BooleanSchema | ObjectSchema | ArraySchema | AnyOfSchema;
 
+/**
+ * Specifies the URL Context configuration.
+ *
+ * @beta
+ */
+export declare interface URLContext {
+}
+
+/**
+ * Metadata related to {@link URLContextTool}.
+ *
+ * @beta
+ */
+export declare interface URLContextMetadata {
+    /**
+     * List of URL metadata used to provide context to the Gemini model.
+     */
+    urlMetadata: URLMetadata[];
+}
+
+/**
+ * A tool that allows you to provide additional context to the models in the form of public web
+ * URLs. By including URLs in your request, the Gemini model will access the content from those
+ * pages to inform and enhance its response.
+ *
+ * @beta
+ */
+export declare interface URLContextTool {
+    /**
+     * Specifies the URL Context configuration.
+     */
+    urlContext: URLContext;
+}
+
+/**
+ * Metadata for a single URL retrieved by the {@link URLContextTool} tool.
+ *
+ * @beta
+ */
+export declare interface URLMetadata {
+    /**
+     * The retrieved URL.
+     */
+    retrievedUrl?: string;
+    /**
+     * The status of the URL retrieval.
+     */
+    urlRetrievalStatus?: URLRetrievalStatus;
+}
+
+/**
+ * The status of a URL retrieval.
+ *
+ * @remarks
+ * <b>URL_RETRIEVAL_STATUS_UNSPECIFIED:</b> Unspecified retrieval status.
+ * <br/>
+ * <b>URL_RETRIEVAL_STATUS_SUCCESS:</b> The URL retrieval was successful.
+ * <br/>
+ * <b>URL_RETRIEVAL_STATUS_ERROR:</b> The URL retrieval failed.
+ * <br/>
+ * <b>URL_RETRIEVAL_STATUS_PAYWALL:</b> The URL retrieval failed because the content is behind a paywall.
+ * <br/>
+ * <b>URL_RETRIEVAL_STATUS_UNSAFE:</b> The URL retrieval failed because the content is unsafe.
+ * <br/>
+ *
+ * @beta
+ */
+export declare const URLRetrievalStatus: {
+    /**
+     * Unspecified retrieval status.
+     */
+    URL_RETRIEVAL_STATUS_UNSPECIFIED: string;
+    /**
+     * The URL retrieval was successful.
+     */
+    URL_RETRIEVAL_STATUS_SUCCESS: string;
+    /**
+     * The URL retrieval failed.
+     */
+    URL_RETRIEVAL_STATUS_ERROR: string;
+    /**
+     * The URL retrieval failed because the content is behind a paywall.
+     */
+    URL_RETRIEVAL_STATUS_PAYWALL: string;
+    /**
+     * The URL retrieval failed because the content is unsafe.
+     */
+    URL_RETRIEVAL_STATUS_UNSAFE: string;
+};
+
+/**
+ * The status of a URL retrieval.
+ *
+ * @remarks
+ * <b>URL_RETRIEVAL_STATUS_UNSPECIFIED:</b> Unspecified retrieval status.
+ * <br/>
+ * <b>URL_RETRIEVAL_STATUS_SUCCESS:</b> The URL retrieval was successful.
+ * <br/>
+ * <b>URL_RETRIEVAL_STATUS_ERROR:</b> The URL retrieval failed.
+ * <br/>
+ * <b>URL_RETRIEVAL_STATUS_PAYWALL:</b> The URL retrieval failed because the content is behind a paywall.
+ * <br/>
+ * <b>URL_RETRIEVAL_STATUS_UNSAFE:</b> The URL retrieval failed because the content is unsafe.
+ * <br/>
+ *
+ * @beta
+ */
+export declare type URLRetrievalStatus = (typeof URLRetrievalStatus)[keyof typeof URLRetrievalStatus];
+
 /**
  * Usage metadata about a {@link GenerateContentResponse}.
  *
@@ -2883,8 +3080,16 @@ export declare interface UsageMetadata {
      */
     thoughtsTokenCount?: number;
     totalTokenCount: number;
+    /**
+     * The number of tokens used by tools.
+     */
+    toolUsePromptTokenCount?: number;
     promptTokensDetails?: ModalityTokenCount[];
     candidatesTokensDetails?: ModalityTokenCount[];
+    /**
+     * A list of tokens used by tools, broken down by modality.
+     */
+    toolUsePromptTokensDetails?: ModalityTokenCount[];
 }
 
 /**