@firebase/ai 2.3.0-20250917161512 → 2.3.0-canary.0bb2fe636

package/dist/esm/src/methods/live-session-helpers.d.ts

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { LiveServerToolCall, Part } from '../types';
+import { FunctionCall, FunctionResponse } from '../types';
 import { LiveSession } from './live-session';
 /**
  * A controller for managing an active audio conversation.
@@ -39,7 +39,7 @@ export interface StartAudioConversationOptions {
      * The handler should perform the function call and return the result as a `Part`,
      * which will then be sent back to the model.
      */
-    functionCallingHandler?: (functionCalls: LiveServerToolCall['functionCalls']) => Promise<Part>;
+    functionCallingHandler?: (functionCalls: FunctionCall[]) => Promise<FunctionResponse>;
 }
 /**
  * Dependencies needed by the {@link AudioConversationRunner}.

package/dist/esm/src/methods/live-session.d.ts

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { GenerativeContentBlob, LiveServerContent, LiveServerToolCall, LiveServerToolCallCancellation, Part } from '../public-types';
+import { FunctionResponse, GenerativeContentBlob, LiveServerContent, LiveServerToolCall, LiveServerToolCallCancellation, Part } from '../public-types';
 import { WebSocketHandler } from '../websocket';
 /**
  * Represents an active, real-time, bidirectional conversation with the model.
@@ -61,6 +61,15 @@ export declare class LiveSession {
      * @beta
      */
     sendMediaChunks(mediaChunks: GenerativeContentBlob[]): Promise<void>;
+    /**
+     * Sends function responses to the server.
+     *
+     * @param functionResponses - The function responses to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    sendFunctionResponses(functionResponses: FunctionResponse[]): Promise<void>;
     /**
      * Sends a stream of {@link GenerativeContentBlob}.
      *

package/dist/esm/src/types/content.d.ts

@@ -138,7 +138,7 @@ export interface FileDataPart {
 /**
  * Represents the code that is executed by the model.
  *
- * @public
+ * @beta
  */
 export interface ExecutableCodePart {
     text?: never;
@@ -157,7 +157,7 @@ export interface ExecutableCodePart {
 /**
  * Represents the code execution result from the model.
  *
- * @public
+ * @beta
  */
 export interface CodeExecutionResultPart {
     text?: never;
@@ -176,7 +176,7 @@ export interface CodeExecutionResultPart {
 /**
  * An interface for executable code returned by the model.
  *
- * @public
+ * @beta
  */
 export interface ExecutableCode {
     /**
@@ -191,7 +191,7 @@ export interface ExecutableCode {
 /**
  * The results of code execution run by the model.
  *
- * @public
+ * @beta
  */
 export interface CodeExecutionResult {
     /**

package/dist/esm/src/types/enums.d.ts

@@ -352,7 +352,7 @@ export type InferenceMode = (typeof InferenceMode)[keyof typeof InferenceMode];
 /**
  * Represents the result of the code execution.
  *
- * @public
+ * @beta
  */
 export declare const Outcome: {
     UNSPECIFIED: string;
@@ -363,13 +363,13 @@ export declare const Outcome: {
 /**
  * Represents the result of the code execution.
  *
- * @public
+ * @beta
  */
 export type Outcome = (typeof Outcome)[keyof typeof Outcome];
 /**
  * The programming language of the code.
  *
- * @public
+ * @beta
  */
 export declare const Language: {
     UNSPECIFIED: string;
@@ -378,6 +378,6 @@ export declare const Language: {
 /**
  * The programming language of the code.
  *
- * @public
+ * @beta
  */
 export type Language = (typeof Language)[keyof typeof Language];

package/dist/esm/src/types/googleai.d.ts

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { Tool, GenerationConfig, Citation, FinishReason, GroundingMetadata, PromptFeedback, SafetyRating, UsageMetadata } from '../public-types';
+import { Tool, GenerationConfig, Citation, FinishReason, GroundingMetadata, PromptFeedback, SafetyRating, UsageMetadata, URLContextMetadata } from '../public-types';
 import { Content, Part } from './content';
 /**
  * @internal
@@ -47,6 +47,7 @@ export interface GoogleAIGenerateContentCandidate {
     safetyRatings?: SafetyRating[];
     citationMetadata?: GoogleAICitationMetadata;
     groundingMetadata?: GroundingMetadata;
+    urlContextMetadata?: URLContextMetadata;
 }
 /**
  * @internal

package/dist/esm/src/types/live-responses.d.ts

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { Content, GenerativeContentBlob, Part } from './content';
+import { Content, FunctionResponse, GenerativeContentBlob, Part } from './content';
 import { LiveGenerationConfig, Tool, ToolConfig } from './requests';
 /**
  * User input that is sent to the model.
@@ -37,6 +37,14 @@ export interface _LiveClientRealtimeInput {
         mediaChunks: GenerativeContentBlob[];
     };
 }
+/**
+ * Function responses that are sent to the model in real time.
+ */
+export interface _LiveClientToolResponse {
+    toolResponse: {
+        functionResponses: FunctionResponse[];
+    };
+}
 /**
  * The first message in a Live session, used to configure generation options.
  *

package/dist/esm/src/types/requests.d.ts

@@ -218,7 +218,7 @@ export interface RequestOptions {
  * Defines a tool that model can call to access external knowledge.
  * @public
  */
-export type Tool = FunctionDeclarationsTool | GoogleSearchTool | CodeExecutionTool;
+export type Tool = FunctionDeclarationsTool | GoogleSearchTool | CodeExecutionTool | URLContextTool;
 /**
  * Structured representation of a function declaration as defined by the
  * {@link https://spec.openapis.org/oas/v3.0.3 | OpenAPI 3.0 specification}.
@@ -273,7 +273,7 @@ export interface GoogleSearchTool {
 /**
  * A tool that enables the model to use code execution.
  *
- * @public
+ * @beta
  */
 export interface CodeExecutionTool {
     /**
@@ -291,6 +291,26 @@ export interface CodeExecutionTool {
  */
 export interface GoogleSearch {
 }
+/**
+ * A tool that allows you to provide additional context to the models in the form of public web
+ * URLs. By including URLs in your request, the Gemini model will access the content from those
+ * pages to inform and enhance its response.
+ *
+ * @beta
+ */
+export interface URLContextTool {
+    /**
+     * Specifies the URL Context configuration.
+     */
+    urlContext: URLContext;
+}
+/**
+ * Specifies the URL Context configuration.
+ *
+ * @beta
+ */
+export interface URLContext {
+}
 /**
  * A `FunctionDeclarationsTool` is a piece of code that enables the system to
  * interact with external systems to perform an action, or set of actions,

package/dist/esm/src/types/responses.d.ts

@@ -102,8 +102,16 @@ export interface UsageMetadata {
      */
     thoughtsTokenCount?: number;
     totalTokenCount: number;
+    /**
+     * The number of tokens used by tools.
+     */
+    toolUsePromptTokenCount?: number;
     promptTokensDetails?: ModalityTokenCount[];
     candidatesTokensDetails?: ModalityTokenCount[];
+    /**
+     * A list of tokens used by tools, broken down by modality.
+     */
+    toolUsePromptTokensDetails?: ModalityTokenCount[];
 }
 /**
  * Represents token counting info for a single modality.
@@ -143,6 +151,7 @@ export interface GenerateContentCandidate {
     safetyRatings?: SafetyRating[];
     citationMetadata?: CitationMetadata;
     groundingMetadata?: GroundingMetadata;
+    urlContextMetadata?: URLContextMetadata;
 }
 /**
  * Citation metadata that may be found on a {@link GenerateContentCandidate}.
@@ -323,6 +332,89 @@ export interface Segment {
      */
     text: string;
 }
+/**
+ * Metadata related to {@link URLContextTool}.
+ *
+ * @beta
+ */
+export interface URLContextMetadata {
+    /**
+     * List of URL metadata used to provide context to the Gemini model.
+     */
+    urlMetadata: URLMetadata[];
+}
+/**
+ * Metadata for a single URL retrieved by the {@link URLContextTool} tool.
+ *
+ * @beta
+ */
+export interface URLMetadata {
+    /**
+     * The retrieved URL.
+     */
+    retrievedUrl?: string;
+    /**
+     * The status of the URL retrieval.
+     */
+    urlRetrievalStatus?: URLRetrievalStatus;
+}
+/**
+ * The status of a URL retrieval.
+ *
+ * @remarks
+ * <b>URL_RETRIEVAL_STATUS_UNSPECIFIED:</b> Unspecified retrieval status.
+ * <br/>
+ * <b>URL_RETRIEVAL_STATUS_SUCCESS:</b> The URL retrieval was successful.
+ * <br/>
+ * <b>URL_RETRIEVAL_STATUS_ERROR:</b> The URL retrieval failed.
+ * <br/>
+ * <b>URL_RETRIEVAL_STATUS_PAYWALL:</b> The URL retrieval failed because the content is behind a paywall.
+ * <br/>
+ * <b>URL_RETRIEVAL_STATUS_UNSAFE:</b> The URL retrieval failed because the content is unsafe.
+ * <br/>
+ *
+ * @beta
+ */
+export declare const URLRetrievalStatus: {
+    /**
+     * Unspecified retrieval status.
+     */
+    URL_RETRIEVAL_STATUS_UNSPECIFIED: string;
+    /**
+     * The URL retrieval was successful.
+     */
+    URL_RETRIEVAL_STATUS_SUCCESS: string;
+    /**
+     * The URL retrieval failed.
+     */
+    URL_RETRIEVAL_STATUS_ERROR: string;
+    /**
+     * The URL retrieval failed because the content is behind a paywall.
+     */
+    URL_RETRIEVAL_STATUS_PAYWALL: string;
+    /**
+     * The URL retrieval failed because the content is unsafe.
+     */
+    URL_RETRIEVAL_STATUS_UNSAFE: string;
+};
+/**
+ * The status of a URL retrieval.
+ *
+ * @remarks
+ * <b>URL_RETRIEVAL_STATUS_UNSPECIFIED:</b> Unspecified retrieval status.
+ * <br/>
+ * <b>URL_RETRIEVAL_STATUS_SUCCESS:</b> The URL retrieval was successful.
+ * <br/>
+ * <b>URL_RETRIEVAL_STATUS_ERROR:</b> The URL retrieval failed.
+ * <br/>
+ * <b>URL_RETRIEVAL_STATUS_PAYWALL:</b> The URL retrieval failed because the content is behind a paywall.
+ * <br/>
+ * <b>URL_RETRIEVAL_STATUS_UNSAFE:</b> The URL retrieval failed because the content is unsafe.
+ * <br/>
+ *
+ * @beta
+ */
+export type URLRetrievalStatus = (typeof URLRetrievalStatus)[keyof typeof URLRetrievalStatus];
 /**
  * @public
  */

package/dist/index.cjs.js

@@ -8,7 +8,7 @@ var util = require('@firebase/util');
 var logger$1 = require('@firebase/logger');
 
 var name = "@firebase/ai";
-var version = "2.3.0-20250917161512";
+var version = "2.3.0-canary.0bb2fe636";
 
 /**
  * @license
@@ -386,7 +386,7 @@ const InferenceMode = {
 /**
  * Represents the result of the code execution.
  *
- * @public
+ * @beta
  */
 const Outcome = {
     UNSPECIFIED: 'OUTCOME_UNSPECIFIED',
@@ -397,7 +397,7 @@ const Outcome = {
 /**
  * The programming language of the code.
  *
- * @public
+ * @beta
  */
 const Language = {
     UNSPECIFIED: 'LANGUAGE_UNSPECIFIED',
@@ -420,6 +420,45 @@ const Language = {
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+/**
+ * The status of a URL retrieval.
+ *
+ * @remarks
+ * <b>URL_RETRIEVAL_STATUS_UNSPECIFIED:</b> Unspecified retrieval status.
+ * <br/>
+ * <b>URL_RETRIEVAL_STATUS_SUCCESS:</b> The URL retrieval was successful.
+ * <br/>
+ * <b>URL_RETRIEVAL_STATUS_ERROR:</b> The URL retrieval failed.
+ * <br/>
+ * <b>URL_RETRIEVAL_STATUS_PAYWALL:</b> The URL retrieval failed because the content is behind a paywall.
+ * <br/>
+ * <b>URL_RETRIEVAL_STATUS_UNSAFE:</b> The URL retrieval failed because the content is unsafe.
+ * <br/>
+ *
+ * @beta
+ */
+const URLRetrievalStatus = {
+    /**
+     * Unspecified retrieval status.
+     */
+    URL_RETRIEVAL_STATUS_UNSPECIFIED: 'URL_RETRIEVAL_STATUS_UNSPECIFIED',
+    /**
+     * The URL retrieval was successful.
+     */
+    URL_RETRIEVAL_STATUS_SUCCESS: 'URL_RETRIEVAL_STATUS_SUCCESS',
+    /**
+     * The URL retrieval failed.
+     */
+    URL_RETRIEVAL_STATUS_ERROR: 'URL_RETRIEVAL_STATUS_ERROR',
+    /**
+     * The URL retrieval failed because the content is behind a paywall.
+     */
+    URL_RETRIEVAL_STATUS_PAYWALL: 'URL_RETRIEVAL_STATUS_PAYWALL',
+    /**
+     * The URL retrieval failed because the content is unsafe.
+     */
+    URL_RETRIEVAL_STATUS_UNSAFE: 'URL_RETRIEVAL_STATUS_UNSAFE'
+};
 /**
  * The types of responses that can be returned by {@link LiveSession.receive}.
  *
@@ -1880,7 +1919,7 @@ function mapGenerateContentCandidates(candidates) {
             // videoMetadata is not supported.
             // Throw early since developers may send a long video as input and only expect to pay
             // for inference on a small portion of the video.
-            if (candidate.content?.parts.some(part => part?.videoMetadata)) {
+            if (candidate.content?.parts?.some(part => part?.videoMetadata)) {
                 throw new AIError(AIErrorCode.UNSUPPORTED, 'Part.videoMetadata is not supported in the Gemini Developer API. Please remove this property.');
             }
             const mappedCandidate = {
@@ -1890,7 +1929,8 @@ function mapGenerateContentCandidates(candidates) {
                 finishMessage: candidate.finishMessage,
                 safetyRatings: mappedSafetyRatings,
                 citationMetadata,
-                groundingMetadata: candidate.groundingMetadata
+                groundingMetadata: candidate.groundingMetadata,
+                urlContextMetadata: candidate.urlContextMetadata
             };
             mappedCandidates.push(mappedCandidate);
         });
@@ -1981,6 +2021,14 @@ async function* generateResponseSequence(stream, apiSettings) {
         else {
             enhancedResponse = createEnhancedContentResponse(value);
         }
+        const firstCandidate = enhancedResponse.candidates?.[0];
+        // Don't yield a response with no useful data for the developer.
+        if (!firstCandidate?.content?.parts &&
+            !firstCandidate?.finishReason &&
+            !firstCandidate?.citationMetadata &&
+            !firstCandidate?.urlContextMetadata) {
+            continue;
+        }
         yield enhancedResponse;
     }
 }
@@ -2060,36 +2108,43 @@ function aggregateResponses(responses) {
                     candidate.safetyRatings;
                 aggregatedResponse.candidates[i].groundingMetadata =
                     candidate.groundingMetadata;
+                // The urlContextMetadata object is defined in the first chunk of the response stream.
+                // In all subsequent chunks, the urlContextMetadata object will be undefined. We need to
+                // make sure that we don't overwrite the first value urlContextMetadata object with undefined.
+                // FIXME: What happens if we receive a second, valid urlContextMetadata object?
+                const urlContextMetadata = candidate.urlContextMetadata;
+                if (typeof urlContextMetadata === 'object' &&
+                    urlContextMetadata !== null &&
+                    Object.keys(urlContextMetadata).length > 0) {
+                    aggregatedResponse.candidates[i].urlContextMetadata =
+                        urlContextMetadata;
+                }
                 /**
                  * Candidates should always have content and parts, but this handles
                  * possible malformed responses.
                  */
-                if (candidate.content && candidate.content.parts) {
+                if (candidate.content) {
+                    // Skip a candidate without parts.
+                    if (!candidate.content.parts) {
+                        continue;
+                    }
                     if (!aggregatedResponse.candidates[i].content) {
                         aggregatedResponse.candidates[i].content = {
                             role: candidate.content.role || 'user',
                             parts: []
                         };
                     }
-                    const newPart = {};
                     for (const part of candidate.content.parts) {
-                        if (part.text !== undefined) {
-                            // The backend can send empty text parts. If these are sent back
-                            // (e.g. in chat history), the backend will respond with an error.
-                            // To prevent this, ignore empty text parts.
-                            if (part.text === '') {
-                                continue;
-                            }
-                            newPart.text = part.text;
-                        }
-                        if (part.functionCall) {
-                            newPart.functionCall = part.functionCall;
+                        const newPart = { ...part };
+                        // The backend can send empty text parts. If these are sent back
+                        // (e.g. in chat history), the backend will respond with an error.
+                        // To prevent this, ignore empty text parts.
+                        if (part.text === '') {
+                            continue;
                         }
-                        if (Object.keys(newPart).length === 0) {
-                            throw new AIError(AIErrorCode.INVALID_CONTENT, 'Part should have at least one property, but there are none. This is likely caused ' +
-                                'by a malformed response from the backend.');
+                        if (Object.keys(newPart).length > 0) {
+                            aggregatedResponse.candidates[i].content.parts.push(newPart);
                         }
-                        aggregatedResponse.candidates[i].content.parts.push(newPart);
                     }
                 }
             }
@@ -2795,6 +2850,25 @@ class LiveSession {
             this.webSocketHandler.send(JSON.stringify(message));
         });
     }
+    /**
+     * Sends function responses to the server.
+     *
+     * @param functionResponses - The function responses to send.
+     * @throws If this session has been closed.
+     *
+     * @beta
+     */
+    async sendFunctionResponses(functionResponses) {
+        if (this.isClosed) {
+            throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
+        }
+        const message = {
+            toolResponse: {
+                functionResponses
+            }
+        };
+        this.webSocketHandler.send(JSON.stringify(message));
+    }
     /**
      * Sends a stream of {@link GenerativeContentBlob}.
      *
@@ -3776,9 +3850,9 @@ class AudioConversationRunner {
                 }
                 else {
                     try {
-                        const resultPart = await this.options.functionCallingHandler(message.functionCalls);
+                        const functionResponse = await this.options.functionCallingHandler(message.functionCalls);
                         if (!this.isStopped) {
-                            void this.liveSession.send([resultPart]);
+                            void this.liveSession.sendFunctionResponses([functionResponse]);
                         }
                     }
                     catch (e) {
@@ -4078,6 +4152,7 @@ exports.ResponseModality = ResponseModality;
 exports.Schema = Schema;
 exports.SchemaType = SchemaType;
 exports.StringSchema = StringSchema;
+exports.URLRetrievalStatus = URLRetrievalStatus;
 exports.VertexAIBackend = VertexAIBackend;
 exports.getAI = getAI;
 exports.getGenerativeModel = getGenerativeModel;