@firebase/ai 2.8.0 → 2.9.0-canary.78384d32c

package/dist/esm/src/methods/chat-session.d.ts

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { Content, GenerateContentResult, GenerateContentStreamResult, Part, RequestOptions, SingleRequestOptions, StartChatParams } from '../types';
+import { Content, FunctionCall, FunctionResponsePart, GenerateContentRequest, GenerateContentResponse, GenerateContentResult, GenerateContentStreamResult, Part, RequestOptions, SingleRequestOptions, StartChatParams } from '../types';
 import { ApiSettings } from '../types/internal';
 import { ChromeAdapter } from '../types/chrome-adapter';
 /**
@@ -42,6 +42,12 @@ export declare class ChatSession {
      * to history.
      */
     getHistory(): Promise<Content[]>;
+    /**
+     * Format Content into a request for generateContent or
+     * generateContentStream.
+     * @internal
+     */
+    _formatRequest(incomingContent: Content, tempHistory: Content[]): GenerateContentRequest;
     /**
      * Sends a chat message and receives a non-streaming
      * {@link GenerateContentResult}
@@ -53,4 +59,19 @@ export declare class ChatSession {
      * and a response promise.
      */
     sendMessageStream(request: string | Array<string | Part>, singleRequestOptions?: SingleRequestOptions): Promise<GenerateContentStreamResult>;
+    /**
+     * Get function calls that the SDK has references to actually call.
+     * This is all-or-nothing. If the model is requesting multiple
+     * function calls, all of them must have references in order for
+     * automatic function calling to work.
+     *
+     * @internal
+     */
+    _getCallableFunctionCalls(response?: GenerateContentResponse): FunctionCall[] | undefined;
+    /**
+     * Call user-defined functions if requested by the model, and return
+     * the response that should be sent to the model.
+     * @internal
+     */
+    _callFunctionsAsNeeded(functionCalls: FunctionCall[]): Promise<FunctionResponsePart[]>;
 }

package/dist/esm/src/methods/generate-content.d.ts

@@ -14,10 +14,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { GenerateContentRequest, GenerateContentResult, GenerateContentStreamResult, SingleRequestOptions } from '../types';
+import { GenerateContentRequest, GenerateContentResponse, GenerateContentResult, GenerateContentStreamResult, SingleRequestOptions } from '../types';
 import { ApiSettings } from '../types/internal';
 import { ChromeAdapter } from '../types/chrome-adapter';
-export declare function generateContentStream(apiSettings: ApiSettings, model: string, params: GenerateContentRequest, chromeAdapter?: ChromeAdapter, singleRequestOptions?: SingleRequestOptions): Promise<GenerateContentStreamResult>;
+export declare function generateContentStream(apiSettings: ApiSettings, model: string, params: GenerateContentRequest, chromeAdapter?: ChromeAdapter, singleRequestOptions?: SingleRequestOptions): Promise<GenerateContentStreamResult & {
+    firstValue?: GenerateContentResponse;
+}>;
 export declare function templateGenerateContent(apiSettings: ApiSettings, templateId: string, templateParams: object, singleRequestOptions?: SingleRequestOptions): Promise<GenerateContentResult>;
 export declare function templateGenerateContentStream(apiSettings: ApiSettings, templateId: string, templateParams: object, singleRequestOptions?: SingleRequestOptions): Promise<GenerateContentStreamResult>;
 export declare function generateContent(apiSettings: ApiSettings, model: string, params: GenerateContentRequest, chromeAdapter?: ChromeAdapter, singleRequestOptions?: SingleRequestOptions): Promise<GenerateContentResult>;

package/dist/esm/src/requests/response-helpers.d.ts

@@ -36,7 +36,7 @@ export declare function getText(response: GenerateContentResponse, partFilter: (
 /**
  * Returns every {@link FunctionCall} associated with first candidate.
  */
-export declare function getFunctionCalls(response: GenerateContentResponse): FunctionCall[] | undefined;
+export declare function getFunctionCalls(response?: GenerateContentResponse): FunctionCall[] | undefined;
 /**
  * Returns every {@link InlineDataPart} in the first candidate if present.
  *

package/dist/esm/src/requests/stream-reader.d.ts

@@ -25,7 +25,9 @@ import { InferenceSource } from '../public-types';
  *
  * @param response - Response from a fetch call
  */
-export declare function processStream(response: Response, apiSettings: ApiSettings, inferenceSource?: InferenceSource): GenerateContentStreamResult;
+export declare function processStream(response: Response, apiSettings: ApiSettings, inferenceSource?: InferenceSource): Promise<GenerateContentStreamResult & {
+    firstValue?: GenerateContentResponse;
+}>;
 /**
  * Reads a raw string stream, buffers incomplete chunks, and yields parsed JSON objects.
  */

package/dist/esm/src/types/content.d.ts

@@ -243,6 +243,7 @@ export interface FunctionResponse {
     id?: string;
     name: string;
     response: object;
+    parts?: Part[];
 }
 /**
  * Interface for sending an image.

package/dist/esm/src/types/requests.d.ts

@@ -231,6 +231,15 @@ export interface RequestOptions {
      * (used regardless of your chosen Gemini API provider).
      */
     baseUrl?: string;
+    /**
+     * Limits amount of sequential function calls the SDK can make during automatic
+     * function calling, in order to prevent infinite loops. If not specified,
+     * this value defaults to 10.
+     *
+     * When it reaches this limit, it will return the last response received
+     * from the model, whether it is a text response or further function calls.
+     */
+    maxSequentalFunctionCalls?: number;
 }
 /**
  * Options that can be provided per-request.
@@ -304,6 +313,11 @@ export interface FunctionDeclaration {
      * case-sensitive. For a function with no parameters, this can be left unset.
      */
     parameters?: ObjectSchema | ObjectSchemaRequest;
+    /**
+     * Reference to an actual function to call. Specifying this will cause the
+     * function to be called automatically when requested by the model.
+     */
+    functionReference?: Function;
 }
 /**
  * A tool that allows a Gemini model to connect to Google Search to access and incorporate

package/dist/index.cjs.js

@@ -8,7 +8,7 @@ var util = require('@firebase/util');
 var logger$1 = require('@firebase/logger');
 
 var name = "@firebase/ai";
-var version = "2.8.0";
+var version = "2.9.0-canary.78384d32c";
 
 /**
  * @license
@@ -1780,6 +1780,9 @@ function getText(response, partFilter) {
  * Returns every {@link FunctionCall} associated with first candidate.
  */
 function getFunctionCalls(response) {
+    if (!response) {
+        return undefined;
+    }
     const functionCalls = [];
     if (response.candidates?.[0].content?.parts) {
         for (const part of response.candidates?.[0].content?.parts) {
@@ -2075,15 +2078,44 @@ const responseLineRE = /^data\: (.*)(?:\n\n|\r\r|\r\n\r\n)/;
  *
  * @param response - Response from a fetch call
  */
-function processStream(response, apiSettings, inferenceSource) {
+async function processStream(response, apiSettings, inferenceSource) {
     const inputStream = response.body.pipeThrough(new TextDecoderStream('utf8', { fatal: true }));
     const responseStream = getResponseStream(inputStream);
     // We split the stream so the user can iterate over partial results (stream1)
     // while we aggregate the full result for history/final response (stream2).
     const [stream1, stream2] = responseStream.tee();
+    const { response: internalResponse, firstValue } = await processStreamInternal(stream2, apiSettings, inferenceSource);
     return {
         stream: generateResponseSequence(stream1, apiSettings, inferenceSource),
-        response: getResponsePromise(stream2, apiSettings, inferenceSource)
+        response: internalResponse,
+        firstValue
+    };
+}
+/**
+ * Consumes streams teed from the input stream for internal needs.
+ * The streams need to be teed because each stream can only be consumed
+ * by one reader.
+ *
+ * "streamForPeek"
+ * This tee is used to peek at the first value for relevant information
+ * that we need to evaluate before returning the stream handle to the
+ * client. For example, we need to check if the response is a function
+ * call that may need to be handled by automatic function calling before
+ * returning a response to the client.
+ *
+ * "streamForAggregation"
+ * We iterate through this tee independently from the user and aggregate
+ * it into a single response when the stream is complete. We need this
+ * aggregate object to add to chat history when using ChatSession. It's
+ * also provided to the user if they want it.
+ */
+async function processStreamInternal(stream, apiSettings, inferenceSource) {
+    const [streamForPeek, streamForAggregation] = stream.tee();
+    const reader = streamForPeek.getReader();
+    const { value } = await reader.read();
+    return {
+        firstValue: value,
+        response: getResponsePromise(streamForAggregation, apiSettings, inferenceSource)
     };
 }
 async function getResponsePromise(stream, apiSettings, inferenceSource) {
@@ -2654,6 +2686,11 @@ function validateChatHistory(history) {
  * by the user, preventing duplicate console logs.
  */
 const SILENT_ERROR = 'SILENT_ERROR';
+/**
+ * Prevent infinite loop if the model continues to request sequential
+ * function calls during automatic function calling.
+ */
+const DEFAULT_MAX_SEQUENTIAL_FUNCTION_CALLS = 10;
 /**
  * ChatSession class that enables sending chat messages and stores
  * history of sent and received messages so far.
@@ -2688,48 +2725,89 @@ class ChatSession {
         return this._history;
     }
     /**
-     * Sends a chat message and receives a non-streaming
-     * {@link GenerateContentResult}
+     * Format Content into a request for generateContent or
+     * generateContentStream.
+     * @internal
      */
-    async sendMessage(request, singleRequestOptions) {
-        await this._sendPromise;
-        const newContent = formatNewContent(request);
-        const generateContentRequest = {
+    _formatRequest(incomingContent, tempHistory) {
+        return {
             safetySettings: this.params?.safetySettings,
             generationConfig: this.params?.generationConfig,
             tools: this.params?.tools,
             toolConfig: this.params?.toolConfig,
             systemInstruction: this.params?.systemInstruction,
-            contents: [...this._history, newContent]
+            contents: [...this._history, ...tempHistory, incomingContent]
         };
+    }
+    /**
+     * Sends a chat message and receives a non-streaming
+     * {@link GenerateContentResult}
+     */
+    async sendMessage(request, singleRequestOptions) {
         let finalResult = {};
-        this._sendPromise = this._sendPromise
-            .then(() => generateContent(this._apiSettings, this.model, generateContentRequest, this.chromeAdapter, {
-            ...this.requestOptions,
-            ...singleRequestOptions
-        }))
-            .then(result => {
-            // TODO: Make this update atomic. If creating `responseContent` throws,
-            // history will contain the user message but not the response, causing
-            // validation errors on the next request.
-            if (result.response.candidates &&
-                result.response.candidates.length > 0) {
-                this._history.push(newContent);
-                const responseContent = {
-                    parts: result.response.candidates?.[0].content.parts || [],
-                    role: result.response.candidates?.[0].content.role || 'model'
-                };
-                this._history.push(responseContent);
-            }
-            else {
-                const blockErrorMessage = formatBlockErrorMessage(result.response);
-                if (blockErrorMessage) {
-                    logger.warn(`sendMessage() was unsuccessful. ${blockErrorMessage}. Inspect response object for details.`);
+        await this._sendPromise;
+        /**
+         * Temporarily store multiple turns for cases like automatic function
+         * calling, only writing them to official history when the entire
+         * sequence has completed successfully.
+         */
+        const tempHistory = [];
+        this._sendPromise = this._sendPromise.then(async () => {
+            let functionCalls;
+            let functionCallTurnCount = 0;
+            const functionCallMaxTurns = this.requestOptions?.maxSequentalFunctionCalls ??
+                DEFAULT_MAX_SEQUENTIAL_FUNCTION_CALLS;
+            // Repeats until model returns a response with no function calls
+            // or until `functionCallMaxTurns` is met or exceeded.
+            do {
+                let formattedContent;
+                if (functionCalls) {
+                    functionCallTurnCount++;
+                    const functionResponseParts = await this._callFunctionsAsNeeded(functionCalls);
+                    formattedContent = formatNewContent(functionResponseParts);
+                }
+                else {
+                    formattedContent = formatNewContent(request);
                 }
+                const formattedRequest = this._formatRequest(formattedContent, tempHistory);
+                tempHistory.push(formattedContent);
+                const result = await generateContent(this._apiSettings, this.model, formattedRequest, this.chromeAdapter, {
+                    ...this.requestOptions,
+                    ...singleRequestOptions
+                });
+                if (result) {
+                    finalResult = result;
+                    functionCalls = this._getCallableFunctionCalls(result.response);
+                    if (result.response.candidates &&
+                        result.response.candidates.length > 0) {
+                        // TODO: Make this update atomic. If creating `responseContent` throws,
+                        // history will contain the user message but not the response, causing
+                        // validation errors on the next request.
+                        const responseContent = {
+                            parts: result.response.candidates?.[0].content.parts || [],
+                            // Response seems to come back without a role set.
+                            role: result.response.candidates?.[0].content.role || 'model'
+                        };
+                        tempHistory.push(responseContent);
+                    }
+                    else {
+                        const blockErrorMessage = formatBlockErrorMessage(result.response);
+                        if (blockErrorMessage) {
+                            logger.warn(`sendMessage() was unsuccessful. ${blockErrorMessage}. Inspect response object for details.`);
+                        }
+                    }
+                }
+                else {
+                    functionCalls = undefined;
+                }
+            } while (functionCalls && functionCallTurnCount < functionCallMaxTurns);
+            if (functionCalls && functionCallTurnCount >= functionCallMaxTurns) {
+                logger.warn(`Automatic function calling exceeded the limit of` +
+                    ` ${functionCallMaxTurns} function calls. Returning last model response.`);
             }
-            finalResult = result;
         });
         await this._sendPromise;
+        this._history = this._history.concat(tempHistory);
         return finalResult;
     }
     /**
@@ -2739,23 +2817,62 @@ class ChatSession {
      */
     async sendMessageStream(request, singleRequestOptions) {
         await this._sendPromise;
-        const newContent = formatNewContent(request);
-        const generateContentRequest = {
-            safetySettings: this.params?.safetySettings,
-            generationConfig: this.params?.generationConfig,
-            tools: this.params?.tools,
-            toolConfig: this.params?.toolConfig,
-            systemInstruction: this.params?.systemInstruction,
-            contents: [...this._history, newContent]
+        /**
+         * Temporarily store multiple turns for cases like automatic function
+         * calling, only writing them to official history when the entire
+         * sequence has completed successfully.
+         */
+        const tempHistory = [];
+        const callGenerateContentStream = async () => {
+            let functionCalls;
+            let functionCallTurnCount = 0;
+            const functionCallMaxTurns = this.requestOptions?.maxSequentalFunctionCalls ??
+                DEFAULT_MAX_SEQUENTIAL_FUNCTION_CALLS;
+            let result;
+            // Repeats until model returns a response with no function calls
+            // or until `functionCallMaxTurns` is met or exceeded.
+            do {
+                let formattedContent;
+                if (functionCalls) {
+                    functionCallTurnCount++;
+                    const functionResponseParts = await this._callFunctionsAsNeeded(functionCalls);
+                    formattedContent = formatNewContent(functionResponseParts);
+                }
+                else {
+                    formattedContent = formatNewContent(request);
+                }
+                tempHistory.push(formattedContent);
+                const formattedRequest = this._formatRequest(formattedContent, tempHistory);
+                result = await generateContentStream(this._apiSettings, this.model, formattedRequest, this.chromeAdapter, {
+                    ...this.requestOptions,
+                    ...singleRequestOptions
+                });
+                functionCalls = this._getCallableFunctionCalls(result.firstValue);
+                if (functionCalls &&
+                    result.firstValue &&
+                    result.firstValue.candidates &&
+                    result.firstValue.candidates.length > 0) {
+                    const responseContent = {
+                        ...result.firstValue.candidates[0].content
+                    };
+                    if (!responseContent.role) {
+                        responseContent.role = 'model';
+                    }
+                    tempHistory.push(responseContent);
+                }
+            } while (functionCalls && functionCallTurnCount < functionCallMaxTurns);
+            if (functionCalls && functionCallTurnCount >= functionCallMaxTurns) {
+                logger.warn(`Automatic function calling exceeded the limit of` +
+                    ` ${functionCallMaxTurns} function calls. Returning last model response.`);
+            }
+            return { stream: result.stream, response: result.response };
         };
-        const streamPromise = generateContentStream(this._apiSettings, this.model, generateContentRequest, this.chromeAdapter, {
-            ...this.requestOptions,
-            ...singleRequestOptions
-        });
-        // We hook into the chain to update history, but we don't block the
-        // return of `streamPromise` to the user.
+        const streamPromise = callGenerateContentStream();
+        // Add onto the chain.
         this._sendPromise = this._sendPromise
-            .then(() => streamPromise)
+            .then(async () => streamPromise)
+            // This must be handled to avoid unhandled rejection, but jump
+            // to the final catch block with a label to not log this error.
             .catch(_ignored => {
             // If the initial fetch fails, the user's `streamPromise` rejects.
             // We swallow the error here to prevent double logging in the final catch.
@@ -2768,7 +2885,7 @@ class ChatSession {
             // TODO: Move response validation logic upstream to `stream-reader` so
             // errors propagate to the user's `result.response` promise.
             if (response.candidates && response.candidates.length > 0) {
-                this._history.push(newContent);
+                this._history = this._history.concat(tempHistory);
                 // TODO: Validate that `response.candidates[0].content` is not null.
                 const responseContent = { ...response.candidates[0].content };
                 if (!responseContent.role) {
@@ -2791,6 +2908,75 @@ class ChatSession {
         });
         return streamPromise;
     }
+    /**
+     * Get function calls that the SDK has references to actually call.
+     * This is all-or-nothing. If the model is requesting multiple
+     * function calls, all of them must have references in order for
+     * automatic function calling to work.
+     *
+     * @internal
+     */
+    _getCallableFunctionCalls(response) {
+        const functionDeclarationsTool = this.params?.tools?.find(tool => tool.functionDeclarations);
+        if (!functionDeclarationsTool?.functionDeclarations) {
+            return;
+        }
+        const functionCalls = getFunctionCalls(response);
+        if (!functionCalls) {
+            return;
+        }
+        for (const functionCall of functionCalls) {
+            const hasFunctionReference = functionDeclarationsTool.functionDeclarations?.some(declaration => declaration.name === functionCall.name &&
+                typeof declaration.functionReference === 'function');
+            if (!hasFunctionReference) {
+                return;
+            }
+        }
+        return functionCalls;
+    }
+    /**
+     * Call user-defined functions if requested by the model, and return
+     * the response that should be sent to the model.
+     * @internal
+     */
+    async _callFunctionsAsNeeded(functionCalls) {
+        const activeCallList = new Map();
+        const promiseList = [];
+        const functionDeclarationsTool = this.params?.tools?.find(tool => tool.functionDeclarations);
+        if (functionDeclarationsTool &&
+            functionDeclarationsTool.functionDeclarations) {
+            for (const functionCall of functionCalls) {
+                const functionDeclaration = functionDeclarationsTool.functionDeclarations.find(declaration => declaration.name === functionCall.name);
+                if (functionDeclaration?.functionReference) {
+                    const results = Promise.resolve(functionDeclaration.functionReference(functionCall.args)).catch(e => {
+                        const wrappedError = new AIError(AIErrorCode.ERROR, `Error in user-defined function "${functionDeclaration.name}": ${e.message}`);
+                        wrappedError.stack = e.stack;
+                        throw wrappedError;
+                    });
+                    activeCallList.set(functionCall.name, {
+                        id: functionCall.id,
+                        results
+                    });
+                    promiseList.push(results);
+                }
+            }
+            // Wait for promises to finish.
+            await Promise.all(promiseList);
+            const functionResponseParts = [];
+            for (const [name, callData] of activeCallList) {
+                functionResponseParts.push({
+                    functionResponse: {
+                        name,
+                        response: await callData.results
+                    }
+                });
+            }
+            return functionResponseParts;
+        }
+        else {
+            throw new AIError(AIErrorCode.REQUEST_ERROR, `No function declarations were provided in "tools".`);
+        }
+    }
 }
 
 /**
@@ -2894,7 +3080,7 @@ class GenerativeModel extends AIModel {
      */
     async generateContentStream(request, singleRequestOptions) {
         const formattedParams = formatGenerateContentInput(request);
-        return generateContentStream(this._apiSettings, this.model, {
+        const { stream, response } = await generateContentStream(this._apiSettings, this.model, {
             generationConfig: this.generationConfig,
             safetySettings: this.safetySettings,
             tools: this.tools,
@@ -2907,6 +3093,7 @@ class GenerativeModel extends AIModel {
             ...this.requestOptions,
             ...singleRequestOptions
         });
+        return { stream, response };
     }
     /**
      * Gets a new {@link ChatSession} instance which can be used for