@google/genai 1.3.0 → 1.5.0

package/README.md

@@ -21,7 +21,21 @@ The Google Gen AI SDK is designed to work with Gemini 2.0 features.
 
 ## Prerequisites
 
-* Node.js version 18 or later
+1. Node.js version 20 or later
+
+### The following are required for Vertex AI users (excluding Vertex AI Studio)
+1.  [Select](https://console.cloud.google.com/project) or [create](https://cloud.google.com/resource-manager/docs/creating-managing-projects#creating_a_project) a Google Cloud project.
+1.  [Enable billing for your project](https://cloud.google.com/billing/docs/how-to/modify-project).
+1.  [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).
+1.  [Configure authentication](https://cloud.google.com/docs/authentication) for your project.
+    *   [Install the gcloud CLI](https://cloud.google.com/sdk/docs/install).
+    *   [Initialize the gcloud CLI](https://cloud.google.com/sdk/docs/initializing).
+    *   Create local authentication credentials for your user account:
+
+    ```sh
+    gcloud auth application-default login
+    ```
+A list of accepted authentication options are listed in [GoogleAuthOptions](https://github.com/googleapis/google-auth-library-nodejs/blob/3ae120d0a45c95e36c59c9ac8286483938781f30/src/auth/googleauth.ts#L87) interface of google-auth-library-node.js GitHub repo.
 
 ## Installation
 
@@ -98,6 +112,33 @@ const ai = new GoogleGenAI({
 });
 ```
 
+### (Optional) (NodeJS only) Using environment variables:
+
+For NodeJS environments, you can create a client by configuring the necessary
+environment variables. Configuration setup instructions depends on whether
+you're using the Gemini Developer API or the Gemini API in Vertex AI.
+
+**Gemini Developer API:** Set `GOOGLE_API_KEY` as shown below:
+
+```bash
+export GOOGLE_API_KEY='your-api-key'
+```
+
+**Gemini API on Vertex AI:** Set `GOOGLE_GENAI_USE_VERTEXAI`,
+`GOOGLE_CLOUD_PROJECT` and `GOOGLE_CLOUD_LOCATION`, as shown below:
+
+```bash
+export GOOGLE_GENAI_USE_VERTEXAI=true
+export GOOGLE_CLOUD_PROJECT='your-project-id'
+export GOOGLE_CLOUD_LOCATION='us-central1'
+```
+
+```typescript
+import {GoogleGenAI} from '@google/genai';
+
+const ai = new GoogleGenAI();
+```
+
 ## API Selection
 
 By default, the SDK uses the beta API endpoints provided by Google to support

package/dist/genai.d.ts

@@ -456,7 +456,7 @@ export declare class Caches extends BaseModule {
      * @example
      * ```ts
      * const cachedContents = await ai.caches.list({config: {'pageSize': 2}});
-     * for (const cachedContent of cachedContents) {
+     * for await (const cachedContent of cachedContents) {
      *   console.log(cachedContent);
      * }
      * ```
@@ -556,6 +556,10 @@ export declare interface CallableToolConfig {
      * Specifies the model's behavior after invoking this tool.
      */
     behavior?: Behavior;
+    /**
+     * Timeout for remote calls in milliseconds. Note this timeout applies only to
+     * tool remote calls, and not making HTTP requests to the API. */
+    timeout?: number;
 }
 
 /** A response candidate generated from the model. */
@@ -2426,6 +2430,8 @@ export declare interface GenerateVideosConfig {
     enhancePrompt?: boolean;
     /** Whether to generate audio along with the video. */
     generateAudio?: boolean;
+    /** Image to use as the last frame of generated videos. Only supported for image to video use cases. */
+    lastFrame?: Image_2;
 }
 
 /** A video generation operation. */
@@ -2442,7 +2448,7 @@ export declare interface GenerateVideosOperation {
     response?: GenerateVideosResponse;
 }
 
-/** Class that represents the parameters for generating an image. */
+/** Class that represents the parameters for generating videos. */
 export declare interface GenerateVideosParameters {
     /** ID of the model to use. For a list of models, see `Google models
      <https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models>`_. */
@@ -2450,8 +2456,11 @@ export declare interface GenerateVideosParameters {
     /** The text prompt for generating the videos. Optional for image to video use cases. */
     prompt?: string;
     /** The input image for generating the videos.
-     Optional if prompt is provided. */
+     Optional if prompt or video is provided. */
     image?: Image_2;
+    /** The input video for video extension use cases.
+     Optional if prompt or image is provided. */
+    video?: Video;
     /** Configuration for generating videos. */
     config?: GenerateVideosConfig;
 }
@@ -5985,6 +5994,7 @@ declare namespace types {
         AuthConfig,
         GoogleMaps,
         UrlContext,
+        VertexAISearchDataStoreSpec,
         VertexAISearch,
         VertexRagStoreRagResource,
         RagRetrievalConfigFilter,
@@ -6081,9 +6091,9 @@ declare namespace types {
         ComputeTokensParameters,
         TokensInfo,
         ComputeTokensResponse,
+        Video,
         GenerateVideosConfig,
         GenerateVideosParameters,
-        Video,
         GeneratedVideo,
         GenerateVideosResponse,
         GenerateVideosOperation,
@@ -6435,6 +6445,8 @@ export declare interface UsageMetadata {
 
 /** Retrieve from Vertex AI Search datastore or engine for grounding. datastore and engine are mutually exclusive. See https://cloud.google.com/products/agent-builder */
 export declare interface VertexAISearch {
+    /** Specifications that define the specific DataStores to be searched, along with configurations for those data stores. This is only considered for Engines with multiple data stores. It should only be set if engine is used. */
+    dataStoreSpecs?: VertexAISearchDataStoreSpec[];
     /** Optional. Fully-qualified Vertex AI Search data store resource ID. Format: `projects/{project}/locations/{location}/collections/{collection}/dataStores/{dataStore}` */
     datastore?: string;
     /** Optional. Fully-qualified Vertex AI Search engine resource ID. Format: `projects/{project}/locations/{location}/collections/{collection}/engines/{engine}` */
@@ -6445,6 +6457,14 @@ export declare interface VertexAISearch {
     maxResults?: number;
 }
 
+/** Define data stores within engine to filter on in a search call and configurations for those data stores. For more information, see https://cloud.google.com/generative-ai-app-builder/docs/reference/rpc/google.cloud.discoveryengine.v1#datastorespec */
+export declare interface VertexAISearchDataStoreSpec {
+    /** Full resource name of DataStore, such as Format: `projects/{project}/locations/{location}/collections/{collection}/dataStores/{dataStore}` */
+    dataStore?: string;
+    /** Optional. Filter specification to filter documents in the data store specified by data_store field. For more information on filtering, see [Filtering](https://cloud.google.com/generative-ai-app-builder/docs/filter-search-metadata) */
+    filter?: string;
+}
+
 /** Retrieve from Vertex RAG Store for grounding. */
 export declare interface VertexRagStore {
     /** Optional. Deprecated. Please use rag_resources instead. */
@@ -6455,6 +6475,8 @@ export declare interface VertexRagStore {
     ragRetrievalConfig?: RagRetrievalConfig;
     /** Optional. Number of top k results to return from the selected corpora. */
     similarityTopK?: number;
+    /** Optional. Currently only supported for Gemini Multimodal Live API. In Gemini Multimodal Live API, if `store_context` bool is specified, Gemini will leverage it to automatically memorize the interactions between the client and Gemini, and retrieve context when needed to augment the response generation for users' ongoing and future interactions. */
+    storeContext?: boolean;
     /** Optional. Only return results with vector distance smaller than the threshold. */
     vectorDistanceThreshold?: number;
 }

package/dist/index.cjs

@@ -1,6 +1,7 @@
 'use strict';
 
 var zod = require('zod');
+var types_js = require('@modelcontextprotocol/sdk/types.js');
 
 /**
  * @license
@@ -1925,16 +1926,16 @@ function flattenTypeArrayToAnyOf(typeList, resultingSchema) {
     }
     const listWithoutNull = typeList.filter((type) => type !== 'null');
     if (listWithoutNull.length === 1) {
-        resultingSchema['type'] = Object.keys(exports.Type).includes(listWithoutNull[0].toUpperCase())
-            ? exports.Type[listWithoutNull[0].toUpperCase()]
+        resultingSchema['type'] = Object.values(exports.Type).includes(listWithoutNull[0].toUpperCase())
+            ? listWithoutNull[0].toUpperCase()
             : exports.Type.TYPE_UNSPECIFIED;
     }
     else {
         resultingSchema['anyOf'] = [];
         for (const i of listWithoutNull) {
             resultingSchema['anyOf'].push({
-                'type': Object.keys(exports.Type).includes(i.toUpperCase())
-                    ? exports.Type[i.toUpperCase()]
+                'type': Object.values(exports.Type).includes(i.toUpperCase())
+                    ? i.toUpperCase()
                     : exports.Type.TYPE_UNSPECIFIED,
             });
         }
@@ -2018,7 +2019,7 @@ function processJsonSchema(_jsonSchema) {
                 // beginning of this function.
                 continue;
             }
-            genAISchema['type'] = Object.keys(exports.Type).includes(fieldValue.toUpperCase())
+            genAISchema['type'] = Object.values(exports.Type).includes(fieldValue.toUpperCase())
                 ? fieldValue.toUpperCase()
                 : exports.Type.TYPE_UNSPECIFIED;
         }
@@ -2377,7 +2378,7 @@ function filterToJsonSchema(schema) {
         }
         else if (fieldName === 'type') {
             const typeValue = fieldValue.toUpperCase();
-            filteredSchema[fieldName] = Object.keys(exports.Type).includes(typeValue)
+            filteredSchema[fieldName] = Object.values(exports.Type).includes(typeValue)
                 ? typeValue
                 : exports.Type.TYPE_UNSPECIFIED;
         }
@@ -3054,6 +3055,10 @@ function googleMapsToVertex$2(apiClient, fromObject) {
     }
     return toObject;
 }
+function urlContextToVertex$2() {
+    const toObject = {};
+    return toObject;
+}
 function toolToVertex$2(apiClient, fromObject) {
     const toObject = {};
     const fromFunctionDeclarations = getValueByPath(fromObject, [
@@ -3092,8 +3097,9 @@ function toolToVertex$2(apiClient, fromObject) {
     if (fromGoogleMaps != null) {
         setValueByPath(toObject, ['googleMaps'], googleMapsToVertex$2(apiClient, fromGoogleMaps));
     }
-    if (getValueByPath(fromObject, ['urlContext']) !== undefined) {
-        throw new Error('urlContext parameter is not supported in Vertex AI.');
+    const fromUrlContext = getValueByPath(fromObject, ['urlContext']);
+    if (fromUrlContext != null) {
+        setValueByPath(toObject, ['urlContext'], urlContextToVertex$2());
     }
     const fromCodeExecution = getValueByPath(fromObject, [
         'codeExecution',
@@ -3603,7 +3609,7 @@ class Caches extends BaseModule {
          * @example
          * ```ts
          * const cachedContents = await ai.caches.list({config: {'pageSize': 2}});
-         * for (const cachedContent of cachedContents) {
+         * for await (const cachedContent of cachedContents) {
          *   console.log(cachedContent);
          * }
          * ```
@@ -4348,7 +4354,7 @@ const CONTENT_TYPE_HEADER = 'Content-Type';
 const SERVER_TIMEOUT_HEADER = 'X-Server-Timeout';
 const USER_AGENT_HEADER = 'User-Agent';
 const GOOGLE_API_CLIENT_HEADER = 'x-goog-api-client';
-const SDK_VERSION = '1.3.0'; // x-release-please-version
+const SDK_VERSION = '1.5.0'; // x-release-please-version
 const LIBRARY_LABEL = `google-genai-sdk/${SDK_VERSION}`;
 const VERTEX_AI_API_DEFAULT_VERSION = 'v1beta1';
 const GOOGLE_AI_API_DEFAULT_VERSION = 'v1beta';
@@ -6042,6 +6048,10 @@ function urlContextToMldev$2() {
     const toObject = {};
     return toObject;
 }
+function urlContextToVertex$1() {
+    const toObject = {};
+    return toObject;
+}
 function toolToMldev$2(apiClient, fromObject) {
     const toObject = {};
     const fromFunctionDeclarations = getValueByPath(fromObject, [
@@ -6125,8 +6135,9 @@ function toolToVertex$1(apiClient, fromObject) {
     if (fromGoogleMaps != null) {
         setValueByPath(toObject, ['googleMaps'], googleMapsToVertex$1(apiClient, fromGoogleMaps));
     }
-    if (getValueByPath(fromObject, ['urlContext']) !== undefined) {
-        throw new Error('urlContext parameter is not supported in Vertex AI.');
+    const fromUrlContext = getValueByPath(fromObject, ['urlContext']);
+    if (fromUrlContext != null) {
+        setValueByPath(toObject, ['urlContext'], urlContextToVertex$1());
     }
     const fromCodeExecution = getValueByPath(fromObject, [
         'codeExecution',
@@ -8722,12 +8733,18 @@ function generateVideosConfigToMldev(apiClient, fromObject, parentObject) {
     if (parentObject !== undefined && fromNegativePrompt != null) {
         setValueByPath(parentObject, ['parameters', 'negativePrompt'], fromNegativePrompt);
     }
-    if (getValueByPath(fromObject, ['enhancePrompt']) !== undefined) {
-        throw new Error('enhancePrompt parameter is not supported in Gemini API.');
+    const fromEnhancePrompt = getValueByPath(fromObject, [
+        'enhancePrompt',
+    ]);
+    if (parentObject !== undefined && fromEnhancePrompt != null) {
+        setValueByPath(parentObject, ['parameters', 'enhancePrompt'], fromEnhancePrompt);
     }
     if (getValueByPath(fromObject, ['generateAudio']) !== undefined) {
         throw new Error('generateAudio parameter is not supported in Gemini API.');
     }
+    if (getValueByPath(fromObject, ['lastFrame']) !== undefined) {
+        throw new Error('lastFrame parameter is not supported in Gemini API.');
+    }
     return toObject;
 }
 function generateVideosParametersToMldev(apiClient, fromObject) {
@@ -8744,6 +8761,9 @@ function generateVideosParametersToMldev(apiClient, fromObject) {
     if (fromImage != null) {
         setValueByPath(toObject, ['instances[0]', 'image'], imageToMldev(apiClient, fromImage));
     }
+    if (getValueByPath(fromObject, ['video']) !== undefined) {
+        throw new Error('video parameter is not supported in Gemini API.');
+    }
     const fromConfig = getValueByPath(fromObject, ['config']);
     if (fromConfig != null) {
         setValueByPath(toObject, ['config'], generateVideosConfigToMldev(apiClient, fromConfig, toObject));
@@ -9115,6 +9135,10 @@ function googleMapsToVertex(apiClient, fromObject) {
     }
     return toObject;
 }
+function urlContextToVertex() {
+    const toObject = {};
+    return toObject;
+}
 function toolToVertex(apiClient, fromObject) {
     const toObject = {};
     const fromFunctionDeclarations = getValueByPath(fromObject, [
@@ -9153,8 +9177,9 @@ function toolToVertex(apiClient, fromObject) {
     if (fromGoogleMaps != null) {
         setValueByPath(toObject, ['googleMaps'], googleMapsToVertex(apiClient, fromGoogleMaps));
     }
-    if (getValueByPath(fromObject, ['urlContext']) !== undefined) {
-        throw new Error('urlContext parameter is not supported in Vertex AI.');
+    const fromUrlContext = getValueByPath(fromObject, ['urlContext']);
+    if (fromUrlContext != null) {
+        setValueByPath(toObject, ['urlContext'], urlContextToVertex());
     }
     const fromCodeExecution = getValueByPath(fromObject, [
         'codeExecution',
@@ -10017,6 +10042,22 @@ function computeTokensParametersToVertex(apiClient, fromObject) {
     }
     return toObject;
 }
+function videoToVertex(apiClient, fromObject) {
+    const toObject = {};
+    const fromUri = getValueByPath(fromObject, ['uri']);
+    if (fromUri != null) {
+        setValueByPath(toObject, ['gcsUri'], fromUri);
+    }
+    const fromVideoBytes = getValueByPath(fromObject, ['videoBytes']);
+    if (fromVideoBytes != null) {
+        setValueByPath(toObject, ['bytesBase64Encoded'], tBytes(apiClient, fromVideoBytes));
+    }
+    const fromMimeType = getValueByPath(fromObject, ['mimeType']);
+    if (fromMimeType != null) {
+        setValueByPath(toObject, ['mimeType'], fromMimeType);
+    }
+    return toObject;
+}
 function generateVideosConfigToVertex(apiClient, fromObject, parentObject) {
     const toObject = {};
     const fromNumberOfVideos = getValueByPath(fromObject, [
@@ -10079,6 +10120,10 @@ function generateVideosConfigToVertex(apiClient, fromObject, parentObject) {
     if (parentObject !== undefined && fromGenerateAudio != null) {
         setValueByPath(parentObject, ['parameters', 'generateAudio'], fromGenerateAudio);
     }
+    const fromLastFrame = getValueByPath(fromObject, ['lastFrame']);
+    if (parentObject !== undefined && fromLastFrame != null) {
+        setValueByPath(parentObject, ['instances[0]', 'lastFrame'], imageToVertex(apiClient, fromLastFrame));
+    }
     return toObject;
 }
 function generateVideosParametersToVertex(apiClient, fromObject) {
@@ -10095,6 +10140,10 @@ function generateVideosParametersToVertex(apiClient, fromObject) {
     if (fromImage != null) {
         setValueByPath(toObject, ['instances[0]', 'image'], imageToVertex(apiClient, fromImage));
     }
+    const fromVideo = getValueByPath(fromObject, ['video']);
+    if (fromVideo != null) {
+        setValueByPath(toObject, ['instances[0]', 'video'], videoToVertex(apiClient, fromVideo));
+    }
     const fromConfig = getValueByPath(fromObject, ['config']);
     if (fromConfig != null) {
         setValueByPath(toObject, ['config'], generateVideosConfigToVertex(apiClient, fromConfig, toObject));
@@ -10745,6 +10794,34 @@ function citationMetadataFromVertex(apiClient, fromObject) {
     }
     return toObject;
 }
+function urlMetadataFromVertex(apiClient, fromObject) {
+    const toObject = {};
+    const fromRetrievedUrl = getValueByPath(fromObject, ['retrievedUrl']);
+    if (fromRetrievedUrl != null) {
+        setValueByPath(toObject, ['retrievedUrl'], fromRetrievedUrl);
+    }
+    const fromUrlRetrievalStatus = getValueByPath(fromObject, [
+        'urlRetrievalStatus',
+    ]);
+    if (fromUrlRetrievalStatus != null) {
+        setValueByPath(toObject, ['urlRetrievalStatus'], fromUrlRetrievalStatus);
+    }
+    return toObject;
+}
+function urlContextMetadataFromVertex(apiClient, fromObject) {
+    const toObject = {};
+    const fromUrlMetadata = getValueByPath(fromObject, ['urlMetadata']);
+    if (fromUrlMetadata != null) {
+        let transformedList = fromUrlMetadata;
+        if (Array.isArray(transformedList)) {
+            transformedList = transformedList.map((item) => {
+                return urlMetadataFromVertex(apiClient, item);
+            });
+        }
+        setValueByPath(toObject, ['urlMetadata'], transformedList);
+    }
+    return toObject;
+}
 function candidateFromVertex(apiClient, fromObject) {
     const toObject = {};
     const fromContent = getValueByPath(fromObject, ['content']);
@@ -10767,6 +10844,12 @@ function candidateFromVertex(apiClient, fromObject) {
     if (fromFinishReason != null) {
         setValueByPath(toObject, ['finishReason'], fromFinishReason);
     }
+    const fromUrlContextMetadata = getValueByPath(fromObject, [
+        'urlContextMetadata',
+    ]);
+    if (fromUrlContextMetadata != null) {
+        setValueByPath(toObject, ['urlContextMetadata'], urlContextMetadataFromVertex(apiClient, fromUrlContextMetadata));
+    }
     const fromAvgLogprobs = getValueByPath(fromObject, ['avgLogprobs']);
     if (fromAvgLogprobs != null) {
         setValueByPath(toObject, ['avgLogprobs'], fromAvgLogprobs);
@@ -11354,9 +11437,18 @@ class McpCallableTool {
         for (const functionCall of functionCalls) {
             if (functionCall.name in this.functionNameToMcpClient) {
                 const mcpClient = this.functionNameToMcpClient[functionCall.name];
+                let requestOptions = undefined;
+                // TODO: b/424238654 - Add support for finer grained timeout control.
+                if (this.config.timeout) {
+                    requestOptions = {
+                        timeout: this.config.timeout,
+                    };
+                }
                 const callToolResponse = await mcpClient.callTool({
                     name: functionCall.name,
                     arguments: functionCall.args,
+                    CallToolResultSchema: types_js.CallToolResultSchema,
+                    requestOptions,
                 });
                 functionCallResponseParts.push({
                     functionResponse: {