@google/genai 1.14.0 → 1.15.0

package/README.md

@@ -339,6 +339,30 @@ if you are specifying those, you need to explicitly provide the full
 `Content[]` structure making it explicit which Parts are 'spoken' by the model,
 or the user. The SDK will throw an exception if you try this.
 
+## Error Handling
+
+To handle errors raised by the API, the SDK provides this [ApiError](https://github.com/googleapis/js-genai/blob/main/src/errors.ts) class.
+
+```typescript
+import {GoogleGenAI} from '@google/genai';
+const GEMINI_API_KEY = process.env.GEMINI_API_KEY;
+
+const ai = new GoogleGenAI({apiKey: GEMINI_API_KEY});
+
+async function main() {
+  await ai.models.generateContent({
+    model: 'non-existent-model',
+    contents: 'Write a 100-word poem.',
+  }).catch((e) => {
+    console.error('error name: ', e.name);
+    console.error('error message: ', e.message);
+    console.error('error status: ', e.status);
+  });
+}
+
+main();
+```
+
 ## How is this different from the other Google AI SDKs
 This SDK (`@google/genai`) is Google Deepmind’s "vanilla" SDK for its generative
 AI offerings, and is where Google Deepmind adds new AI features.

package/dist/genai.d.ts

@@ -1885,6 +1885,14 @@ export declare interface EnterpriseWebSearch {
     excludeDomains?: string[];
 }
 
+/** An entity representing the segmented area. */
+export declare interface EntityLabel {
+    /** The label of the segmented entity. */
+    label?: string;
+    /** The confidence score of the detected label. */
+    score?: number;
+}
+
 /** The environment being operated. */
 export declare enum Environment {
     /**
@@ -2702,6 +2710,14 @@ export declare interface GeneratedImage {
     enhancedPrompt?: string;
 }
 
+/** A generated image mask. */
+export declare interface GeneratedImageMask {
+    /** The generated image mask. */
+    mask?: Image_2;
+    /** The detected entities on the segmented area. */
+    labels?: EntityLabel[];
+}
+
 /** A generated video. */
 export declare interface GeneratedVideo {
     /** The output video */
@@ -2840,6 +2856,12 @@ export declare interface GenerateVideosConfig {
     generateAudio?: boolean;
     /** Image to use as the last frame of generated videos. Only supported for image to video use cases. */
     lastFrame?: Image_2;
+    /** The images to use as the references to generate the videos.
+     If this field is provided, the text prompt field must also be provided.
+     The image, video, or last_frame field are not supported. Each image must
+     be associated with a type. Veo 2 supports up to 3 asset images *or* 1
+     style image. */
+    referenceImages?: VideoGenerationReferenceImage[];
     /** Compression quality of the generated videos. */
     compressionQuality?: VideoCompressionQuality;
 }
@@ -5127,6 +5149,27 @@ export declare class Models extends BaseModule {
      * ```
      */
     recontextImage(params: types.RecontextImageParameters): Promise<types.RecontextImageResponse>;
+    /**
+     * Segments an image, creating a mask of a specified area.
+     *
+     * @param params - The parameters for segmenting an image.
+     * @return The response from the API.
+     *
+     * @example
+     * ```ts
+     * const response = await ai.models.segmentImage({
+     *  model: 'image-segmentation-001',
+     *  source: {
+     *    image: image,
+     *  },
+     *  config: {
+     *    mode: 'foreground',
+     *  },
+     * });
+     * console.log(response?.generatedMasks?.[0]?.mask?.imageBytes);
+     * ```
+     */
+    segmentImage(params: types.SegmentImageParameters): Promise<types.SegmentImageResponse>;
     /**
      * Fetches information about a model by name.
      *
@@ -5969,6 +6012,12 @@ export declare interface Schema {
 
 export declare type SchemaUnion = Schema | unknown;
 
+/** An image mask representing a brush scribble. */
+export declare interface ScribbleImage {
+    /** The brush scribble to guide segmentation. Valid for the interactive mode. */
+    image?: Image_2;
+}
+
 /** Google search entry point. */
 export declare interface SearchEntryPoint {
     /** Optional. Web content snippet that can be embedded in a web page or an app webview. */
@@ -5990,6 +6039,75 @@ export declare interface Segment {
     text?: string;
 }
 
+/** Configuration for segmenting an image. */
+export declare interface SegmentImageConfig {
+    /** Used to override HTTP request options. */
+    httpOptions?: HttpOptions;
+    /** Abort signal which can be used to cancel the request.
+
+     NOTE: AbortSignal is a client-only operation. Using it to cancel an
+     operation will not cancel the request in the service. You will still
+     be charged usage for any applicable operations.
+     */
+    abortSignal?: AbortSignal;
+    /** The segmentation mode to use. */
+    mode?: SegmentMode;
+    /** The maximum number of predictions to return up to, by top
+     confidence score. */
+    maxPredictions?: number;
+    /** The confidence score threshold for the detections as a decimal
+     value. Only predictions with a confidence score higher than this
+     threshold will be returned. */
+    confidenceThreshold?: number;
+    /** A decimal value representing how much dilation to apply to the
+     masks. 0 for no dilation. 1.0 means the masked area covers the whole
+     image. */
+    maskDilation?: number;
+    /** The binary color threshold to apply to the masks. The threshold
+     can be set to a decimal value between 0 and 255 non-inclusive.
+     Set to -1 for no binary color thresholding. */
+    binaryColorThreshold?: number;
+}
+
+/** The parameters for segmenting an image. */
+export declare interface SegmentImageParameters {
+    /** ID of the model to use. For a list of models, see `Google models
+     <https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models>`_. */
+    model: string;
+    /** A set of source input(s) for image segmentation. */
+    source: SegmentImageSource;
+    /** Configuration for image segmentation. */
+    config?: SegmentImageConfig;
+}
+
+/** The output images response. */
+export declare class SegmentImageResponse {
+    /** List of generated image masks.
+     */
+    generatedMasks?: GeneratedImageMask[];
+}
+
+/** A set of source input(s) for image segmentation. */
+export declare interface SegmentImageSource {
+    /** A text prompt for guiding the model during image segmentation.
+     Required for prompt mode and semantic mode, disallowed for other modes. */
+    prompt?: string;
+    /** The image to be segmented. */
+    image?: Image_2;
+    /** The brush scribble to guide segmentation.
+     Required for the interactive mode, disallowed for other modes. */
+    scribbleImage?: ScribbleImage;
+}
+
+/** Enum that represents the segmentation mode. */
+export declare enum SegmentMode {
+    FOREGROUND = "FOREGROUND",
+    BACKGROUND = "BACKGROUND",
+    PROMPT = "PROMPT",
+    SEMANTIC = "SEMANTIC",
+    INTERACTIVE = "INTERACTIVE"
+}
+
 /** Parameters for sending a message within a chat session.
 
  These parameters are used with the `chat.sendMessage()` method.
@@ -6877,6 +6995,7 @@ declare namespace types {
         ControlReferenceType,
         SubjectReferenceType,
         EditMode,
+        SegmentMode,
         VideoCompressionQuality,
         FileState,
         FileSource,
@@ -7007,6 +7126,13 @@ declare namespace types {
         RecontextImageConfig,
         RecontextImageParameters,
         RecontextImageResponse,
+        ScribbleImage,
+        SegmentImageSource,
+        SegmentImageConfig,
+        SegmentImageParameters,
+        EntityLabel,
+        GeneratedImageMask,
+        SegmentImageResponse,
         GetModelConfig,
         GetModelParameters,
         Endpoint,
@@ -7031,6 +7157,7 @@ declare namespace types {
         TokensInfo,
         ComputeTokensResponse,
         Video,
+        VideoGenerationReferenceImage,
         GenerateVideosConfig,
         GenerateVideosParameters,
         GeneratedVideo,
@@ -7495,6 +7622,17 @@ export declare enum VideoCompressionQuality {
     LOSSLESS = "LOSSLESS"
 }
 
+/** A reference image for video generation. */
+export declare interface VideoGenerationReferenceImage {
+    /** The reference image.
+     */
+    image?: Image_2;
+    /** The type of the reference image, which defines how the reference
+     image will be used to generate the video. Supported values are 'asset'
+     or 'style'. */
+    referenceType?: string;
+}
+
 /** Describes how the video in the Part should be used by the model. */
 export declare interface VideoMetadata {
     /** The frame rate of the video sent to the model. If not specified, the

package/dist/index.cjs

@@ -869,6 +869,15 @@ exports.EditMode = void 0;
     EditMode["EDIT_MODE_BGSWAP"] = "EDIT_MODE_BGSWAP";
     EditMode["EDIT_MODE_PRODUCT_IMAGE"] = "EDIT_MODE_PRODUCT_IMAGE";
 })(exports.EditMode || (exports.EditMode = {}));
+/** Enum that represents the segmentation mode. */
+exports.SegmentMode = void 0;
+(function (SegmentMode) {
+    SegmentMode["FOREGROUND"] = "FOREGROUND";
+    SegmentMode["BACKGROUND"] = "BACKGROUND";
+    SegmentMode["PROMPT"] = "PROMPT";
+    SegmentMode["SEMANTIC"] = "SEMANTIC";
+    SegmentMode["INTERACTIVE"] = "INTERACTIVE";
+})(exports.SegmentMode || (exports.SegmentMode = {}));
 /** Enum that controls the compression quality of the generated videos. */
 exports.VideoCompressionQuality = void 0;
 (function (VideoCompressionQuality) {
@@ -1515,6 +1524,9 @@ class UpscaleImageResponse {
 /** The output images response. */
 class RecontextImageResponse {
 }
+/** The output images response. */
+class SegmentImageResponse {
+}
 class ListModelsResponse {
 }
 class DeleteModelResponse {
@@ -4253,6 +4265,7 @@ class Batches extends BaseModule {
          * ```
          */
         this.create = async (params) => {
+            var _a, _b;
             if (this.apiClient.isVertexAI()) {
                 const timestamp = Date.now();
                 const timestampStr = timestamp.toString();
@@ -4277,6 +4290,55 @@ class Batches extends BaseModule {
                     }
                 }
             }
+            else {
+                if (Array.isArray(params.src) ||
+                    (typeof params.src !== 'string' && params.src.inlinedRequests)) {
+                    // Move system instruction to httpOptions extraBody.
+                    let path = '';
+                    let queryParams = {};
+                    const body = createBatchJobParametersToMldev(this.apiClient, params);
+                    path = formatMap('{model}:batchGenerateContent', body['_url']);
+                    queryParams = body['_query'];
+                    // Move system instruction to 'request':
+                    // {'systemInstruction': system_instruction}
+                    const batch = body['batch'];
+                    const inputConfig = batch['inputConfig'];
+                    const requestsWrapper = inputConfig['requests'];
+                    const requests = requestsWrapper['requests'];
+                    const newRequests = [];
+                    for (const request of requests) {
+                        const requestDict = request;
+                        if (requestDict['systemInstruction']) {
+                            const systemInstructionValue = requestDict['systemInstruction'];
+                            delete requestDict['systemInstruction'];
+                            const requestContent = requestDict['request'];
+                            requestContent['systemInstruction'] = systemInstructionValue;
+                            requestDict['request'] = requestContent;
+                        }
+                        newRequests.push(requestDict);
+                    }
+                    requestsWrapper['requests'] = newRequests;
+                    delete body['config'];
+                    delete body['_url'];
+                    delete body['_query'];
+                    const response = this.apiClient
+                        .request({
+                        path: path,
+                        queryParams: queryParams,
+                        body: JSON.stringify(body),
+                        httpMethod: 'POST',
+                        httpOptions: (_a = params.config) === null || _a === void 0 ? void 0 : _a.httpOptions,
+                        abortSignal: (_b = params.config) === null || _b === void 0 ? void 0 : _b.abortSignal,
+                    })
+                        .then((httpResponse) => {
+                        return httpResponse.json();
+                    });
+                    return response.then((apiResponse) => {
+                        const resp = batchJobFromMldev(apiResponse);
+                        return resp;
+                    });
+                }
+            }
             return await this.createInternal(params);
         };
         /**
@@ -6187,9 +6249,6 @@ function isValidContent(content) {
         if (part === undefined || Object.keys(part).length === 0) {
             return false;
         }
-        if (!part.thought && part.text !== undefined && part.text === '') {
-            return false;
-        }
     }
     return true;
 }
@@ -6512,7 +6571,7 @@ const CONTENT_TYPE_HEADER = 'Content-Type';
 const SERVER_TIMEOUT_HEADER = 'X-Server-Timeout';
 const USER_AGENT_HEADER = 'User-Agent';
 const GOOGLE_API_CLIENT_HEADER = 'x-goog-api-client';
-const SDK_VERSION = '1.14.0'; // x-release-please-version
+const SDK_VERSION = '1.15.0'; // x-release-please-version
 const LIBRARY_LABEL = `google-genai-sdk/${SDK_VERSION}`;
 const VERTEX_AI_API_DEFAULT_VERSION = 'v1beta1';
 const GOOGLE_AI_API_DEFAULT_VERSION = 'v1beta';
@@ -11091,6 +11150,9 @@ function generateVideosConfigToMldev(fromObject, parentObject) {
     if (getValueByPath(fromObject, ['lastFrame']) !== undefined) {
         throw new Error('lastFrame parameter is not supported in Gemini API.');
     }
+    if (getValueByPath(fromObject, ['referenceImages']) !== undefined) {
+        throw new Error('referenceImages parameter is not supported in Gemini API.');
+    }
     if (getValueByPath(fromObject, ['compressionQuality']) !== undefined) {
         throw new Error('compressionQuality parameter is not supported in Gemini API.');
     }
@@ -12401,6 +12463,78 @@ function recontextImageParametersToVertex(apiClient, fromObject) {
     }
     return toObject;
 }
+function scribbleImageToVertex(fromObject) {
+    const toObject = {};
+    const fromImage = getValueByPath(fromObject, ['image']);
+    if (fromImage != null) {
+        setValueByPath(toObject, ['image'], imageToVertex(fromImage));
+    }
+    return toObject;
+}
+function segmentImageSourceToVertex(fromObject, parentObject) {
+    const toObject = {};
+    const fromPrompt = getValueByPath(fromObject, ['prompt']);
+    if (parentObject !== undefined && fromPrompt != null) {
+        setValueByPath(parentObject, ['instances[0]', 'prompt'], fromPrompt);
+    }
+    const fromImage = getValueByPath(fromObject, ['image']);
+    if (parentObject !== undefined && fromImage != null) {
+        setValueByPath(parentObject, ['instances[0]', 'image'], imageToVertex(fromImage));
+    }
+    const fromScribbleImage = getValueByPath(fromObject, [
+        'scribbleImage',
+    ]);
+    if (parentObject !== undefined && fromScribbleImage != null) {
+        setValueByPath(parentObject, ['instances[0]', 'scribble'], scribbleImageToVertex(fromScribbleImage));
+    }
+    return toObject;
+}
+function segmentImageConfigToVertex(fromObject, parentObject) {
+    const toObject = {};
+    const fromMode = getValueByPath(fromObject, ['mode']);
+    if (parentObject !== undefined && fromMode != null) {
+        setValueByPath(parentObject, ['parameters', 'mode'], fromMode);
+    }
+    const fromMaxPredictions = getValueByPath(fromObject, [
+        'maxPredictions',
+    ]);
+    if (parentObject !== undefined && fromMaxPredictions != null) {
+        setValueByPath(parentObject, ['parameters', 'maxPredictions'], fromMaxPredictions);
+    }
+    const fromConfidenceThreshold = getValueByPath(fromObject, [
+        'confidenceThreshold',
+    ]);
+    if (parentObject !== undefined && fromConfidenceThreshold != null) {
+        setValueByPath(parentObject, ['parameters', 'confidenceThreshold'], fromConfidenceThreshold);
+    }
+    const fromMaskDilation = getValueByPath(fromObject, ['maskDilation']);
+    if (parentObject !== undefined && fromMaskDilation != null) {
+        setValueByPath(parentObject, ['parameters', 'maskDilation'], fromMaskDilation);
+    }
+    const fromBinaryColorThreshold = getValueByPath(fromObject, [
+        'binaryColorThreshold',
+    ]);
+    if (parentObject !== undefined && fromBinaryColorThreshold != null) {
+        setValueByPath(parentObject, ['parameters', 'binaryColorThreshold'], fromBinaryColorThreshold);
+    }
+    return toObject;
+}
+function segmentImageParametersToVertex(apiClient, fromObject) {
+    const toObject = {};
+    const fromModel = getValueByPath(fromObject, ['model']);
+    if (fromModel != null) {
+        setValueByPath(toObject, ['_url', 'model'], tModel(apiClient, fromModel));
+    }
+    const fromSource = getValueByPath(fromObject, ['source']);
+    if (fromSource != null) {
+        setValueByPath(toObject, ['config'], segmentImageSourceToVertex(fromSource, toObject));
+    }
+    const fromConfig = getValueByPath(fromObject, ['config']);
+    if (fromConfig != null) {
+        setValueByPath(toObject, ['config'], segmentImageConfigToVertex(fromConfig, toObject));
+    }
+    return toObject;
+}
 function getModelParametersToVertex(apiClient, fromObject) {
     const toObject = {};
     const fromModel = getValueByPath(fromObject, ['model']);
@@ -12569,6 +12703,20 @@ function videoToVertex(fromObject) {
     }
     return toObject;
 }
+function videoGenerationReferenceImageToVertex(fromObject) {
+    const toObject = {};
+    const fromImage = getValueByPath(fromObject, ['image']);
+    if (fromImage != null) {
+        setValueByPath(toObject, ['image'], imageToVertex(fromImage));
+    }
+    const fromReferenceType = getValueByPath(fromObject, [
+        'referenceType',
+    ]);
+    if (fromReferenceType != null) {
+        setValueByPath(toObject, ['referenceType'], fromReferenceType);
+    }
+    return toObject;
+}
 function generateVideosConfigToVertex(fromObject, parentObject) {
     const toObject = {};
     const fromNumberOfVideos = getValueByPath(fromObject, [
@@ -12635,6 +12783,18 @@ function generateVideosConfigToVertex(fromObject, parentObject) {
     if (parentObject !== undefined && fromLastFrame != null) {
         setValueByPath(parentObject, ['instances[0]', 'lastFrame'], imageToVertex(fromLastFrame));
     }
+    const fromReferenceImages = getValueByPath(fromObject, [
+        'referenceImages',
+    ]);
+    if (parentObject !== undefined && fromReferenceImages != null) {
+        let transformedList = fromReferenceImages;
+        if (Array.isArray(transformedList)) {
+            transformedList = transformedList.map((item) => {
+                return videoGenerationReferenceImageToVertex(item);
+            });
+        }
+        setValueByPath(parentObject, ['instances[0]', 'referenceImages'], transformedList);
+    }
     const fromCompressionQuality = getValueByPath(fromObject, [
         'compressionQuality',
     ]);
@@ -13684,6 +13844,50 @@ function recontextImageResponseFromVertex(fromObject) {
     }
     return toObject;
 }
+function entityLabelFromVertex(fromObject) {
+    const toObject = {};
+    const fromLabel = getValueByPath(fromObject, ['label']);
+    if (fromLabel != null) {
+        setValueByPath(toObject, ['label'], fromLabel);
+    }
+    const fromScore = getValueByPath(fromObject, ['score']);
+    if (fromScore != null) {
+        setValueByPath(toObject, ['score'], fromScore);
+    }
+    return toObject;
+}
+function generatedImageMaskFromVertex(fromObject) {
+    const toObject = {};
+    const fromMask = getValueByPath(fromObject, ['_self']);
+    if (fromMask != null) {
+        setValueByPath(toObject, ['mask'], imageFromVertex(fromMask));
+    }
+    const fromLabels = getValueByPath(fromObject, ['labels']);
+    if (fromLabels != null) {
+        let transformedList = fromLabels;
+        if (Array.isArray(transformedList)) {
+            transformedList = transformedList.map((item) => {
+                return entityLabelFromVertex(item);
+            });
+        }
+        setValueByPath(toObject, ['labels'], transformedList);
+    }
+    return toObject;
+}
+function segmentImageResponseFromVertex(fromObject) {
+    const toObject = {};
+    const fromGeneratedMasks = getValueByPath(fromObject, ['predictions']);
+    if (fromGeneratedMasks != null) {
+        let transformedList = fromGeneratedMasks;
+        if (Array.isArray(transformedList)) {
+            transformedList = transformedList.map((item) => {
+                return generatedImageMaskFromVertex(item);
+            });
+        }
+        setValueByPath(toObject, ['generatedMasks'], transformedList);
+    }
+    return toObject;
+}
 function endpointFromVertex(fromObject) {
     const toObject = {};
     const fromName = getValueByPath(fromObject, ['endpoint']);
@@ -15800,6 +16004,61 @@ class Models extends BaseModule {
             throw new Error('This method is only supported by the Vertex AI.');
         }
     }
+    /**
+     * Segments an image, creating a mask of a specified area.
+     *
+     * @param params - The parameters for segmenting an image.
+     * @return The response from the API.
+     *
+     * @example
+     * ```ts
+     * const response = await ai.models.segmentImage({
+     *  model: 'image-segmentation-001',
+     *  source: {
+     *    image: image,
+     *  },
+     *  config: {
+     *    mode: 'foreground',
+     *  },
+     * });
+     * console.log(response?.generatedMasks?.[0]?.mask?.imageBytes);
+     * ```
+     */
+    async segmentImage(params) {
+        var _a, _b;
+        let response;
+        let path = '';
+        let queryParams = {};
+        if (this.apiClient.isVertexAI()) {
+            const body = segmentImageParametersToVertex(this.apiClient, params);
+            path = formatMap('{model}:predict', body['_url']);
+            queryParams = body['_query'];
+            delete body['config'];
+            delete body['_url'];
+            delete body['_query'];
+            response = this.apiClient
+                .request({
+                path: path,
+                queryParams: queryParams,
+                body: JSON.stringify(body),
+                httpMethod: 'POST',
+                httpOptions: (_a = params.config) === null || _a === void 0 ? void 0 : _a.httpOptions,
+                abortSignal: (_b = params.config) === null || _b === void 0 ? void 0 : _b.abortSignal,
+            })
+                .then((httpResponse) => {
+                return httpResponse.json();
+            });
+            return response.then((apiResponse) => {
+                const resp = segmentImageResponseFromVertex(apiResponse);
+                const typedResp = new SegmentImageResponse();
+                Object.assign(typedResp, resp);
+                return typedResp;
+            });
+        }
+        else {
+            throw new Error('This method is only supported by the Vertex AI.');
+        }
+    }
     /**
      * Fetches information about a model by name.
      *
@@ -18438,6 +18697,7 @@ exports.Pager = Pager;
 exports.RawReferenceImage = RawReferenceImage;
 exports.RecontextImageResponse = RecontextImageResponse;
 exports.ReplayResponse = ReplayResponse;
+exports.SegmentImageResponse = SegmentImageResponse;
 exports.Session = Session;
 exports.StyleReferenceImage = StyleReferenceImage;
 exports.SubjectReferenceImage = SubjectReferenceImage;