@google/genai 1.14.0 → 1.16.0

package/README.md

@@ -339,6 +339,30 @@ if you are specifying those, you need to explicitly provide the full
 `Content[]` structure making it explicit which Parts are 'spoken' by the model,
 or the user. The SDK will throw an exception if you try this.
 
+## Error Handling
+
+To handle errors raised by the API, the SDK provides this [ApiError](https://github.com/googleapis/js-genai/blob/main/src/errors.ts) class.
+
+```typescript
+import {GoogleGenAI} from '@google/genai';
+const GEMINI_API_KEY = process.env.GEMINI_API_KEY;
+
+const ai = new GoogleGenAI({apiKey: GEMINI_API_KEY});
+
+async function main() {
+  await ai.models.generateContent({
+    model: 'non-existent-model',
+    contents: 'Write a 100-word poem.',
+  }).catch((e) => {
+    console.error('error name: ', e.name);
+    console.error('error message: ', e.message);
+    console.error('error status: ', e.status);
+  });
+}
+
+main();
+```
+
 ## How is this different from the other Google AI SDKs
 This SDK (`@google/genai`) is Google Deepmind’s "vanilla" SDK for its generative
 AI offerings, and is where Google Deepmind adds new AI features.

package/dist/genai.d.ts

@@ -817,6 +817,27 @@ export declare interface CancelBatchJobParameters {
     config?: CancelBatchJobConfig;
 }
 
+/** Optional parameters for tunings.cancel method. */
+export declare interface CancelTuningJobConfig {
+    /** Used to override HTTP request options. */
+    httpOptions?: HttpOptions;
+    /** Abort signal which can be used to cancel the request.
+
+     NOTE: AbortSignal is a client-only operation. Using it to cancel an
+     operation will not cancel the request in the service. You will still
+     be charged usage for any applicable operations.
+     */
+    abortSignal?: AbortSignal;
+}
+
+/** Parameters for the cancel method. */
+export declare interface CancelTuningJobParameters {
+    /** The resource name of the tuning job. */
+    name: string;
+    /** Optional parameters for the request. */
+    config?: CancelTuningJobConfig;
+}
+
 /** A response candidate generated from the model. */
 export declare interface Candidate {
     /** Contains the multi-part content of the response.
@@ -1561,6 +1582,8 @@ export declare interface DeleteCachedContentParameters {
 
 /** Empty response for caches.delete method. */
 export declare class DeleteCachedContentResponse {
+    /** Used to retain the full HTTP response. */
+    sdkHttpResponse?: HttpResponse;
 }
 
 /** Used to override the default configuration. */
@@ -1586,6 +1609,8 @@ export declare interface DeleteFileParameters {
 
 /** Response for the delete file method. */
 export declare class DeleteFileResponse {
+    /** Used to retain the full HTTP response. */
+    sdkHttpResponse?: HttpResponse;
 }
 
 /** Configuration for deleting a tuned model. */
@@ -1609,6 +1634,8 @@ export declare interface DeleteModelParameters {
 }
 
 export declare class DeleteModelResponse {
+    /** Used to retain the full HTTP response. */
+    sdkHttpResponse?: HttpResponse;
 }
 
 /** The return value of delete operation. */
@@ -1885,6 +1912,14 @@ export declare interface EnterpriseWebSearch {
     excludeDomains?: string[];
 }
 
+/** An entity representing the segmented area. */
+export declare interface EntityLabel {
+    /** The label of the segmented entity. */
+    label?: string;
+    /** The confidence score of the detected label. */
+    score?: number;
+}
+
 /** The environment being operated. */
 export declare enum Environment {
     /**
@@ -2260,7 +2295,11 @@ export declare enum FunctionCallingConfigMode {
     /**
      * Model will not predict any function calls. Model behavior is same as when not passing any function declarations.
      */
-    NONE = "NONE"
+    NONE = "NONE",
+    /**
+     * Model decides to predict either a function call or a natural language response, but will validate function calls with constrained decoding. If "allowed_function_names" are set, the predicted function call will be limited to any one of "allowed_function_names", else the predicted function call will be any one of the provided "function_declarations".
+     */
+    VALIDATED = "VALIDATED"
 }
 
 /** Defines a function that the model can generate JSON inputs for.
@@ -2702,6 +2741,14 @@ export declare interface GeneratedImage {
     enhancedPrompt?: string;
 }
 
+/** A generated image mask. */
+export declare interface GeneratedImageMask {
+    /** The generated image mask. */
+    mask?: Image_2;
+    /** The detected entities on the segmented area. */
+    labels?: EntityLabel[];
+}
+
 /** A generated video. */
 export declare interface GeneratedVideo {
     /** The output video */
@@ -2840,11 +2887,17 @@ export declare interface GenerateVideosConfig {
     generateAudio?: boolean;
     /** Image to use as the last frame of generated videos. Only supported for image to video use cases. */
     lastFrame?: Image_2;
+    /** The images to use as the references to generate the videos.
+     If this field is provided, the text prompt field must also be provided.
+     The image, video, or last_frame field are not supported. Each image must
+     be associated with a type. Veo 2 supports up to 3 asset images *or* 1
+     style image. */
+    referenceImages?: VideoGenerationReferenceImage[];
     /** Compression quality of the generated videos. */
     compressionQuality?: VideoCompressionQuality;
 }
 
-/** A video generation long-running operation. */
+/** A video generation operation. */
 export declare class GenerateVideosOperation implements Operation<GenerateVideosResponse> {
     /** The server-assigned name, which is only unique within the same service that originally returns it. If you use the default HTTP mapping, the `name` should be a resource name ending with `operations/{unique_id}`. */
     name?: string;
@@ -2854,7 +2907,7 @@ export declare class GenerateVideosOperation implements Operation<GenerateVideos
     done?: boolean;
     /** The error result of the operation in case of failure or cancellation. */
     error?: Record<string, unknown>;
-    /** The response if the operation is successful. */
+    /** The generated videos. */
     response?: GenerateVideosResponse;
     /** The full HTTP response. */
     sdkHttpResponse?: HttpResponse;
@@ -2878,6 +2931,8 @@ export declare interface GenerateVideosParameters {
     /** The input video for video extension use cases.
      Optional if prompt or image is provided. */
     video?: Video;
+    /** A set of source input(s) for video generation. */
+    source?: GenerateVideosSource;
     /** Configuration for generating videos. */
     config?: GenerateVideosConfig;
 }
@@ -2892,6 +2947,19 @@ export declare class GenerateVideosResponse {
     raiMediaFilteredReasons?: string[];
 }
 
+/** A set of source input(s) for video generation. */
+export declare interface GenerateVideosSource {
+    /** The text prompt for generating the videos.
+     Optional if image or video is provided. */
+    prompt?: string;
+    /** The input image for generating the videos.
+     Optional if prompt or video is provided. */
+    image?: Image_2;
+    /** The input video for video extension use cases.
+     Optional if prompt or image is provided. */
+    video?: Video;
+}
+
 /** Generation config. */
 export declare interface GenerationConfig {
     /** Optional. Config for model selection. */
@@ -3075,7 +3143,6 @@ export declare interface GetOperationParameters {
     config?: GetOperationConfig;
 }
 
-/** Optional parameters for tunings.get method. */
 export declare interface GetTuningJobConfig {
     /** Used to override HTTP request options. */
     httpOptions?: HttpOptions;
@@ -4630,6 +4697,7 @@ export declare interface LiveServerSessionResumptionUpdate {
     lastConsumedClientMessageIndex?: string;
 }
 
+/** Sent in response to a `LiveGenerateContentSetup` message from the client. */
 export declare interface LiveServerSetupComplete {
     /** The session id of the live session. */
     sessionId?: string;
@@ -5028,7 +5096,9 @@ export declare class Models extends BaseModule {
      * ```ts
      * const operation = await ai.models.generateVideos({
      *  model: 'veo-2.0-generate-001',
-     *  prompt: 'A neon hologram of a cat driving at top speed',
+     *  source: {
+     *    prompt: 'A neon hologram of a cat driving at top speed',
+     *  },
      *  config: {
      *    numberOfVideos: 1
      * });
@@ -5127,6 +5197,27 @@ export declare class Models extends BaseModule {
      * ```
      */
     recontextImage(params: types.RecontextImageParameters): Promise<types.RecontextImageResponse>;
+    /**
+     * Segments an image, creating a mask of a specified area.
+     *
+     * @param params - The parameters for segmenting an image.
+     * @return The response from the API.
+     *
+     * @example
+     * ```ts
+     * const response = await ai.models.segmentImage({
+     *  model: 'image-segmentation-001',
+     *  source: {
+     *    image: image,
+     *  },
+     *  config: {
+     *    mode: 'foreground',
+     *  },
+     * });
+     * console.log(response?.generatedMasks?.[0]?.mask?.imageBytes);
+     * ```
+     */
+    segmentImage(params: types.SegmentImageParameters): Promise<types.SegmentImageResponse>;
     /**
      * Fetches information about a model by name.
      *
@@ -5293,10 +5384,10 @@ export declare interface OperationFromAPIResponseParameters {
 
 /** Parameters for the get method of the operations module. */
 export declare interface OperationGetParameters<T, U extends Operation<T>> {
-    /** The operation to be retrieved. */
-    operation: U;
     /** Used to override the default configuration. */
     config?: GetOperationConfig;
+    /** The operation to be retrieved. */
+    operation: U;
 }
 
 export declare class Operations extends BaseModule {
@@ -5695,6 +5786,8 @@ export declare interface RecontextImageConfig {
     /** Whether allow to generate person images, and restrict to specific
      ages. */
     personGeneration?: PersonGeneration;
+    /** Whether to add a SynthID watermark to the generated images. */
+    addWatermark?: boolean;
     /** MIME type of the generated image. */
     outputMimeType?: string;
     /** Compression quality of the generated image (for ``image/jpeg``
@@ -5969,6 +6062,12 @@ export declare interface Schema {
 
 export declare type SchemaUnion = Schema | unknown;
 
+/** An image mask representing a brush scribble. */
+export declare interface ScribbleImage {
+    /** The brush scribble to guide segmentation. Valid for the interactive mode. */
+    image?: Image_2;
+}
+
 /** Google search entry point. */
 export declare interface SearchEntryPoint {
     /** Optional. Web content snippet that can be embedded in a web page or an app webview. */
@@ -5990,6 +6089,75 @@ export declare interface Segment {
     text?: string;
 }
 
+/** Configuration for segmenting an image. */
+export declare interface SegmentImageConfig {
+    /** Used to override HTTP request options. */
+    httpOptions?: HttpOptions;
+    /** Abort signal which can be used to cancel the request.
+
+     NOTE: AbortSignal is a client-only operation. Using it to cancel an
+     operation will not cancel the request in the service. You will still
+     be charged usage for any applicable operations.
+     */
+    abortSignal?: AbortSignal;
+    /** The segmentation mode to use. */
+    mode?: SegmentMode;
+    /** The maximum number of predictions to return up to, by top
+     confidence score. */
+    maxPredictions?: number;
+    /** The confidence score threshold for the detections as a decimal
+     value. Only predictions with a confidence score higher than this
+     threshold will be returned. */
+    confidenceThreshold?: number;
+    /** A decimal value representing how much dilation to apply to the
+     masks. 0 for no dilation. 1.0 means the masked area covers the whole
+     image. */
+    maskDilation?: number;
+    /** The binary color threshold to apply to the masks. The threshold
+     can be set to a decimal value between 0 and 255 non-inclusive.
+     Set to -1 for no binary color thresholding. */
+    binaryColorThreshold?: number;
+}
+
+/** The parameters for segmenting an image. */
+export declare interface SegmentImageParameters {
+    /** ID of the model to use. For a list of models, see `Google models
+     <https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models>`_. */
+    model: string;
+    /** A set of source input(s) for image segmentation. */
+    source: SegmentImageSource;
+    /** Configuration for image segmentation. */
+    config?: SegmentImageConfig;
+}
+
+/** The output images response. */
+export declare class SegmentImageResponse {
+    /** List of generated image masks.
+     */
+    generatedMasks?: GeneratedImageMask[];
+}
+
+/** A set of source input(s) for image segmentation. */
+export declare interface SegmentImageSource {
+    /** A text prompt for guiding the model during image segmentation.
+     Required for prompt mode and semantic mode, disallowed for other modes. */
+    prompt?: string;
+    /** The image to be segmented. */
+    image?: Image_2;
+    /** The brush scribble to guide segmentation.
+     Required for the interactive mode, disallowed for other modes. */
+    scribbleImage?: ScribbleImage;
+}
+
+/** Enum that represents the segmentation mode. */
+export declare enum SegmentMode {
+    FOREGROUND = "FOREGROUND",
+    BACKGROUND = "BACKGROUND",
+    PROMPT = "PROMPT",
+    SEMANTIC = "SEMANTIC",
+    INTERACTIVE = "INTERACTIVE"
+}
+
 /** Parameters for sending a message within a chat session.
 
  These parameters are used with the `chat.sendMessage()` method.
@@ -6504,7 +6672,7 @@ export declare class Tokens extends BaseModule {
 
 /** Tokens info with a list of tokens and the corresponding list of token ids. */
 export declare interface TokensInfo {
-    /** Optional. Optional fields for the role from the corresponding Content. */
+    /** Optional fields for the role from the corresponding Content. */
     role?: string;
     /** A list of token ids from the input. */
     tokenIds?: string[];
@@ -6771,6 +6939,18 @@ declare class Tunings extends BaseModule {
     tune: (params: types.CreateTuningJobParameters) => Promise<types.TuningJob>;
     private getInternal;
     private listInternal;
+    /**
+     * Cancels a tuning job.
+     *
+     * @param params - The parameters for the cancel request.
+     * @return The empty response returned by the API.
+     *
+     * @example
+     * ```ts
+     * await ai.tunings.cancel({name: '...'}); // The server-generated resource name.
+     * ```
+     */
+    cancel(params: types.CancelTuningJobParameters): Promise<void>;
     private tuneInternal;
     private tuneMldevInternal;
 }
@@ -6877,6 +7057,8 @@ declare namespace types {
         ControlReferenceType,
         SubjectReferenceType,
         EditMode,
+        SegmentMode,
+        VideoGenerationReferenceType,
         VideoCompressionQuality,
         FileState,
         FileSource,
@@ -7007,6 +7189,13 @@ declare namespace types {
         RecontextImageConfig,
         RecontextImageParameters,
         RecontextImageResponse,
+        ScribbleImage,
+        SegmentImageSource,
+        SegmentImageConfig,
+        SegmentImageParameters,
+        EntityLabel,
+        GeneratedImageMask,
+        SegmentImageResponse,
         GetModelConfig,
         GetModelParameters,
         Endpoint,
@@ -7031,10 +7220,14 @@ declare namespace types {
         TokensInfo,
         ComputeTokensResponse,
         Video,
+        GenerateVideosSource,
+        VideoGenerationReferenceImage,
         GenerateVideosConfig,
         GenerateVideosParameters,
         GeneratedVideo,
         GenerateVideosResponse,
+        Operation,
+        GenerateVideosOperation,
         GetTuningJobConfig,
         GetTuningJobParameters,
         TunedModelCheckpoint,
@@ -7060,6 +7253,8 @@ declare namespace types {
         ListTuningJobsConfig,
         ListTuningJobsParameters,
         ListTuningJobsResponse,
+        CancelTuningJobConfig,
+        CancelTuningJobParameters,
         TuningExample,
         TuningDataset,
         TuningValidationDataset,
@@ -7140,10 +7335,7 @@ declare namespace types {
         LiveServerGoAway,
         LiveServerSessionResumptionUpdate,
         LiveServerMessage,
-        OperationGetParameters,
         OperationFromAPIResponseParameters,
-        Operation,
-        GenerateVideosOperation,
         AutomaticActivityDetection,
         RealtimeInputConfig,
         SessionResumptionConfig,
@@ -7187,6 +7379,7 @@ declare namespace types {
         LiveConnectConstraints,
         CreateAuthTokenConfig,
         CreateAuthTokenParameters,
+        OperationGetParameters,
         CreateTuningJobParameters,
         BlobImageUnion,
         PartUnion,
@@ -7317,6 +7510,8 @@ export declare interface UpscaleImageConfig {
      be charged usage for any applicable operations.
      */
     abortSignal?: AbortSignal;
+    /** Cloud Storage URI used to store the generated images. */
+    outputGcsUri?: string;
     /** Whether to include a reason for filtered-out images in the
      response. */
     includeRaiReason?: boolean;
@@ -7495,6 +7690,31 @@ export declare enum VideoCompressionQuality {
     LOSSLESS = "LOSSLESS"
 }
 
+/** A reference image for video generation. */
+export declare interface VideoGenerationReferenceImage {
+    /** The reference image.
+     */
+    image?: Image_2;
+    /** The type of the reference image, which defines how the reference
+     image will be used to generate the video. */
+    referenceType?: VideoGenerationReferenceType;
+}
+
+/** Enum for the reference type of a video generation reference image. */
+export declare enum VideoGenerationReferenceType {
+    /**
+     * A reference image that provides assets to the generated video,
+     such as the scene, an object, a character, etc.
+     */
+    ASSET = "ASSET",
+    /**
+     * A reference image that provides aesthetics including colors,
+     lighting, texture, etc., to be used as the style of the generated video,
+     such as 'anime', 'photography', 'origami', etc.
+     */
+    STYLE = "STYLE"
+}
+
 /** Describes how the video in the Part should be used by the model. */
 export declare interface VideoMetadata {
     /** The frame rate of the video sent to the model. If not specified, the