@google/genai 1.13.0 → 1.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -0
- package/dist/genai.d.ts +308 -47
- package/dist/index.cjs +507 -84
- package/dist/index.mjs +507 -85
- package/dist/index.mjs.map +1 -1
- package/dist/node/index.cjs +507 -84
- package/dist/node/index.mjs +507 -85
- package/dist/node/index.mjs.map +1 -1
- package/dist/node/node.d.ts +308 -47
- package/dist/web/index.mjs +507 -85
- package/dist/web/index.mjs.map +1 -1
- package/dist/web/web.d.ts +308 -47
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -339,6 +339,30 @@ if you are specifying those, you need to explicitly provide the full
|
|
|
339
339
|
`Content[]` structure making it explicit which Parts are 'spoken' by the model,
|
|
340
340
|
or the user. The SDK will throw an exception if you try this.
|
|
341
341
|
|
|
342
|
+
## Error Handling
|
|
343
|
+
|
|
344
|
+
To handle errors raised by the API, the SDK provides this [ApiError](https://github.com/googleapis/js-genai/blob/main/src/errors.ts) class.
|
|
345
|
+
|
|
346
|
+
```typescript
|
|
347
|
+
import {GoogleGenAI} from '@google/genai';
|
|
348
|
+
const GEMINI_API_KEY = process.env.GEMINI_API_KEY;
|
|
349
|
+
|
|
350
|
+
const ai = new GoogleGenAI({apiKey: GEMINI_API_KEY});
|
|
351
|
+
|
|
352
|
+
async function main() {
|
|
353
|
+
await ai.models.generateContent({
|
|
354
|
+
model: 'non-existent-model',
|
|
355
|
+
contents: 'Write a 100-word poem.',
|
|
356
|
+
}).catch((e) => {
|
|
357
|
+
console.error('error name: ', e.name);
|
|
358
|
+
console.error('error message: ', e.message);
|
|
359
|
+
console.error('error status: ', e.status);
|
|
360
|
+
});
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
main();
|
|
364
|
+
```
|
|
365
|
+
|
|
342
366
|
## How is this different from the other Google AI SDKs
|
|
343
367
|
This SDK (`@google/genai`) is Google Deepmind’s "vanilla" SDK for its generative
|
|
344
368
|
AI offerings, and is where Google Deepmind adds new AI features.
|
package/dist/genai.d.ts
CHANGED
|
@@ -1425,6 +1425,8 @@ export declare interface CreateTuningJobConfig {
|
|
|
1425
1425
|
learningRateMultiplier?: number;
|
|
1426
1426
|
/** If set to true, disable intermediate checkpoints for SFT and only the last checkpoint will be exported. Otherwise, enable intermediate checkpoints for SFT. */
|
|
1427
1427
|
exportLastCheckpointOnly?: boolean;
|
|
1428
|
+
/** The optional checkpoint id of the pre-tuned model to use for tuning, if applicable. */
|
|
1429
|
+
preTunedModelCheckpointId?: string;
|
|
1428
1430
|
/** Adapter size for tuning. */
|
|
1429
1431
|
adapterSize?: AdapterSize;
|
|
1430
1432
|
/** The batch size hyperparameter for tuning. If not set, a default of 4 or 16 will be used based on the number of training examples. */
|
|
@@ -1435,7 +1437,7 @@ export declare interface CreateTuningJobConfig {
|
|
|
1435
1437
|
|
|
1436
1438
|
/** Supervised fine-tuning job creation parameters - optional fields. */
|
|
1437
1439
|
export declare interface CreateTuningJobParameters {
|
|
1438
|
-
/** The base model that is being tuned, e.g., "gemini-
|
|
1440
|
+
/** The base model that is being tuned, e.g., "gemini-2.5-flash". */
|
|
1439
1441
|
baseModel: string;
|
|
1440
1442
|
/** Cloud Storage path to file containing training dataset for tuning. The dataset must be formatted as a JSONL file. */
|
|
1441
1443
|
trainingDataset: TuningDataset;
|
|
@@ -1443,6 +1445,18 @@ export declare interface CreateTuningJobParameters {
|
|
|
1443
1445
|
config?: CreateTuningJobConfig;
|
|
1444
1446
|
}
|
|
1445
1447
|
|
|
1448
|
+
/** Supervised fine-tuning job creation parameters - optional fields. */
|
|
1449
|
+
export declare interface CreateTuningJobParametersPrivate {
|
|
1450
|
+
/** The base model that is being tuned, e.g., "gemini-2.5-flash". */
|
|
1451
|
+
baseModel?: string;
|
|
1452
|
+
/** The PreTunedModel that is being tuned. */
|
|
1453
|
+
preTunedModel?: PreTunedModel;
|
|
1454
|
+
/** Cloud Storage path to file containing training dataset for tuning. The dataset must be formatted as a JSONL file. */
|
|
1455
|
+
trainingDataset: TuningDataset;
|
|
1456
|
+
/** Configuration for the tuning job. */
|
|
1457
|
+
config?: CreateTuningJobConfig;
|
|
1458
|
+
}
|
|
1459
|
+
|
|
1446
1460
|
/**
|
|
1447
1461
|
* Creates a `Content` object with a user role from a `PartListUnion` object or `string`.
|
|
1448
1462
|
*/
|
|
@@ -1612,34 +1626,6 @@ export declare interface DistillationDataStats {
|
|
|
1612
1626
|
trainingDatasetStats?: DatasetStats;
|
|
1613
1627
|
}
|
|
1614
1628
|
|
|
1615
|
-
/** Hyperparameters for Distillation. */
|
|
1616
|
-
export declare interface DistillationHyperParameters {
|
|
1617
|
-
/** Optional. Adapter size for distillation. */
|
|
1618
|
-
adapterSize?: AdapterSize;
|
|
1619
|
-
/** Optional. Number of complete passes the model makes over the entire training dataset during training. */
|
|
1620
|
-
epochCount?: string;
|
|
1621
|
-
/** Optional. Multiplier for adjusting the default learning rate. */
|
|
1622
|
-
learningRateMultiplier?: number;
|
|
1623
|
-
}
|
|
1624
|
-
|
|
1625
|
-
/** Tuning Spec for Distillation. */
|
|
1626
|
-
export declare interface DistillationSpec {
|
|
1627
|
-
/** The base teacher model that is being distilled. See [Supported models](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/tuning#supported_models). */
|
|
1628
|
-
baseTeacherModel?: string;
|
|
1629
|
-
/** Optional. Hyperparameters for Distillation. */
|
|
1630
|
-
hyperParameters?: DistillationHyperParameters;
|
|
1631
|
-
/** Deprecated. A path in a Cloud Storage bucket, which will be treated as the root output directory of the distillation pipeline. It is used by the system to generate the paths of output artifacts. */
|
|
1632
|
-
pipelineRootDirectory?: string;
|
|
1633
|
-
/** The student model that is being tuned, e.g., "google/gemma-2b-1.1-it". Deprecated. Use base_model instead. */
|
|
1634
|
-
studentModel?: string;
|
|
1635
|
-
/** Deprecated. Cloud Storage path to file containing training dataset for tuning. The dataset must be formatted as a JSONL file. */
|
|
1636
|
-
trainingDatasetUri?: string;
|
|
1637
|
-
/** The resource name of the Tuned teacher model. Format: `projects/{project}/locations/{location}/models/{model}`. */
|
|
1638
|
-
tunedTeacherModelSource?: string;
|
|
1639
|
-
/** Optional. Cloud Storage path to file containing validation dataset for tuning. The dataset must be formatted as a JSONL file. */
|
|
1640
|
-
validationDatasetUri?: string;
|
|
1641
|
-
}
|
|
1642
|
-
|
|
1643
1629
|
export declare type DownloadableFileUnion = string | File_2 | GeneratedVideo | Video;
|
|
1644
1630
|
|
|
1645
1631
|
declare interface Downloader {
|
|
@@ -1895,9 +1881,19 @@ export declare enum EndSensitivity {
|
|
|
1895
1881
|
|
|
1896
1882
|
/** Tool to search public web data, powered by Vertex AI Search and Sec4 compliance. */
|
|
1897
1883
|
export declare interface EnterpriseWebSearch {
|
|
1884
|
+
/** Optional. List of domains to be excluded from the search results. The default limit is 2000 domains. */
|
|
1885
|
+
excludeDomains?: string[];
|
|
1886
|
+
}
|
|
1887
|
+
|
|
1888
|
+
/** An entity representing the segmented area. */
|
|
1889
|
+
export declare interface EntityLabel {
|
|
1890
|
+
/** The label of the segmented entity. */
|
|
1891
|
+
label?: string;
|
|
1892
|
+
/** The confidence score of the detected label. */
|
|
1893
|
+
score?: number;
|
|
1898
1894
|
}
|
|
1899
1895
|
|
|
1900
|
-
/**
|
|
1896
|
+
/** The environment being operated. */
|
|
1901
1897
|
export declare enum Environment {
|
|
1902
1898
|
/**
|
|
1903
1899
|
* Defaults to browser.
|
|
@@ -2331,6 +2327,22 @@ export declare enum FunctionResponseScheduling {
|
|
|
2331
2327
|
INTERRUPT = "INTERRUPT"
|
|
2332
2328
|
}
|
|
2333
2329
|
|
|
2330
|
+
/** Input example for preference optimization. */
|
|
2331
|
+
export declare interface GeminiPreferenceExample {
|
|
2332
|
+
/** List of completions for a given prompt. */
|
|
2333
|
+
completions?: GeminiPreferenceExampleCompletion[];
|
|
2334
|
+
/** Multi-turn contents that represents the Prompt. */
|
|
2335
|
+
contents?: Content[];
|
|
2336
|
+
}
|
|
2337
|
+
|
|
2338
|
+
/** Completion and its preference score. */
|
|
2339
|
+
export declare interface GeminiPreferenceExampleCompletion {
|
|
2340
|
+
/** Single turn completion for the given prompt. */
|
|
2341
|
+
completion?: Content;
|
|
2342
|
+
/** The score for the given completion. */
|
|
2343
|
+
score?: number;
|
|
2344
|
+
}
|
|
2345
|
+
|
|
2334
2346
|
/** Optional model configuration parameters.
|
|
2335
2347
|
|
|
2336
2348
|
For more information, see `Content generation parameters
|
|
@@ -2698,6 +2710,14 @@ export declare interface GeneratedImage {
|
|
|
2698
2710
|
enhancedPrompt?: string;
|
|
2699
2711
|
}
|
|
2700
2712
|
|
|
2713
|
+
/** A generated image mask. */
|
|
2714
|
+
export declare interface GeneratedImageMask {
|
|
2715
|
+
/** The generated image mask. */
|
|
2716
|
+
mask?: Image_2;
|
|
2717
|
+
/** The detected entities on the segmented area. */
|
|
2718
|
+
labels?: EntityLabel[];
|
|
2719
|
+
}
|
|
2720
|
+
|
|
2701
2721
|
/** A generated video. */
|
|
2702
2722
|
export declare interface GeneratedVideo {
|
|
2703
2723
|
/** The output video */
|
|
@@ -2836,6 +2856,12 @@ export declare interface GenerateVideosConfig {
|
|
|
2836
2856
|
generateAudio?: boolean;
|
|
2837
2857
|
/** Image to use as the last frame of generated videos. Only supported for image to video use cases. */
|
|
2838
2858
|
lastFrame?: Image_2;
|
|
2859
|
+
/** The images to use as the references to generate the videos.
|
|
2860
|
+
If this field is provided, the text prompt field must also be provided.
|
|
2861
|
+
The image, video, or last_frame field are not supported. Each image must
|
|
2862
|
+
be associated with a type. Veo 2 supports up to 3 asset images *or* 1
|
|
2863
|
+
style image. */
|
|
2864
|
+
referenceImages?: VideoGenerationReferenceImage[];
|
|
2839
2865
|
/** Compression quality of the generated videos. */
|
|
2840
2866
|
compressionQuality?: VideoCompressionQuality;
|
|
2841
2867
|
}
|
|
@@ -2960,7 +2986,7 @@ export declare interface GenerationConfigRoutingConfigManualRoutingMode {
|
|
|
2960
2986
|
export declare interface GenerationConfigThinkingConfig {
|
|
2961
2987
|
/** Optional. Indicates whether to include thoughts in the response. If true, thoughts are returned only when available. */
|
|
2962
2988
|
includeThoughts?: boolean;
|
|
2963
|
-
/** Optional. Indicates the thinking budget in tokens.
|
|
2989
|
+
/** Optional. Indicates the thinking budget in tokens. */
|
|
2964
2990
|
thinkingBudget?: number;
|
|
2965
2991
|
}
|
|
2966
2992
|
|
|
@@ -3224,6 +3250,9 @@ export declare interface GoogleSearch {
|
|
|
3224
3250
|
If customers set a start time, they must set an end time (and vice versa).
|
|
3225
3251
|
*/
|
|
3226
3252
|
timeRangeFilter?: Interval;
|
|
3253
|
+
/** Optional. List of domains to be excluded from the search results.
|
|
3254
|
+
The default limit is 2000 domains. */
|
|
3255
|
+
excludeDomains?: string[];
|
|
3227
3256
|
}
|
|
3228
3257
|
|
|
3229
3258
|
/** Tool to retrieve public web data for grounding, powered by Google. */
|
|
@@ -3244,14 +3273,64 @@ export declare interface GoogleTypeDate {
|
|
|
3244
3273
|
|
|
3245
3274
|
/** Grounding chunk. */
|
|
3246
3275
|
export declare interface GroundingChunk {
|
|
3276
|
+
/** Grounding chunk from Google Maps. */
|
|
3277
|
+
maps?: GroundingChunkMaps;
|
|
3247
3278
|
/** Grounding chunk from context retrieved by the retrieval tools. */
|
|
3248
3279
|
retrievedContext?: GroundingChunkRetrievedContext;
|
|
3249
3280
|
/** Grounding chunk from the web. */
|
|
3250
3281
|
web?: GroundingChunkWeb;
|
|
3251
3282
|
}
|
|
3252
3283
|
|
|
3284
|
+
/** Chunk from Google Maps. */
|
|
3285
|
+
export declare interface GroundingChunkMaps {
|
|
3286
|
+
/** Sources used to generate the place answer. This includes review snippets and photos that were used to generate the answer, as well as uris to flag content. */
|
|
3287
|
+
placeAnswerSources?: GroundingChunkMapsPlaceAnswerSources;
|
|
3288
|
+
/** This Place's resource name, in `places/{place_id}` format. Can be used to look up the Place. */
|
|
3289
|
+
placeId?: string;
|
|
3290
|
+
/** Text of the chunk. */
|
|
3291
|
+
text?: string;
|
|
3292
|
+
/** Title of the chunk. */
|
|
3293
|
+
title?: string;
|
|
3294
|
+
/** URI reference of the chunk. */
|
|
3295
|
+
uri?: string;
|
|
3296
|
+
}
|
|
3297
|
+
|
|
3298
|
+
/** Sources used to generate the place answer. */
|
|
3299
|
+
export declare interface GroundingChunkMapsPlaceAnswerSources {
|
|
3300
|
+
/** A link where users can flag a problem with the generated answer. */
|
|
3301
|
+
flagContentUri?: string;
|
|
3302
|
+
/** Snippets of reviews that are used to generate the answer. */
|
|
3303
|
+
reviewSnippets?: GroundingChunkMapsPlaceAnswerSourcesReviewSnippet[];
|
|
3304
|
+
}
|
|
3305
|
+
|
|
3306
|
+
/** Author attribution for a photo or review. */
|
|
3307
|
+
export declare interface GroundingChunkMapsPlaceAnswerSourcesAuthorAttribution {
|
|
3308
|
+
/** Name of the author of the Photo or Review. */
|
|
3309
|
+
displayName?: string;
|
|
3310
|
+
/** Profile photo URI of the author of the Photo or Review. */
|
|
3311
|
+
photoUri?: string;
|
|
3312
|
+
/** URI of the author of the Photo or Review. */
|
|
3313
|
+
uri?: string;
|
|
3314
|
+
}
|
|
3315
|
+
|
|
3316
|
+
/** Encapsulates a review snippet. */
|
|
3317
|
+
export declare interface GroundingChunkMapsPlaceAnswerSourcesReviewSnippet {
|
|
3318
|
+
/** This review's author. */
|
|
3319
|
+
authorAttribution?: GroundingChunkMapsPlaceAnswerSourcesAuthorAttribution;
|
|
3320
|
+
/** A link where users can flag a problem with the review. */
|
|
3321
|
+
flagContentUri?: string;
|
|
3322
|
+
/** A link to show the review on Google Maps. */
|
|
3323
|
+
googleMapsUri?: string;
|
|
3324
|
+
/** A string of formatted recent time, expressing the review time relative to the current time in a form appropriate for the language and country. */
|
|
3325
|
+
relativePublishTimeDescription?: string;
|
|
3326
|
+
/** A reference representing this place review which may be used to look up this place review again. */
|
|
3327
|
+
review?: string;
|
|
3328
|
+
}
|
|
3329
|
+
|
|
3253
3330
|
/** Chunk from context retrieved by the retrieval tools. */
|
|
3254
3331
|
export declare interface GroundingChunkRetrievedContext {
|
|
3332
|
+
/** Output only. The full document name for the referenced Vertex AI Search document. */
|
|
3333
|
+
documentName?: string;
|
|
3255
3334
|
/** Additional context for the RAG retrieval result. This is only populated when using the RAG retrieval tool. */
|
|
3256
3335
|
ragChunk?: RagChunk;
|
|
3257
3336
|
/** Text of the attribution. */
|
|
@@ -3274,6 +3353,8 @@ export declare interface GroundingChunkWeb {
|
|
|
3274
3353
|
|
|
3275
3354
|
/** Metadata returned to client when grounding is enabled. */
|
|
3276
3355
|
export declare interface GroundingMetadata {
|
|
3356
|
+
/** Optional. Output only. Resource name of the Google Maps widget context token to be used with the PlacesContextElement widget to render contextual data. This is populated only for Google Maps grounding. */
|
|
3357
|
+
googleMapsWidgetContextToken?: string;
|
|
3277
3358
|
/** List of supporting references retrieved from specified grounding source. */
|
|
3278
3359
|
groundingChunks?: GroundingChunk[];
|
|
3279
3360
|
/** Optional. List of grounding support. */
|
|
@@ -5068,6 +5149,27 @@ export declare class Models extends BaseModule {
|
|
|
5068
5149
|
* ```
|
|
5069
5150
|
*/
|
|
5070
5151
|
recontextImage(params: types.RecontextImageParameters): Promise<types.RecontextImageResponse>;
|
|
5152
|
+
/**
|
|
5153
|
+
* Segments an image, creating a mask of a specified area.
|
|
5154
|
+
*
|
|
5155
|
+
* @param params - The parameters for segmenting an image.
|
|
5156
|
+
* @return The response from the API.
|
|
5157
|
+
*
|
|
5158
|
+
* @example
|
|
5159
|
+
* ```ts
|
|
5160
|
+
* const response = await ai.models.segmentImage({
|
|
5161
|
+
* model: 'image-segmentation-001',
|
|
5162
|
+
* source: {
|
|
5163
|
+
* image: image,
|
|
5164
|
+
* },
|
|
5165
|
+
* config: {
|
|
5166
|
+
* mode: 'foreground',
|
|
5167
|
+
* },
|
|
5168
|
+
* });
|
|
5169
|
+
* console.log(response?.generatedMasks?.[0]?.mask?.imageBytes);
|
|
5170
|
+
* ```
|
|
5171
|
+
*/
|
|
5172
|
+
segmentImage(params: types.SegmentImageParameters): Promise<types.SegmentImageResponse>;
|
|
5071
5173
|
/**
|
|
5072
5174
|
* Fetches information about a model by name.
|
|
5073
5175
|
*
|
|
@@ -5472,6 +5574,36 @@ export declare interface PrebuiltVoiceConfig {
|
|
|
5472
5574
|
voiceName?: string;
|
|
5473
5575
|
}
|
|
5474
5576
|
|
|
5577
|
+
/** Statistics computed for datasets used for preference optimization. */
|
|
5578
|
+
export declare interface PreferenceOptimizationDataStats {
|
|
5579
|
+
/** Output only. Dataset distributions for scores variance per example. */
|
|
5580
|
+
scoreVariancePerExampleDistribution?: DatasetDistribution;
|
|
5581
|
+
/** Output only. Dataset distributions for scores. */
|
|
5582
|
+
scoresDistribution?: DatasetDistribution;
|
|
5583
|
+
/** Output only. Number of billable tokens in the tuning dataset. */
|
|
5584
|
+
totalBillableTokenCount?: string;
|
|
5585
|
+
/** Output only. Number of examples in the tuning dataset. */
|
|
5586
|
+
tuningDatasetExampleCount?: string;
|
|
5587
|
+
/** Output only. Number of tuning steps for this Tuning Job. */
|
|
5588
|
+
tuningStepCount?: string;
|
|
5589
|
+
/** Output only. Sample user examples in the training dataset. */
|
|
5590
|
+
userDatasetExamples?: GeminiPreferenceExample[];
|
|
5591
|
+
/** Output only. Dataset distributions for the user input tokens. */
|
|
5592
|
+
userInputTokenDistribution?: DatasetDistribution;
|
|
5593
|
+
/** Output only. Dataset distributions for the user output tokens. */
|
|
5594
|
+
userOutputTokenDistribution?: DatasetDistribution;
|
|
5595
|
+
}
|
|
5596
|
+
|
|
5597
|
+
/** A pre-tuned model for continuous tuning. */
|
|
5598
|
+
export declare interface PreTunedModel {
|
|
5599
|
+
/** Output only. The name of the base model this PreTunedModel was tuned from. */
|
|
5600
|
+
baseModel?: string;
|
|
5601
|
+
/** Optional. The source checkpoint id. If not specified, the default checkpoint will be used. */
|
|
5602
|
+
checkpointId?: string;
|
|
5603
|
+
/** The resource name of the Model. E.g., a model resource name with a specified version id or alias: `projects/{project}/locations/{location}/models/{model}@{version_id}` `projects/{project}/locations/{location}/models/{model}@{alias}` Or, omit the version id to use the default version: `projects/{project}/locations/{location}/models/{model}` */
|
|
5604
|
+
tunedModelName?: string;
|
|
5605
|
+
}
|
|
5606
|
+
|
|
5475
5607
|
/** Config for proactivity features. */
|
|
5476
5608
|
export declare interface ProactivityConfig {
|
|
5477
5609
|
/** If enabled, the model can reject responding to the last prompt. For
|
|
@@ -5880,6 +6012,12 @@ export declare interface Schema {
|
|
|
5880
6012
|
|
|
5881
6013
|
export declare type SchemaUnion = Schema | unknown;
|
|
5882
6014
|
|
|
6015
|
+
/** An image mask representing a brush scribble. */
|
|
6016
|
+
export declare interface ScribbleImage {
|
|
6017
|
+
/** The brush scribble to guide segmentation. Valid for the interactive mode. */
|
|
6018
|
+
image?: Image_2;
|
|
6019
|
+
}
|
|
6020
|
+
|
|
5883
6021
|
/** Google search entry point. */
|
|
5884
6022
|
export declare interface SearchEntryPoint {
|
|
5885
6023
|
/** Optional. Web content snippet that can be embedded in a web page or an app webview. */
|
|
@@ -5901,6 +6039,75 @@ export declare interface Segment {
|
|
|
5901
6039
|
text?: string;
|
|
5902
6040
|
}
|
|
5903
6041
|
|
|
6042
|
+
/** Configuration for segmenting an image. */
|
|
6043
|
+
export declare interface SegmentImageConfig {
|
|
6044
|
+
/** Used to override HTTP request options. */
|
|
6045
|
+
httpOptions?: HttpOptions;
|
|
6046
|
+
/** Abort signal which can be used to cancel the request.
|
|
6047
|
+
|
|
6048
|
+
NOTE: AbortSignal is a client-only operation. Using it to cancel an
|
|
6049
|
+
operation will not cancel the request in the service. You will still
|
|
6050
|
+
be charged usage for any applicable operations.
|
|
6051
|
+
*/
|
|
6052
|
+
abortSignal?: AbortSignal;
|
|
6053
|
+
/** The segmentation mode to use. */
|
|
6054
|
+
mode?: SegmentMode;
|
|
6055
|
+
/** The maximum number of predictions to return up to, by top
|
|
6056
|
+
confidence score. */
|
|
6057
|
+
maxPredictions?: number;
|
|
6058
|
+
/** The confidence score threshold for the detections as a decimal
|
|
6059
|
+
value. Only predictions with a confidence score higher than this
|
|
6060
|
+
threshold will be returned. */
|
|
6061
|
+
confidenceThreshold?: number;
|
|
6062
|
+
/** A decimal value representing how much dilation to apply to the
|
|
6063
|
+
masks. 0 for no dilation. 1.0 means the masked area covers the whole
|
|
6064
|
+
image. */
|
|
6065
|
+
maskDilation?: number;
|
|
6066
|
+
/** The binary color threshold to apply to the masks. The threshold
|
|
6067
|
+
can be set to a decimal value between 0 and 255 non-inclusive.
|
|
6068
|
+
Set to -1 for no binary color thresholding. */
|
|
6069
|
+
binaryColorThreshold?: number;
|
|
6070
|
+
}
|
|
6071
|
+
|
|
6072
|
+
/** The parameters for segmenting an image. */
|
|
6073
|
+
export declare interface SegmentImageParameters {
|
|
6074
|
+
/** ID of the model to use. For a list of models, see `Google models
|
|
6075
|
+
<https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models>`_. */
|
|
6076
|
+
model: string;
|
|
6077
|
+
/** A set of source input(s) for image segmentation. */
|
|
6078
|
+
source: SegmentImageSource;
|
|
6079
|
+
/** Configuration for image segmentation. */
|
|
6080
|
+
config?: SegmentImageConfig;
|
|
6081
|
+
}
|
|
6082
|
+
|
|
6083
|
+
/** The output images response. */
|
|
6084
|
+
export declare class SegmentImageResponse {
|
|
6085
|
+
/** List of generated image masks.
|
|
6086
|
+
*/
|
|
6087
|
+
generatedMasks?: GeneratedImageMask[];
|
|
6088
|
+
}
|
|
6089
|
+
|
|
6090
|
+
/** A set of source input(s) for image segmentation. */
|
|
6091
|
+
export declare interface SegmentImageSource {
|
|
6092
|
+
/** A text prompt for guiding the model during image segmentation.
|
|
6093
|
+
Required for prompt mode and semantic mode, disallowed for other modes. */
|
|
6094
|
+
prompt?: string;
|
|
6095
|
+
/** The image to be segmented. */
|
|
6096
|
+
image?: Image_2;
|
|
6097
|
+
/** The brush scribble to guide segmentation.
|
|
6098
|
+
Required for the interactive mode, disallowed for other modes. */
|
|
6099
|
+
scribbleImage?: ScribbleImage;
|
|
6100
|
+
}
|
|
6101
|
+
|
|
6102
|
+
/** Enum that represents the segmentation mode. */
|
|
6103
|
+
export declare enum SegmentMode {
|
|
6104
|
+
FOREGROUND = "FOREGROUND",
|
|
6105
|
+
BACKGROUND = "BACKGROUND",
|
|
6106
|
+
PROMPT = "PROMPT",
|
|
6107
|
+
SEMANTIC = "SEMANTIC",
|
|
6108
|
+
INTERACTIVE = "INTERACTIVE"
|
|
6109
|
+
}
|
|
6110
|
+
|
|
5904
6111
|
/** Parameters for sending a message within a chat session.
|
|
5905
6112
|
|
|
5906
6113
|
These parameters are used with the `chat.sendMessage()` method.
|
|
@@ -6204,9 +6411,13 @@ export declare enum SubjectReferenceType {
|
|
|
6204
6411
|
export declare interface SupervisedHyperParameters {
|
|
6205
6412
|
/** Optional. Adapter size for tuning. */
|
|
6206
6413
|
adapterSize?: AdapterSize;
|
|
6414
|
+
/** Optional. Batch size for tuning. This feature is only available for open source models. */
|
|
6415
|
+
batchSize?: string;
|
|
6207
6416
|
/** Optional. Number of complete passes the model makes over the entire training dataset during training. */
|
|
6208
6417
|
epochCount?: string;
|
|
6209
|
-
/** Optional.
|
|
6418
|
+
/** Optional. Learning rate for tuning. Mutually exclusive with `learning_rate_multiplier`. This feature is only available for open source models. */
|
|
6419
|
+
learningRate?: number;
|
|
6420
|
+
/** Optional. Multiplier for adjusting the default learning rate. Mutually exclusive with `learning_rate`. This feature is only available for 1P models. */
|
|
6210
6421
|
learningRateMultiplier?: number;
|
|
6211
6422
|
}
|
|
6212
6423
|
|
|
@@ -6278,6 +6489,8 @@ export declare interface SupervisedTuningSpec {
|
|
|
6278
6489
|
hyperParameters?: SupervisedHyperParameters;
|
|
6279
6490
|
/** Required. Training dataset used for tuning. The dataset can be specified as either a Cloud Storage path to a JSONL file or as the resource name of a Vertex Multimodal Dataset. */
|
|
6280
6491
|
trainingDatasetUri?: string;
|
|
6492
|
+
/** Tuning mode. */
|
|
6493
|
+
tuningMode?: TuningMode;
|
|
6281
6494
|
/** Optional. Validation dataset used for tuning. The dataset can be specified as either a Cloud Storage path to a JSONL file or as the resource name of a Vertex Multimodal Dataset. */
|
|
6282
6495
|
validationDatasetUri?: string;
|
|
6283
6496
|
}
|
|
@@ -6437,10 +6650,12 @@ export declare interface Tool {
|
|
|
6437
6650
|
googleMaps?: GoogleMaps;
|
|
6438
6651
|
/** Optional. Tool to support URL context retrieval. */
|
|
6439
6652
|
urlContext?: UrlContext;
|
|
6653
|
+
/** Optional. Tool to support the model interacting directly with the
|
|
6654
|
+
computer. If enabled, it automatically populates computer-use specific
|
|
6655
|
+
Function Declarations. */
|
|
6656
|
+
computerUse?: ToolComputerUse;
|
|
6440
6657
|
/** Optional. CodeExecution tool type. Enables the model to execute code as part of generation. */
|
|
6441
6658
|
codeExecution?: ToolCodeExecution;
|
|
6442
|
-
/** Optional. Tool to support the model interacting directly with the computer. If enabled, it automatically populates computer-use specific Function Declarations. */
|
|
6443
|
-
computerUse?: ToolComputerUse;
|
|
6444
6659
|
}
|
|
6445
6660
|
|
|
6446
6661
|
/** Tool that executes code generated by the model, and automatically returns the result to the model. See also [ExecutableCode]and [CodeExecutionResult] which are input and output to this tool. */
|
|
@@ -6495,7 +6710,7 @@ export declare interface Transcription {
|
|
|
6495
6710
|
}
|
|
6496
6711
|
|
|
6497
6712
|
export declare interface TunedModel {
|
|
6498
|
-
/** Output only. The resource name of the TunedModel. Format: `projects/{project}/locations/{location}/models/{model}
|
|
6713
|
+
/** Output only. The resource name of the TunedModel. Format: `projects/{project}/locations/{location}/models/{model}@{version_id}` When tuning from a base model, the version_id will be 1. For continuous tuning, the version id will be incremented by 1 from the last version id in the parent model. E.g., `projects/{project}/locations/{location}/models/{model}@{last_version_id + 1}` */
|
|
6499
6714
|
model?: string;
|
|
6500
6715
|
/** Output only. A resource name of an Endpoint. Format: `projects/{project}/locations/{location}/endpoints/{endpoint}`. */
|
|
6501
6716
|
endpoint?: string;
|
|
@@ -6546,6 +6761,8 @@ export declare interface TuningDataset {
|
|
|
6546
6761
|
export declare interface TuningDataStats {
|
|
6547
6762
|
/** Output only. Statistics for distillation. */
|
|
6548
6763
|
distillationDataStats?: DistillationDataStats;
|
|
6764
|
+
/** Output only. Statistics for preference optimization. */
|
|
6765
|
+
preferenceOptimizationDataStats?: PreferenceOptimizationDataStats;
|
|
6549
6766
|
/** The SFT Tuning data stats. */
|
|
6550
6767
|
supervisedTuningDataStats?: SupervisedTuningDataStats;
|
|
6551
6768
|
}
|
|
@@ -6581,6 +6798,8 @@ export declare interface TuningJob {
|
|
|
6581
6798
|
baseModel?: string;
|
|
6582
6799
|
/** Output only. The tuned model resources associated with this TuningJob. */
|
|
6583
6800
|
tunedModel?: TunedModel;
|
|
6801
|
+
/** The pre-tuned model for continuous tuning. */
|
|
6802
|
+
preTunedModel?: PreTunedModel;
|
|
6584
6803
|
/** Tuning Spec for Supervised Fine Tuning. */
|
|
6585
6804
|
supervisedTuningSpec?: SupervisedTuningSpec;
|
|
6586
6805
|
/** Output only. The tuning data statistics associated with this TuningJob. */
|
|
@@ -6589,24 +6808,38 @@ export declare interface TuningJob {
|
|
|
6589
6808
|
encryptionSpec?: EncryptionSpec;
|
|
6590
6809
|
/** Tuning Spec for open sourced and third party Partner models. */
|
|
6591
6810
|
partnerModelTuningSpec?: PartnerModelTuningSpec;
|
|
6592
|
-
/**
|
|
6593
|
-
|
|
6811
|
+
/** Optional. The user-provided path to custom model weights. Set this field to tune a custom model. The path must be a Cloud Storage directory that contains the model weights in .safetensors format along with associated model metadata files. If this field is set, the base_model field must still be set to indicate which base model the custom model is derived from. This feature is only available for open source models. */
|
|
6812
|
+
customBaseModel?: string;
|
|
6594
6813
|
/** Output only. The Experiment associated with this TuningJob. */
|
|
6595
6814
|
experiment?: string;
|
|
6596
6815
|
/** Optional. The labels with user-defined metadata to organize TuningJob and generated resources such as Model and Endpoint. Label keys and values can be no longer than 64 characters (Unicode codepoints), can only contain lowercase letters, numeric characters, underscores and dashes. International characters are allowed. See https://goo.gl/xmQnxf for more information and examples of labels. */
|
|
6597
6816
|
labels?: Record<string, string>;
|
|
6817
|
+
/** Optional. Cloud Storage path to the directory where tuning job outputs are written to. This field is only available and required for open source models. */
|
|
6818
|
+
outputUri?: string;
|
|
6598
6819
|
/** Output only. The resource name of the PipelineJob associated with the TuningJob. Format: `projects/{project}/locations/{location}/pipelineJobs/{pipeline_job}`. */
|
|
6599
6820
|
pipelineJob?: string;
|
|
6600
|
-
/** Output only. Reserved for future use. */
|
|
6601
|
-
satisfiesPzi?: boolean;
|
|
6602
|
-
/** Output only. Reserved for future use. */
|
|
6603
|
-
satisfiesPzs?: boolean;
|
|
6604
6821
|
/** The service account that the tuningJob workload runs as. If not specified, the Vertex AI Secure Fine-Tuned Service Agent in the project will be used. See https://cloud.google.com/iam/docs/service-agents#vertex-ai-secure-fine-tuning-service-agent Users starting the pipeline must have the `iam.serviceAccounts.actAs` permission on this service account. */
|
|
6605
6822
|
serviceAccount?: string;
|
|
6606
6823
|
/** Optional. The display name of the TunedModel. The name can be up to 128 characters long and can consist of any UTF-8 characters. */
|
|
6607
6824
|
tunedModelDisplayName?: string;
|
|
6608
6825
|
}
|
|
6609
6826
|
|
|
6827
|
+
/** Tuning mode. */
|
|
6828
|
+
export declare enum TuningMode {
|
|
6829
|
+
/**
|
|
6830
|
+
* Tuning mode is unspecified.
|
|
6831
|
+
*/
|
|
6832
|
+
TUNING_MODE_UNSPECIFIED = "TUNING_MODE_UNSPECIFIED",
|
|
6833
|
+
/**
|
|
6834
|
+
* Full fine-tuning mode.
|
|
6835
|
+
*/
|
|
6836
|
+
TUNING_MODE_FULL = "TUNING_MODE_FULL",
|
|
6837
|
+
/**
|
|
6838
|
+
* PEFT adapter tuning mode.
|
|
6839
|
+
*/
|
|
6840
|
+
TUNING_MODE_PEFT_ADAPTER = "TUNING_MODE_PEFT_ADAPTER"
|
|
6841
|
+
}
|
|
6842
|
+
|
|
6610
6843
|
/** A long-running operation. */
|
|
6611
6844
|
export declare interface TuningOperation {
|
|
6612
6845
|
/** Used to retain the full HTTP response. */
|
|
@@ -6739,7 +6972,6 @@ declare namespace types {
|
|
|
6739
6972
|
Mode,
|
|
6740
6973
|
AuthType,
|
|
6741
6974
|
ApiSpec,
|
|
6742
|
-
Environment,
|
|
6743
6975
|
UrlRetrievalStatus,
|
|
6744
6976
|
FinishReason,
|
|
6745
6977
|
HarmProbability,
|
|
@@ -6749,10 +6981,12 @@ declare namespace types {
|
|
|
6749
6981
|
Modality,
|
|
6750
6982
|
MediaResolution,
|
|
6751
6983
|
JobState,
|
|
6984
|
+
TuningMode,
|
|
6752
6985
|
AdapterSize,
|
|
6753
6986
|
FeatureSelectionPreference,
|
|
6754
6987
|
Behavior,
|
|
6755
6988
|
DynamicRetrievalConfigMode,
|
|
6989
|
+
Environment,
|
|
6756
6990
|
FunctionCallingConfigMode,
|
|
6757
6991
|
SafetyFilterLevel,
|
|
6758
6992
|
PersonGeneration,
|
|
@@ -6761,6 +6995,7 @@ declare namespace types {
|
|
|
6761
6995
|
ControlReferenceType,
|
|
6762
6996
|
SubjectReferenceType,
|
|
6763
6997
|
EditMode,
|
|
6998
|
+
SegmentMode,
|
|
6764
6999
|
VideoCompressionQuality,
|
|
6765
7000
|
FileState,
|
|
6766
7001
|
FileSource,
|
|
@@ -6800,6 +7035,7 @@ declare namespace types {
|
|
|
6800
7035
|
AuthConfig,
|
|
6801
7036
|
GoogleMaps,
|
|
6802
7037
|
UrlContext,
|
|
7038
|
+
ToolComputerUse,
|
|
6803
7039
|
ApiAuthApiKeyConfig,
|
|
6804
7040
|
ApiAuth,
|
|
6805
7041
|
ExternalApiElasticSearchParams,
|
|
@@ -6817,7 +7053,6 @@ declare namespace types {
|
|
|
6817
7053
|
VertexRagStore,
|
|
6818
7054
|
Retrieval,
|
|
6819
7055
|
ToolCodeExecution,
|
|
6820
|
-
ToolComputerUse,
|
|
6821
7056
|
Tool,
|
|
6822
7057
|
FunctionCallingConfig,
|
|
6823
7058
|
LatLng,
|
|
@@ -6842,6 +7077,10 @@ declare namespace types {
|
|
|
6842
7077
|
CitationMetadata,
|
|
6843
7078
|
UrlMetadata,
|
|
6844
7079
|
UrlContextMetadata,
|
|
7080
|
+
GroundingChunkMapsPlaceAnswerSourcesAuthorAttribution,
|
|
7081
|
+
GroundingChunkMapsPlaceAnswerSourcesReviewSnippet,
|
|
7082
|
+
GroundingChunkMapsPlaceAnswerSources,
|
|
7083
|
+
GroundingChunkMaps,
|
|
6845
7084
|
RagChunkPageSpan,
|
|
6846
7085
|
RagChunk,
|
|
6847
7086
|
GroundingChunkRetrievedContext,
|
|
@@ -6887,6 +7126,13 @@ declare namespace types {
|
|
|
6887
7126
|
RecontextImageConfig,
|
|
6888
7127
|
RecontextImageParameters,
|
|
6889
7128
|
RecontextImageResponse,
|
|
7129
|
+
ScribbleImage,
|
|
7130
|
+
SegmentImageSource,
|
|
7131
|
+
SegmentImageConfig,
|
|
7132
|
+
SegmentImageParameters,
|
|
7133
|
+
EntityLabel,
|
|
7134
|
+
GeneratedImageMask,
|
|
7135
|
+
SegmentImageResponse,
|
|
6890
7136
|
GetModelConfig,
|
|
6891
7137
|
GetModelParameters,
|
|
6892
7138
|
Endpoint,
|
|
@@ -6911,6 +7157,7 @@ declare namespace types {
|
|
|
6911
7157
|
TokensInfo,
|
|
6912
7158
|
ComputeTokensResponse,
|
|
6913
7159
|
Video,
|
|
7160
|
+
VideoGenerationReferenceImage,
|
|
6914
7161
|
GenerateVideosConfig,
|
|
6915
7162
|
GenerateVideosParameters,
|
|
6916
7163
|
GeneratedVideo,
|
|
@@ -6920,20 +7167,22 @@ declare namespace types {
|
|
|
6920
7167
|
TunedModelCheckpoint,
|
|
6921
7168
|
TunedModel,
|
|
6922
7169
|
GoogleRpcStatus,
|
|
7170
|
+
PreTunedModel,
|
|
6923
7171
|
SupervisedHyperParameters,
|
|
6924
7172
|
SupervisedTuningSpec,
|
|
6925
7173
|
DatasetDistributionDistributionBucket,
|
|
6926
7174
|
DatasetDistribution,
|
|
6927
7175
|
DatasetStats,
|
|
6928
7176
|
DistillationDataStats,
|
|
7177
|
+
GeminiPreferenceExampleCompletion,
|
|
7178
|
+
GeminiPreferenceExample,
|
|
7179
|
+
PreferenceOptimizationDataStats,
|
|
6929
7180
|
SupervisedTuningDatasetDistributionDatasetBucket,
|
|
6930
7181
|
SupervisedTuningDatasetDistribution,
|
|
6931
7182
|
SupervisedTuningDataStats,
|
|
6932
7183
|
TuningDataStats,
|
|
6933
7184
|
EncryptionSpec,
|
|
6934
7185
|
PartnerModelTuningSpec,
|
|
6935
|
-
DistillationHyperParameters,
|
|
6936
|
-
DistillationSpec,
|
|
6937
7186
|
TuningJob,
|
|
6938
7187
|
ListTuningJobsConfig,
|
|
6939
7188
|
ListTuningJobsParameters,
|
|
@@ -6942,7 +7191,7 @@ declare namespace types {
|
|
|
6942
7191
|
TuningDataset,
|
|
6943
7192
|
TuningValidationDataset,
|
|
6944
7193
|
CreateTuningJobConfig,
|
|
6945
|
-
|
|
7194
|
+
CreateTuningJobParametersPrivate,
|
|
6946
7195
|
TuningOperation,
|
|
6947
7196
|
CreateCachedContentConfig,
|
|
6948
7197
|
CreateCachedContentParameters,
|
|
@@ -7034,8 +7283,8 @@ declare namespace types {
|
|
|
7034
7283
|
ActivityStart,
|
|
7035
7284
|
ActivityEnd,
|
|
7036
7285
|
LiveClientRealtimeInput,
|
|
7037
|
-
LiveSendRealtimeInputParameters,
|
|
7038
7286
|
LiveClientToolResponse,
|
|
7287
|
+
LiveSendRealtimeInputParameters,
|
|
7039
7288
|
LiveClientMessage,
|
|
7040
7289
|
LiveConnectConfig,
|
|
7041
7290
|
LiveConnectParameters,
|
|
@@ -7065,6 +7314,7 @@ declare namespace types {
|
|
|
7065
7314
|
LiveConnectConstraints,
|
|
7066
7315
|
CreateAuthTokenConfig,
|
|
7067
7316
|
CreateAuthTokenParameters,
|
|
7317
|
+
CreateTuningJobParameters,
|
|
7068
7318
|
BlobImageUnion,
|
|
7069
7319
|
PartUnion,
|
|
7070
7320
|
PartListUnion,
|
|
@@ -7372,6 +7622,17 @@ export declare enum VideoCompressionQuality {
|
|
|
7372
7622
|
LOSSLESS = "LOSSLESS"
|
|
7373
7623
|
}
|
|
7374
7624
|
|
|
7625
|
+
/** A reference image for video generation. */
|
|
7626
|
+
export declare interface VideoGenerationReferenceImage {
|
|
7627
|
+
/** The reference image.
|
|
7628
|
+
*/
|
|
7629
|
+
image?: Image_2;
|
|
7630
|
+
/** The type of the reference image, which defines how the reference
|
|
7631
|
+
image will be used to generate the video. Supported values are 'asset'
|
|
7632
|
+
or 'style'. */
|
|
7633
|
+
referenceType?: string;
|
|
7634
|
+
}
|
|
7635
|
+
|
|
7375
7636
|
/** Describes how the video in the Part should be used by the model. */
|
|
7376
7637
|
export declare interface VideoMetadata {
|
|
7377
7638
|
/** The frame rate of the video sent to the model. If not specified, the
|