@firebase/ai 2.3.0 → 2.4.0-canary.22e0a1adb

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/dist/ai-public.d.ts +240 -35
  2. package/dist/ai.d.ts +241 -35
  3. package/dist/esm/index.esm.js +295 -92
  4. package/dist/esm/index.esm.js.map +1 -1
  5. package/dist/esm/src/api.d.ts +1 -1
  6. package/dist/esm/src/methods/chrome-adapter.d.ts +1 -1
  7. package/dist/esm/src/methods/live-session-helpers.d.ts +2 -2
  8. package/dist/esm/src/methods/live-session.d.ts +71 -7
  9. package/dist/esm/src/models/imagen-model.d.ts +2 -2
  10. package/dist/esm/src/requests/hybrid-helpers.d.ts +7 -2
  11. package/dist/esm/src/requests/imagen-image-format.d.ts +3 -3
  12. package/dist/esm/src/requests/response-helpers.d.ts +2 -2
  13. package/dist/esm/src/requests/stream-reader.d.ts +2 -1
  14. package/dist/esm/src/types/content.d.ts +4 -4
  15. package/dist/esm/src/types/enums.d.ts +19 -4
  16. package/dist/esm/src/types/googleai.d.ts +2 -1
  17. package/dist/esm/src/types/imagen/requests.d.ts +9 -9
  18. package/dist/esm/src/types/imagen/responses.d.ts +3 -3
  19. package/dist/esm/src/types/live-responses.d.ts +16 -2
  20. package/dist/esm/src/types/requests.d.ts +22 -2
  21. package/dist/esm/src/types/responses.d.ts +99 -1
  22. package/dist/index.cjs.js +296 -91
  23. package/dist/index.cjs.js.map +1 -1
  24. package/dist/index.node.cjs.js +276 -84
  25. package/dist/index.node.cjs.js.map +1 -1
  26. package/dist/index.node.mjs +275 -85
  27. package/dist/index.node.mjs.map +1 -1
  28. package/dist/src/api.d.ts +1 -1
  29. package/dist/src/methods/chrome-adapter.d.ts +1 -1
  30. package/dist/src/methods/live-session-helpers.d.ts +2 -2
  31. package/dist/src/methods/live-session.d.ts +71 -7
  32. package/dist/src/models/imagen-model.d.ts +2 -2
  33. package/dist/src/requests/hybrid-helpers.d.ts +7 -2
  34. package/dist/src/requests/imagen-image-format.d.ts +3 -3
  35. package/dist/src/requests/response-helpers.d.ts +2 -2
  36. package/dist/src/requests/stream-reader.d.ts +2 -1
  37. package/dist/src/types/content.d.ts +4 -4
  38. package/dist/src/types/enums.d.ts +19 -4
  39. package/dist/src/types/googleai.d.ts +2 -1
  40. package/dist/src/types/imagen/requests.d.ts +9 -9
  41. package/dist/src/types/imagen/responses.d.ts +3 -3
  42. package/dist/src/types/live-responses.d.ts +16 -2
  43. package/dist/src/types/requests.d.ts +22 -2
  44. package/dist/src/types/responses.d.ts +99 -1
  45. package/package.json +8 -8
@@ -81,7 +81,7 @@ export declare function getGenerativeModel(ai: AI, modelParams: ModelParams | Hy
81
81
  * @throws If the `apiKey` or `projectId` fields are missing in your
82
82
  * Firebase config.
83
83
  *
84
- * @beta
84
+ * @public
85
85
  */
86
86
  export declare function getImagenModel(ai: AI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel;
87
87
  /**
@@ -25,11 +25,11 @@ import { LanguageModel } from '../types/language-model';
25
25
  export declare class ChromeAdapterImpl implements ChromeAdapter {
26
26
  languageModelProvider: LanguageModel;
27
27
  mode: InferenceMode;
28
- onDeviceParams: OnDeviceParams;
29
28
  static SUPPORTED_MIME_TYPES: string[];
30
29
  private isDownloading;
31
30
  private downloadPromise;
32
31
  private oldSession;
32
+ onDeviceParams: OnDeviceParams;
33
33
  constructor(languageModelProvider: LanguageModel, mode: InferenceMode, onDeviceParams?: OnDeviceParams);
34
34
  /**
35
35
  * Checks if a given request can be made on-device.
@@ -14,7 +14,7 @@
14
14
  * See the License for the specific language governing permissions and
15
15
  * limitations under the License.
16
16
  */
17
- import { LiveServerToolCall, Part } from '../types';
17
+ import { FunctionCall, FunctionResponse } from '../types';
18
18
  import { LiveSession } from './live-session';
19
19
  /**
20
20
  * A controller for managing an active audio conversation.
@@ -39,7 +39,7 @@ export interface StartAudioConversationOptions {
39
39
  * The handler should perform the function call and return the result as a `Part`,
40
40
  * which will then be sent back to the model.
41
41
  */
42
- functionCallingHandler?: (functionCalls: LiveServerToolCall['functionCalls']) => Promise<Part>;
42
+ functionCallingHandler?: (functionCalls: FunctionCall[]) => Promise<FunctionResponse>;
43
43
  }
44
44
  /**
45
45
  * Dependencies needed by the {@link AudioConversationRunner}.
@@ -14,7 +14,7 @@
14
14
  * See the License for the specific language governing permissions and
15
15
  * limitations under the License.
16
16
  */
17
- import { GenerativeContentBlob, LiveServerContent, LiveServerToolCall, LiveServerToolCallCancellation, Part } from '../public-types';
17
+ import { FunctionResponse, GenerativeContentBlob, LiveServerContent, LiveServerToolCall, LiveServerToolCallCancellation, Part } from '../public-types';
18
18
  import { WebSocketHandler } from '../websocket';
19
19
  /**
20
20
  * Represents an active, real-time, bidirectional conversation with the model.
@@ -53,23 +53,65 @@ export declare class LiveSession {
53
53
  */
54
54
  send(request: string | Array<string | Part>, turnComplete?: boolean): Promise<void>;
55
55
  /**
56
- * Sends realtime input to the server.
56
+ * Sends text to the server in realtime.
57
57
  *
58
- * @param mediaChunks - The media chunks to send.
58
+ * @example
59
+ * ```javascript
60
+ * liveSession.sendTextRealtime("Hello, how are you?");
61
+ * ```
62
+ *
63
+ * @param text - The text data to send.
59
64
  * @throws If this session has been closed.
60
65
  *
61
66
  * @beta
62
67
  */
63
- sendMediaChunks(mediaChunks: GenerativeContentBlob[]): Promise<void>;
68
+ sendTextRealtime(text: string): Promise<void>;
64
69
  /**
65
- * Sends a stream of {@link GenerativeContentBlob}.
70
+ * Sends audio data to the server in realtime.
66
71
  *
67
- * @param mediaChunkStream - The stream of {@link GenerativeContentBlob} to send.
72
+ * @remarks The server requires that the audio data is base64-encoded 16-bit PCM at 16kHz
73
+ * little-endian.
74
+ *
75
+ * @example
76
+ * ```javascript
77
+ * // const pcmData = ... base64-encoded 16-bit PCM at 16kHz little-endian.
78
+ * const blob = { mimeType: "audio/pcm", data: pcmData };
79
+ * liveSession.sendAudioRealtime(blob);
80
+ * ```
81
+ *
82
+ * @param blob - The base64-encoded PCM data to send to the server in realtime.
68
83
  * @throws If this session has been closed.
69
84
  *
70
85
  * @beta
71
86
  */
72
- sendMediaStream(mediaChunkStream: ReadableStream<GenerativeContentBlob>): Promise<void>;
87
+ sendAudioRealtime(blob: GenerativeContentBlob): Promise<void>;
88
+ /**
89
+ * Sends video data to the server in realtime.
90
+ *
91
+ * @remarks The server requires that the video is sent as individual video frames at 1 FPS. It
92
+ * is recommended to set `mimeType` to `image/jpeg`.
93
+ *
94
+ * @example
95
+ * ```javascript
96
+ * // const videoFrame = ... base64-encoded JPEG data
97
+ * const blob = { mimeType: "image/jpeg", data: videoFrame };
98
+ * liveSession.sendVideoRealtime(blob);
99
+ * ```
100
+ * @param blob - The base64-encoded video data to send to the server in realtime.
101
+ * @throws If this session has been closed.
102
+ *
103
+ * @beta
104
+ */
105
+ sendVideoRealtime(blob: GenerativeContentBlob): Promise<void>;
106
+ /**
107
+ * Sends function responses to the server.
108
+ *
109
+ * @param functionResponses - The function responses to send.
110
+ * @throws If this session has been closed.
111
+ *
112
+ * @beta
113
+ */
114
+ sendFunctionResponses(functionResponses: FunctionResponse[]): Promise<void>;
73
115
  /**
74
116
  * Yields messages received from the server.
75
117
  * This can only be used by one consumer at a time.
@@ -87,4 +129,26 @@ export declare class LiveSession {
87
129
  * @beta
88
130
  */
89
131
  close(): Promise<void>;
132
+ /**
133
+ * Sends realtime input to the server.
134
+ *
135
+ * @deprecated Use `sendTextRealtime()`, `sendAudioRealtime()`, and `sendVideoRealtime()` instead.
136
+ *
137
+ * @param mediaChunks - The media chunks to send.
138
+ * @throws If this session has been closed.
139
+ *
140
+ * @beta
141
+ */
142
+ sendMediaChunks(mediaChunks: GenerativeContentBlob[]): Promise<void>;
143
+ /**
144
+ * @deprecated Use `sendTextRealtime()`, `sendAudioRealtime()`, and `sendVideoRealtime()` instead.
145
+ *
146
+ * Sends a stream of {@link GenerativeContentBlob}.
147
+ *
148
+ * @param mediaChunkStream - The stream of {@link GenerativeContentBlob} to send.
149
+ * @throws If this session has been closed.
150
+ *
151
+ * @beta
152
+ */
153
+ sendMediaStream(mediaChunkStream: ReadableStream<GenerativeContentBlob>): Promise<void>;
90
154
  }
@@ -37,7 +37,7 @@ import { AIModel } from './ai-model';
37
37
  * }
38
38
  * ```
39
39
  *
40
- * @beta
40
+ * @public
41
41
  */
42
42
  export declare class ImagenModel extends AIModel {
43
43
  requestOptions?: RequestOptions | undefined;
@@ -76,7 +76,7 @@ export declare class ImagenModel extends AIModel {
76
76
  * returned object will have a `filteredReason` property.
77
77
  * If all images are filtered, the `images` array will be empty.
78
78
  *
79
- * @beta
79
+ * @public
80
80
  */
81
81
  generateImages(prompt: string): Promise<ImagenGenerationResponse<ImagenInlineImage>>;
82
82
  /**
@@ -14,7 +14,11 @@
14
14
  * See the License for the specific language governing permissions and
15
15
  * limitations under the License.
16
16
  */
17
- import { GenerateContentRequest, ChromeAdapter } from '../types';
17
+ import { GenerateContentRequest, ChromeAdapter, InferenceSource } from '../types';
18
+ interface CallResult<Response> {
19
+ response: Response;
20
+ inferenceSource: InferenceSource;
21
+ }
18
22
  /**
19
23
  * Dispatches a request to the appropriate backend (on-device or in-cloud)
20
24
  * based on the inference mode.
@@ -25,4 +29,5 @@ import { GenerateContentRequest, ChromeAdapter } from '../types';
25
29
  * @param inCloudCall - The function to call for in-cloud inference.
26
30
  * @returns The response from the backend.
27
31
  */
28
- export declare function callCloudOrDevice<Response>(request: GenerateContentRequest, chromeAdapter: ChromeAdapter | undefined, onDeviceCall: () => Promise<Response>, inCloudCall: () => Promise<Response>): Promise<Response>;
32
+ export declare function callCloudOrDevice<Response>(request: GenerateContentRequest, chromeAdapter: ChromeAdapter | undefined, onDeviceCall: () => Promise<Response>, inCloudCall: () => Promise<Response>): Promise<CallResult<Response>>;
33
+ export {};
@@ -29,7 +29,7 @@
29
29
  * }
30
30
  * ```
31
31
  *
32
- * @beta
32
+ * @public
33
33
  */
34
34
  export declare class ImagenImageFormat {
35
35
  /**
@@ -47,7 +47,7 @@ export declare class ImagenImageFormat {
47
47
  * @param compressionQuality - The level of compression (a number between 0 and 100).
48
48
  * @returns An {@link ImagenImageFormat} object for a JPEG image.
49
49
  *
50
- * @beta
50
+ * @public
51
51
  */
52
52
  static jpeg(compressionQuality?: number): ImagenImageFormat;
53
53
  /**
@@ -55,7 +55,7 @@ export declare class ImagenImageFormat {
55
55
  *
56
56
  * @returns An {@link ImagenImageFormat} object for a PNG image.
57
57
  *
58
- * @beta
58
+ * @public
59
59
  */
60
60
  static png(): ImagenImageFormat;
61
61
  }
@@ -14,12 +14,12 @@
14
14
  * See the License for the specific language governing permissions and
15
15
  * limitations under the License.
16
16
  */
17
- import { EnhancedGenerateContentResponse, FunctionCall, GenerateContentResponse, ImagenGCSImage, ImagenInlineImage, InlineDataPart, Part } from '../types';
17
+ import { EnhancedGenerateContentResponse, FunctionCall, GenerateContentResponse, ImagenGCSImage, ImagenInlineImage, InlineDataPart, Part, InferenceSource } from '../types';
18
18
  /**
19
19
  * Creates an EnhancedGenerateContentResponse object that has helper functions and
20
20
  * other modifications that improve usability.
21
21
  */
22
- export declare function createEnhancedContentResponse(response: GenerateContentResponse): EnhancedGenerateContentResponse;
22
+ export declare function createEnhancedContentResponse(response: GenerateContentResponse, inferenceSource?: InferenceSource): EnhancedGenerateContentResponse;
23
23
  /**
24
24
  * Adds convenience helper methods to a response object, including stream
25
25
  * chunks (as long as each chunk is a complete GenerateContentResponse JSON).
@@ -16,6 +16,7 @@
16
16
  */
17
17
  import { GenerateContentResponse, GenerateContentStreamResult } from '../types';
18
18
  import { ApiSettings } from '../types/internal';
19
+ import { InferenceSource } from '../public-types';
19
20
  /**
20
21
  * Process a response.body stream from the backend and return an
21
22
  * iterator that provides one complete GenerateContentResponse at a time
@@ -24,7 +25,7 @@ import { ApiSettings } from '../types/internal';
24
25
  *
25
26
  * @param response - Response from a fetch call
26
27
  */
27
- export declare function processStream(response: Response, apiSettings: ApiSettings): GenerateContentStreamResult;
28
+ export declare function processStream(response: Response, apiSettings: ApiSettings, inferenceSource?: InferenceSource): GenerateContentStreamResult;
28
29
  /**
29
30
  * Reads a raw stream from the fetch response and join incomplete
30
31
  * chunks, returning a new stream that provides a single complete
@@ -138,7 +138,7 @@ export interface FileDataPart {
138
138
  /**
139
139
  * Represents the code that is executed by the model.
140
140
  *
141
- * @public
141
+ * @beta
142
142
  */
143
143
  export interface ExecutableCodePart {
144
144
  text?: never;
@@ -157,7 +157,7 @@ export interface ExecutableCodePart {
157
157
  /**
158
158
  * Represents the code execution result from the model.
159
159
  *
160
- * @public
160
+ * @beta
161
161
  */
162
162
  export interface CodeExecutionResultPart {
163
163
  text?: never;
@@ -176,7 +176,7 @@ export interface CodeExecutionResultPart {
176
176
  /**
177
177
  * An interface for executable code returned by the model.
178
178
  *
179
- * @public
179
+ * @beta
180
180
  */
181
181
  export interface ExecutableCode {
182
182
  /**
@@ -191,7 +191,7 @@ export interface ExecutableCode {
191
191
  /**
192
192
  * The results of code execution run by the model.
193
193
  *
194
- * @public
194
+ * @beta
195
195
  */
196
196
  export interface CodeExecutionResult {
197
197
  /**
@@ -349,10 +349,25 @@ export declare const InferenceMode: {
349
349
  * @beta
350
350
  */
351
351
  export type InferenceMode = (typeof InferenceMode)[keyof typeof InferenceMode];
352
+ /**
353
+ * Indicates whether inference happened on-device or in-cloud.
354
+ *
355
+ * @beta
356
+ */
357
+ export declare const InferenceSource: {
358
+ readonly ON_DEVICE: "on_device";
359
+ readonly IN_CLOUD: "in_cloud";
360
+ };
361
+ /**
362
+ * Indicates whether inference happened on-device or in-cloud.
363
+ *
364
+ * @beta
365
+ */
366
+ export type InferenceSource = (typeof InferenceSource)[keyof typeof InferenceSource];
352
367
  /**
353
368
  * Represents the result of the code execution.
354
369
  *
355
- * @public
370
+ * @beta
356
371
  */
357
372
  export declare const Outcome: {
358
373
  UNSPECIFIED: string;
@@ -363,13 +378,13 @@ export declare const Outcome: {
363
378
  /**
364
379
  * Represents the result of the code execution.
365
380
  *
366
- * @public
381
+ * @beta
367
382
  */
368
383
  export type Outcome = (typeof Outcome)[keyof typeof Outcome];
369
384
  /**
370
385
  * The programming language of the code.
371
386
  *
372
- * @public
387
+ * @beta
373
388
  */
374
389
  export declare const Language: {
375
390
  UNSPECIFIED: string;
@@ -378,6 +393,6 @@ export declare const Language: {
378
393
  /**
379
394
  * The programming language of the code.
380
395
  *
381
- * @public
396
+ * @beta
382
397
  */
383
398
  export type Language = (typeof Language)[keyof typeof Language];
@@ -14,7 +14,7 @@
14
14
  * See the License for the specific language governing permissions and
15
15
  * limitations under the License.
16
16
  */
17
- import { Tool, GenerationConfig, Citation, FinishReason, GroundingMetadata, PromptFeedback, SafetyRating, UsageMetadata } from '../public-types';
17
+ import { Tool, GenerationConfig, Citation, FinishReason, GroundingMetadata, PromptFeedback, SafetyRating, UsageMetadata, URLContextMetadata } from '../public-types';
18
18
  import { Content, Part } from './content';
19
19
  /**
20
20
  * @internal
@@ -47,6 +47,7 @@ export interface GoogleAIGenerateContentCandidate {
47
47
  safetyRatings?: SafetyRating[];
48
48
  citationMetadata?: GoogleAICitationMetadata;
49
49
  groundingMetadata?: GroundingMetadata;
50
+ urlContextMetadata?: URLContextMetadata;
50
51
  }
51
52
  /**
52
53
  * @internal
@@ -18,7 +18,7 @@ import { ImagenImageFormat } from '../../requests/imagen-image-format';
18
18
  /**
19
19
  * Parameters for configuring an {@link ImagenModel}.
20
20
  *
21
- * @beta
21
+ * @public
22
22
  */
23
23
  export interface ImagenModelParams {
24
24
  /**
@@ -46,7 +46,7 @@ export interface ImagenModelParams {
46
46
  * See the {@link http://firebase.google.com/docs/vertex-ai/generate-images-imagen | documentation} for
47
47
  * more details.
48
48
  *
49
- * @beta
49
+ * @public
50
50
  */
51
51
  export interface ImagenGenerationConfig {
52
52
  /**
@@ -104,7 +104,7 @@ export interface ImagenGenerationConfig {
104
104
  * and the {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters | Responsible AI and usage guidelines}
105
105
  * for more details.
106
106
  *
107
- * @beta
107
+ * @public
108
108
  */
109
109
  export declare const ImagenSafetyFilterLevel: {
110
110
  /**
@@ -137,7 +137,7 @@ export declare const ImagenSafetyFilterLevel: {
137
137
  * and the {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters | Responsible AI and usage guidelines}
138
138
  * for more details.
139
139
  *
140
- * @beta
140
+ * @public
141
141
  */
142
142
  export type ImagenSafetyFilterLevel = (typeof ImagenSafetyFilterLevel)[keyof typeof ImagenSafetyFilterLevel];
143
143
  /**
@@ -146,7 +146,7 @@ export type ImagenSafetyFilterLevel = (typeof ImagenSafetyFilterLevel)[keyof typ
146
146
  * See the <a href="http://firebase.google.com/docs/vertex-ai/generate-images">personGeneration</a>
147
147
  * documentation for more details.
148
148
  *
149
- * @beta
149
+ * @public
150
150
  */
151
151
  export declare const ImagenPersonFilterLevel: {
152
152
  /**
@@ -176,7 +176,7 @@ export declare const ImagenPersonFilterLevel: {
176
176
  * See the <a href="http://firebase.google.com/docs/vertex-ai/generate-images">personGeneration</a>
177
177
  * documentation for more details.
178
178
  *
179
- * @beta
179
+ * @public
180
180
  */
181
181
  export type ImagenPersonFilterLevel = (typeof ImagenPersonFilterLevel)[keyof typeof ImagenPersonFilterLevel];
182
182
  /**
@@ -185,7 +185,7 @@ export type ImagenPersonFilterLevel = (typeof ImagenPersonFilterLevel)[keyof typ
185
185
  * See the {@link http://firebase.google.com/docs/vertex-ai/generate-images | documentation }
186
186
  * for more details.
187
187
  *
188
- * @beta
188
+ * @public
189
189
  */
190
190
  export interface ImagenSafetySettings {
191
191
  /**
@@ -207,7 +207,7 @@ export interface ImagenSafetySettings {
207
207
  * See the {@link http://firebase.google.com/docs/vertex-ai/generate-images | documentation }
208
208
  * for more details and examples of the supported aspect ratios.
209
209
  *
210
- * @beta
210
+ * @public
211
211
  */
212
212
  export declare const ImagenAspectRatio: {
213
213
  /**
@@ -240,6 +240,6 @@ export declare const ImagenAspectRatio: {
240
240
  * See the {@link http://firebase.google.com/docs/vertex-ai/generate-images | documentation }
241
241
  * for more details and examples of the supported aspect ratios.
242
242
  *
243
- * @beta
243
+ * @public
244
244
  */
245
245
  export type ImagenAspectRatio = (typeof ImagenAspectRatio)[keyof typeof ImagenAspectRatio];
@@ -17,7 +17,7 @@
17
17
  /**
18
18
  * An image generated by Imagen, represented as inline data.
19
19
  *
20
- * @beta
20
+ * @public
21
21
  */
22
22
  export interface ImagenInlineImage {
23
23
  /**
@@ -35,7 +35,7 @@ export interface ImagenInlineImage {
35
35
  * An image generated by Imagen, stored in a Cloud Storage for Firebase bucket.
36
36
  *
37
37
  * This feature is not available yet.
38
- * @beta
38
+ * @public
39
39
  */
40
40
  export interface ImagenGCSImage {
41
41
  /**
@@ -54,7 +54,7 @@ export interface ImagenGCSImage {
54
54
  /**
55
55
  * The response from a request to generate images with Imagen.
56
56
  *
57
- * @beta
57
+ * @public
58
58
  */
59
59
  export interface ImagenGenerationResponse<T extends ImagenInlineImage | ImagenGCSImage> {
60
60
  /**
@@ -14,7 +14,7 @@
14
14
  * See the License for the specific language governing permissions and
15
15
  * limitations under the License.
16
16
  */
17
- import { Content, GenerativeContentBlob, Part } from './content';
17
+ import { Content, FunctionResponse, GenerativeContentBlob, Part } from './content';
18
18
  import { LiveGenerationConfig, Tool, ToolConfig } from './requests';
19
19
  /**
20
20
  * User input that is sent to the model.
@@ -34,7 +34,21 @@ export interface _LiveClientContent {
34
34
  */
35
35
  export interface _LiveClientRealtimeInput {
36
36
  realtimeInput: {
37
- mediaChunks: GenerativeContentBlob[];
37
+ text?: string;
38
+ audio?: GenerativeContentBlob;
39
+ video?: GenerativeContentBlob;
40
+ /**
41
+ * @deprecated Use `text`, `audio`, and `video` instead.
42
+ */
43
+ mediaChunks?: GenerativeContentBlob[];
44
+ };
45
+ }
46
+ /**
47
+ * Function responses that are sent to the model in real time.
48
+ */
49
+ export interface _LiveClientToolResponse {
50
+ toolResponse: {
51
+ functionResponses: FunctionResponse[];
38
52
  };
39
53
  }
40
54
  /**
@@ -218,7 +218,7 @@ export interface RequestOptions {
218
218
  * Defines a tool that model can call to access external knowledge.
219
219
  * @public
220
220
  */
221
- export type Tool = FunctionDeclarationsTool | GoogleSearchTool | CodeExecutionTool;
221
+ export type Tool = FunctionDeclarationsTool | GoogleSearchTool | CodeExecutionTool | URLContextTool;
222
222
  /**
223
223
  * Structured representation of a function declaration as defined by the
224
224
  * {@link https://spec.openapis.org/oas/v3.0.3 | OpenAPI 3.0 specification}.
@@ -273,7 +273,7 @@ export interface GoogleSearchTool {
273
273
  /**
274
274
  * A tool that enables the model to use code execution.
275
275
  *
276
- * @public
276
+ * @beta
277
277
  */
278
278
  export interface CodeExecutionTool {
279
279
  /**
@@ -291,6 +291,26 @@ export interface CodeExecutionTool {
291
291
  */
292
292
  export interface GoogleSearch {
293
293
  }
294
+ /**
295
+ * A tool that allows you to provide additional context to the models in the form of public web
296
+ * URLs. By including URLs in your request, the Gemini model will access the content from those
297
+ * pages to inform and enhance its response.
298
+ *
299
+ * @beta
300
+ */
301
+ export interface URLContextTool {
302
+ /**
303
+ * Specifies the URL Context configuration.
304
+ */
305
+ urlContext: URLContext;
306
+ }
307
+ /**
308
+ * Specifies the URL Context configuration.
309
+ *
310
+ * @beta
311
+ */
312
+ export interface URLContext {
313
+ }
294
314
  /**
295
315
  * A `FunctionDeclarationsTool` is a piece of code that enables the system to
296
316
  * interact with external systems to perform an action, or set of actions,
@@ -15,7 +15,7 @@
15
15
  * limitations under the License.
16
16
  */
17
17
  import { Content, FunctionCall, InlineDataPart } from './content';
18
- import { BlockReason, FinishReason, HarmCategory, HarmProbability, HarmSeverity, Modality } from './enums';
18
+ import { BlockReason, FinishReason, HarmCategory, HarmProbability, HarmSeverity, InferenceSource, Modality } from './enums';
19
19
  /**
20
20
  * Result object returned from {@link GenerativeModel.generateContent} call.
21
21
  *
@@ -76,6 +76,12 @@ export interface EnhancedGenerateContentResponse extends GenerateContentResponse
76
76
  * set to `true`.
77
77
  */
78
78
  thoughtSummary: () => string | undefined;
79
+ /**
80
+ * Indicates whether inference happened on-device or in-cloud.
81
+ *
82
+ * @beta
83
+ */
84
+ inferenceSource?: InferenceSource;
79
85
  }
80
86
  /**
81
87
  * Individual response from {@link GenerativeModel.generateContent} and
@@ -102,8 +108,16 @@ export interface UsageMetadata {
102
108
  */
103
109
  thoughtsTokenCount?: number;
104
110
  totalTokenCount: number;
111
+ /**
112
+ * The number of tokens used by tools.
113
+ */
114
+ toolUsePromptTokenCount?: number;
105
115
  promptTokensDetails?: ModalityTokenCount[];
106
116
  candidatesTokensDetails?: ModalityTokenCount[];
117
+ /**
118
+ * A list of tokens used by tools, broken down by modality.
119
+ */
120
+ toolUsePromptTokensDetails?: ModalityTokenCount[];
107
121
  }
108
122
  /**
109
123
  * Represents token counting info for a single modality.
@@ -143,6 +157,7 @@ export interface GenerateContentCandidate {
143
157
  safetyRatings?: SafetyRating[];
144
158
  citationMetadata?: CitationMetadata;
145
159
  groundingMetadata?: GroundingMetadata;
160
+ urlContextMetadata?: URLContextMetadata;
146
161
  }
147
162
  /**
148
163
  * Citation metadata that may be found on a {@link GenerateContentCandidate}.
@@ -323,6 +338,89 @@ export interface Segment {
323
338
  */
324
339
  text: string;
325
340
  }
341
+ /**
342
+ * Metadata related to {@link URLContextTool}.
343
+ *
344
+ * @beta
345
+ */
346
+ export interface URLContextMetadata {
347
+ /**
348
+ * List of URL metadata used to provide context to the Gemini model.
349
+ */
350
+ urlMetadata: URLMetadata[];
351
+ }
352
+ /**
353
+ * Metadata for a single URL retrieved by the {@link URLContextTool} tool.
354
+ *
355
+ * @beta
356
+ */
357
+ export interface URLMetadata {
358
+ /**
359
+ * The retrieved URL.
360
+ */
361
+ retrievedUrl?: string;
362
+ /**
363
+ * The status of the URL retrieval.
364
+ */
365
+ urlRetrievalStatus?: URLRetrievalStatus;
366
+ }
367
+ /**
368
+ * The status of a URL retrieval.
369
+ *
370
+ * @remarks
371
+ * <b>URL_RETRIEVAL_STATUS_UNSPECIFIED:</b> Unspecified retrieval status.
372
+ * <br/>
373
+ * <b>URL_RETRIEVAL_STATUS_SUCCESS:</b> The URL retrieval was successful.
374
+ * <br/>
375
+ * <b>URL_RETRIEVAL_STATUS_ERROR:</b> The URL retrieval failed.
376
+ * <br/>
377
+ * <b>URL_RETRIEVAL_STATUS_PAYWALL:</b> The URL retrieval failed because the content is behind a paywall.
378
+ * <br/>
379
+ * <b>URL_RETRIEVAL_STATUS_UNSAFE:</b> The URL retrieval failed because the content is unsafe.
380
+ * <br/>
381
+ *
382
+ * @beta
383
+ */
384
+ export declare const URLRetrievalStatus: {
385
+ /**
386
+ * Unspecified retrieval status.
387
+ */
388
+ URL_RETRIEVAL_STATUS_UNSPECIFIED: string;
389
+ /**
390
+ * The URL retrieval was successful.
391
+ */
392
+ URL_RETRIEVAL_STATUS_SUCCESS: string;
393
+ /**
394
+ * The URL retrieval failed.
395
+ */
396
+ URL_RETRIEVAL_STATUS_ERROR: string;
397
+ /**
398
+ * The URL retrieval failed because the content is behind a paywall.
399
+ */
400
+ URL_RETRIEVAL_STATUS_PAYWALL: string;
401
+ /**
402
+ * The URL retrieval failed because the content is unsafe.
403
+ */
404
+ URL_RETRIEVAL_STATUS_UNSAFE: string;
405
+ };
406
+ /**
407
+ * The status of a URL retrieval.
408
+ *
409
+ * @remarks
410
+ * <b>URL_RETRIEVAL_STATUS_UNSPECIFIED:</b> Unspecified retrieval status.
411
+ * <br/>
412
+ * <b>URL_RETRIEVAL_STATUS_SUCCESS:</b> The URL retrieval was successful.
413
+ * <br/>
414
+ * <b>URL_RETRIEVAL_STATUS_ERROR:</b> The URL retrieval failed.
415
+ * <br/>
416
+ * <b>URL_RETRIEVAL_STATUS_PAYWALL:</b> The URL retrieval failed because the content is behind a paywall.
417
+ * <br/>
418
+ * <b>URL_RETRIEVAL_STATUS_UNSAFE:</b> The URL retrieval failed because the content is unsafe.
419
+ * <br/>
420
+ *
421
+ * @beta
422
+ */
423
+ export type URLRetrievalStatus = (typeof URLRetrievalStatus)[keyof typeof URLRetrievalStatus];
326
424
  /**
327
425
  * @public
328
426
  */