@firebase/ai 2.3.0 → 2.4.0-canary.22e0a1adb
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-public.d.ts +240 -35
- package/dist/ai.d.ts +241 -35
- package/dist/esm/index.esm.js +295 -92
- package/dist/esm/index.esm.js.map +1 -1
- package/dist/esm/src/api.d.ts +1 -1
- package/dist/esm/src/methods/chrome-adapter.d.ts +1 -1
- package/dist/esm/src/methods/live-session-helpers.d.ts +2 -2
- package/dist/esm/src/methods/live-session.d.ts +71 -7
- package/dist/esm/src/models/imagen-model.d.ts +2 -2
- package/dist/esm/src/requests/hybrid-helpers.d.ts +7 -2
- package/dist/esm/src/requests/imagen-image-format.d.ts +3 -3
- package/dist/esm/src/requests/response-helpers.d.ts +2 -2
- package/dist/esm/src/requests/stream-reader.d.ts +2 -1
- package/dist/esm/src/types/content.d.ts +4 -4
- package/dist/esm/src/types/enums.d.ts +19 -4
- package/dist/esm/src/types/googleai.d.ts +2 -1
- package/dist/esm/src/types/imagen/requests.d.ts +9 -9
- package/dist/esm/src/types/imagen/responses.d.ts +3 -3
- package/dist/esm/src/types/live-responses.d.ts +16 -2
- package/dist/esm/src/types/requests.d.ts +22 -2
- package/dist/esm/src/types/responses.d.ts +99 -1
- package/dist/index.cjs.js +296 -91
- package/dist/index.cjs.js.map +1 -1
- package/dist/index.node.cjs.js +276 -84
- package/dist/index.node.cjs.js.map +1 -1
- package/dist/index.node.mjs +275 -85
- package/dist/index.node.mjs.map +1 -1
- package/dist/src/api.d.ts +1 -1
- package/dist/src/methods/chrome-adapter.d.ts +1 -1
- package/dist/src/methods/live-session-helpers.d.ts +2 -2
- package/dist/src/methods/live-session.d.ts +71 -7
- package/dist/src/models/imagen-model.d.ts +2 -2
- package/dist/src/requests/hybrid-helpers.d.ts +7 -2
- package/dist/src/requests/imagen-image-format.d.ts +3 -3
- package/dist/src/requests/response-helpers.d.ts +2 -2
- package/dist/src/requests/stream-reader.d.ts +2 -1
- package/dist/src/types/content.d.ts +4 -4
- package/dist/src/types/enums.d.ts +19 -4
- package/dist/src/types/googleai.d.ts +2 -1
- package/dist/src/types/imagen/requests.d.ts +9 -9
- package/dist/src/types/imagen/responses.d.ts +3 -3
- package/dist/src/types/live-responses.d.ts +16 -2
- package/dist/src/types/requests.d.ts +22 -2
- package/dist/src/types/responses.d.ts +99 -1
- package/package.json +8 -8
package/dist/esm/src/api.d.ts
CHANGED
|
@@ -81,7 +81,7 @@ export declare function getGenerativeModel(ai: AI, modelParams: ModelParams | Hy
|
|
|
81
81
|
* @throws If the `apiKey` or `projectId` fields are missing in your
|
|
82
82
|
* Firebase config.
|
|
83
83
|
*
|
|
84
|
-
* @
|
|
84
|
+
* @public
|
|
85
85
|
*/
|
|
86
86
|
export declare function getImagenModel(ai: AI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel;
|
|
87
87
|
/**
|
|
@@ -25,11 +25,11 @@ import { LanguageModel } from '../types/language-model';
|
|
|
25
25
|
export declare class ChromeAdapterImpl implements ChromeAdapter {
|
|
26
26
|
languageModelProvider: LanguageModel;
|
|
27
27
|
mode: InferenceMode;
|
|
28
|
-
onDeviceParams: OnDeviceParams;
|
|
29
28
|
static SUPPORTED_MIME_TYPES: string[];
|
|
30
29
|
private isDownloading;
|
|
31
30
|
private downloadPromise;
|
|
32
31
|
private oldSession;
|
|
32
|
+
onDeviceParams: OnDeviceParams;
|
|
33
33
|
constructor(languageModelProvider: LanguageModel, mode: InferenceMode, onDeviceParams?: OnDeviceParams);
|
|
34
34
|
/**
|
|
35
35
|
* Checks if a given request can be made on-device.
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
* See the License for the specific language governing permissions and
|
|
15
15
|
* limitations under the License.
|
|
16
16
|
*/
|
|
17
|
-
import {
|
|
17
|
+
import { FunctionCall, FunctionResponse } from '../types';
|
|
18
18
|
import { LiveSession } from './live-session';
|
|
19
19
|
/**
|
|
20
20
|
* A controller for managing an active audio conversation.
|
|
@@ -39,7 +39,7 @@ export interface StartAudioConversationOptions {
|
|
|
39
39
|
* The handler should perform the function call and return the result as a `Part`,
|
|
40
40
|
* which will then be sent back to the model.
|
|
41
41
|
*/
|
|
42
|
-
functionCallingHandler?: (functionCalls:
|
|
42
|
+
functionCallingHandler?: (functionCalls: FunctionCall[]) => Promise<FunctionResponse>;
|
|
43
43
|
}
|
|
44
44
|
/**
|
|
45
45
|
* Dependencies needed by the {@link AudioConversationRunner}.
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
* See the License for the specific language governing permissions and
|
|
15
15
|
* limitations under the License.
|
|
16
16
|
*/
|
|
17
|
-
import { GenerativeContentBlob, LiveServerContent, LiveServerToolCall, LiveServerToolCallCancellation, Part } from '../public-types';
|
|
17
|
+
import { FunctionResponse, GenerativeContentBlob, LiveServerContent, LiveServerToolCall, LiveServerToolCallCancellation, Part } from '../public-types';
|
|
18
18
|
import { WebSocketHandler } from '../websocket';
|
|
19
19
|
/**
|
|
20
20
|
* Represents an active, real-time, bidirectional conversation with the model.
|
|
@@ -53,23 +53,65 @@ export declare class LiveSession {
|
|
|
53
53
|
*/
|
|
54
54
|
send(request: string | Array<string | Part>, turnComplete?: boolean): Promise<void>;
|
|
55
55
|
/**
|
|
56
|
-
* Sends
|
|
56
|
+
* Sends text to the server in realtime.
|
|
57
57
|
*
|
|
58
|
-
* @
|
|
58
|
+
* @example
|
|
59
|
+
* ```javascript
|
|
60
|
+
* liveSession.sendTextRealtime("Hello, how are you?");
|
|
61
|
+
* ```
|
|
62
|
+
*
|
|
63
|
+
* @param text - The text data to send.
|
|
59
64
|
* @throws If this session has been closed.
|
|
60
65
|
*
|
|
61
66
|
* @beta
|
|
62
67
|
*/
|
|
63
|
-
|
|
68
|
+
sendTextRealtime(text: string): Promise<void>;
|
|
64
69
|
/**
|
|
65
|
-
* Sends
|
|
70
|
+
* Sends audio data to the server in realtime.
|
|
66
71
|
*
|
|
67
|
-
* @
|
|
72
|
+
* @remarks The server requires that the audio data is base64-encoded 16-bit PCM at 16kHz
|
|
73
|
+
* little-endian.
|
|
74
|
+
*
|
|
75
|
+
* @example
|
|
76
|
+
* ```javascript
|
|
77
|
+
* // const pcmData = ... base64-encoded 16-bit PCM at 16kHz little-endian.
|
|
78
|
+
* const blob = { mimeType: "audio/pcm", data: pcmData };
|
|
79
|
+
* liveSession.sendAudioRealtime(blob);
|
|
80
|
+
* ```
|
|
81
|
+
*
|
|
82
|
+
* @param blob - The base64-encoded PCM data to send to the server in realtime.
|
|
68
83
|
* @throws If this session has been closed.
|
|
69
84
|
*
|
|
70
85
|
* @beta
|
|
71
86
|
*/
|
|
72
|
-
|
|
87
|
+
sendAudioRealtime(blob: GenerativeContentBlob): Promise<void>;
|
|
88
|
+
/**
|
|
89
|
+
* Sends video data to the server in realtime.
|
|
90
|
+
*
|
|
91
|
+
* @remarks The server requires that the video is sent as individual video frames at 1 FPS. It
|
|
92
|
+
* is recommended to set `mimeType` to `image/jpeg`.
|
|
93
|
+
*
|
|
94
|
+
* @example
|
|
95
|
+
* ```javascript
|
|
96
|
+
* // const videoFrame = ... base64-encoded JPEG data
|
|
97
|
+
* const blob = { mimeType: "image/jpeg", data: videoFrame };
|
|
98
|
+
* liveSession.sendVideoRealtime(blob);
|
|
99
|
+
* ```
|
|
100
|
+
* @param blob - The base64-encoded video data to send to the server in realtime.
|
|
101
|
+
* @throws If this session has been closed.
|
|
102
|
+
*
|
|
103
|
+
* @beta
|
|
104
|
+
*/
|
|
105
|
+
sendVideoRealtime(blob: GenerativeContentBlob): Promise<void>;
|
|
106
|
+
/**
|
|
107
|
+
* Sends function responses to the server.
|
|
108
|
+
*
|
|
109
|
+
* @param functionResponses - The function responses to send.
|
|
110
|
+
* @throws If this session has been closed.
|
|
111
|
+
*
|
|
112
|
+
* @beta
|
|
113
|
+
*/
|
|
114
|
+
sendFunctionResponses(functionResponses: FunctionResponse[]): Promise<void>;
|
|
73
115
|
/**
|
|
74
116
|
* Yields messages received from the server.
|
|
75
117
|
* This can only be used by one consumer at a time.
|
|
@@ -87,4 +129,26 @@ export declare class LiveSession {
|
|
|
87
129
|
* @beta
|
|
88
130
|
*/
|
|
89
131
|
close(): Promise<void>;
|
|
132
|
+
/**
|
|
133
|
+
* Sends realtime input to the server.
|
|
134
|
+
*
|
|
135
|
+
* @deprecated Use `sendTextRealtime()`, `sendAudioRealtime()`, and `sendVideoRealtime()` instead.
|
|
136
|
+
*
|
|
137
|
+
* @param mediaChunks - The media chunks to send.
|
|
138
|
+
* @throws If this session has been closed.
|
|
139
|
+
*
|
|
140
|
+
* @beta
|
|
141
|
+
*/
|
|
142
|
+
sendMediaChunks(mediaChunks: GenerativeContentBlob[]): Promise<void>;
|
|
143
|
+
/**
|
|
144
|
+
* @deprecated Use `sendTextRealtime()`, `sendAudioRealtime()`, and `sendVideoRealtime()` instead.
|
|
145
|
+
*
|
|
146
|
+
* Sends a stream of {@link GenerativeContentBlob}.
|
|
147
|
+
*
|
|
148
|
+
* @param mediaChunkStream - The stream of {@link GenerativeContentBlob} to send.
|
|
149
|
+
* @throws If this session has been closed.
|
|
150
|
+
*
|
|
151
|
+
* @beta
|
|
152
|
+
*/
|
|
153
|
+
sendMediaStream(mediaChunkStream: ReadableStream<GenerativeContentBlob>): Promise<void>;
|
|
90
154
|
}
|
|
@@ -37,7 +37,7 @@ import { AIModel } from './ai-model';
|
|
|
37
37
|
* }
|
|
38
38
|
* ```
|
|
39
39
|
*
|
|
40
|
-
* @
|
|
40
|
+
* @public
|
|
41
41
|
*/
|
|
42
42
|
export declare class ImagenModel extends AIModel {
|
|
43
43
|
requestOptions?: RequestOptions | undefined;
|
|
@@ -76,7 +76,7 @@ export declare class ImagenModel extends AIModel {
|
|
|
76
76
|
* returned object will have a `filteredReason` property.
|
|
77
77
|
* If all images are filtered, the `images` array will be empty.
|
|
78
78
|
*
|
|
79
|
-
* @
|
|
79
|
+
* @public
|
|
80
80
|
*/
|
|
81
81
|
generateImages(prompt: string): Promise<ImagenGenerationResponse<ImagenInlineImage>>;
|
|
82
82
|
/**
|
|
@@ -14,7 +14,11 @@
|
|
|
14
14
|
* See the License for the specific language governing permissions and
|
|
15
15
|
* limitations under the License.
|
|
16
16
|
*/
|
|
17
|
-
import { GenerateContentRequest, ChromeAdapter } from '../types';
|
|
17
|
+
import { GenerateContentRequest, ChromeAdapter, InferenceSource } from '../types';
|
|
18
|
+
interface CallResult<Response> {
|
|
19
|
+
response: Response;
|
|
20
|
+
inferenceSource: InferenceSource;
|
|
21
|
+
}
|
|
18
22
|
/**
|
|
19
23
|
* Dispatches a request to the appropriate backend (on-device or in-cloud)
|
|
20
24
|
* based on the inference mode.
|
|
@@ -25,4 +29,5 @@ import { GenerateContentRequest, ChromeAdapter } from '../types';
|
|
|
25
29
|
* @param inCloudCall - The function to call for in-cloud inference.
|
|
26
30
|
* @returns The response from the backend.
|
|
27
31
|
*/
|
|
28
|
-
export declare function callCloudOrDevice<Response>(request: GenerateContentRequest, chromeAdapter: ChromeAdapter | undefined, onDeviceCall: () => Promise<Response>, inCloudCall: () => Promise<Response>): Promise<Response
|
|
32
|
+
export declare function callCloudOrDevice<Response>(request: GenerateContentRequest, chromeAdapter: ChromeAdapter | undefined, onDeviceCall: () => Promise<Response>, inCloudCall: () => Promise<Response>): Promise<CallResult<Response>>;
|
|
33
|
+
export {};
|
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
* }
|
|
30
30
|
* ```
|
|
31
31
|
*
|
|
32
|
-
* @
|
|
32
|
+
* @public
|
|
33
33
|
*/
|
|
34
34
|
export declare class ImagenImageFormat {
|
|
35
35
|
/**
|
|
@@ -47,7 +47,7 @@ export declare class ImagenImageFormat {
|
|
|
47
47
|
* @param compressionQuality - The level of compression (a number between 0 and 100).
|
|
48
48
|
* @returns An {@link ImagenImageFormat} object for a JPEG image.
|
|
49
49
|
*
|
|
50
|
-
* @
|
|
50
|
+
* @public
|
|
51
51
|
*/
|
|
52
52
|
static jpeg(compressionQuality?: number): ImagenImageFormat;
|
|
53
53
|
/**
|
|
@@ -55,7 +55,7 @@ export declare class ImagenImageFormat {
|
|
|
55
55
|
*
|
|
56
56
|
* @returns An {@link ImagenImageFormat} object for a PNG image.
|
|
57
57
|
*
|
|
58
|
-
* @
|
|
58
|
+
* @public
|
|
59
59
|
*/
|
|
60
60
|
static png(): ImagenImageFormat;
|
|
61
61
|
}
|
|
@@ -14,12 +14,12 @@
|
|
|
14
14
|
* See the License for the specific language governing permissions and
|
|
15
15
|
* limitations under the License.
|
|
16
16
|
*/
|
|
17
|
-
import { EnhancedGenerateContentResponse, FunctionCall, GenerateContentResponse, ImagenGCSImage, ImagenInlineImage, InlineDataPart, Part } from '../types';
|
|
17
|
+
import { EnhancedGenerateContentResponse, FunctionCall, GenerateContentResponse, ImagenGCSImage, ImagenInlineImage, InlineDataPart, Part, InferenceSource } from '../types';
|
|
18
18
|
/**
|
|
19
19
|
* Creates an EnhancedGenerateContentResponse object that has helper functions and
|
|
20
20
|
* other modifications that improve usability.
|
|
21
21
|
*/
|
|
22
|
-
export declare function createEnhancedContentResponse(response: GenerateContentResponse): EnhancedGenerateContentResponse;
|
|
22
|
+
export declare function createEnhancedContentResponse(response: GenerateContentResponse, inferenceSource?: InferenceSource): EnhancedGenerateContentResponse;
|
|
23
23
|
/**
|
|
24
24
|
* Adds convenience helper methods to a response object, including stream
|
|
25
25
|
* chunks (as long as each chunk is a complete GenerateContentResponse JSON).
|
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
*/
|
|
17
17
|
import { GenerateContentResponse, GenerateContentStreamResult } from '../types';
|
|
18
18
|
import { ApiSettings } from '../types/internal';
|
|
19
|
+
import { InferenceSource } from '../public-types';
|
|
19
20
|
/**
|
|
20
21
|
* Process a response.body stream from the backend and return an
|
|
21
22
|
* iterator that provides one complete GenerateContentResponse at a time
|
|
@@ -24,7 +25,7 @@ import { ApiSettings } from '../types/internal';
|
|
|
24
25
|
*
|
|
25
26
|
* @param response - Response from a fetch call
|
|
26
27
|
*/
|
|
27
|
-
export declare function processStream(response: Response, apiSettings: ApiSettings): GenerateContentStreamResult;
|
|
28
|
+
export declare function processStream(response: Response, apiSettings: ApiSettings, inferenceSource?: InferenceSource): GenerateContentStreamResult;
|
|
28
29
|
/**
|
|
29
30
|
* Reads a raw stream from the fetch response and join incomplete
|
|
30
31
|
* chunks, returning a new stream that provides a single complete
|
|
@@ -138,7 +138,7 @@ export interface FileDataPart {
|
|
|
138
138
|
/**
|
|
139
139
|
* Represents the code that is executed by the model.
|
|
140
140
|
*
|
|
141
|
-
* @
|
|
141
|
+
* @beta
|
|
142
142
|
*/
|
|
143
143
|
export interface ExecutableCodePart {
|
|
144
144
|
text?: never;
|
|
@@ -157,7 +157,7 @@ export interface ExecutableCodePart {
|
|
|
157
157
|
/**
|
|
158
158
|
* Represents the code execution result from the model.
|
|
159
159
|
*
|
|
160
|
-
* @
|
|
160
|
+
* @beta
|
|
161
161
|
*/
|
|
162
162
|
export interface CodeExecutionResultPart {
|
|
163
163
|
text?: never;
|
|
@@ -176,7 +176,7 @@ export interface CodeExecutionResultPart {
|
|
|
176
176
|
/**
|
|
177
177
|
* An interface for executable code returned by the model.
|
|
178
178
|
*
|
|
179
|
-
* @
|
|
179
|
+
* @beta
|
|
180
180
|
*/
|
|
181
181
|
export interface ExecutableCode {
|
|
182
182
|
/**
|
|
@@ -191,7 +191,7 @@ export interface ExecutableCode {
|
|
|
191
191
|
/**
|
|
192
192
|
* The results of code execution run by the model.
|
|
193
193
|
*
|
|
194
|
-
* @
|
|
194
|
+
* @beta
|
|
195
195
|
*/
|
|
196
196
|
export interface CodeExecutionResult {
|
|
197
197
|
/**
|
|
@@ -349,10 +349,25 @@ export declare const InferenceMode: {
|
|
|
349
349
|
* @beta
|
|
350
350
|
*/
|
|
351
351
|
export type InferenceMode = (typeof InferenceMode)[keyof typeof InferenceMode];
|
|
352
|
+
/**
|
|
353
|
+
* Indicates whether inference happened on-device or in-cloud.
|
|
354
|
+
*
|
|
355
|
+
* @beta
|
|
356
|
+
*/
|
|
357
|
+
export declare const InferenceSource: {
|
|
358
|
+
readonly ON_DEVICE: "on_device";
|
|
359
|
+
readonly IN_CLOUD: "in_cloud";
|
|
360
|
+
};
|
|
361
|
+
/**
|
|
362
|
+
* Indicates whether inference happened on-device or in-cloud.
|
|
363
|
+
*
|
|
364
|
+
* @beta
|
|
365
|
+
*/
|
|
366
|
+
export type InferenceSource = (typeof InferenceSource)[keyof typeof InferenceSource];
|
|
352
367
|
/**
|
|
353
368
|
* Represents the result of the code execution.
|
|
354
369
|
*
|
|
355
|
-
* @
|
|
370
|
+
* @beta
|
|
356
371
|
*/
|
|
357
372
|
export declare const Outcome: {
|
|
358
373
|
UNSPECIFIED: string;
|
|
@@ -363,13 +378,13 @@ export declare const Outcome: {
|
|
|
363
378
|
/**
|
|
364
379
|
* Represents the result of the code execution.
|
|
365
380
|
*
|
|
366
|
-
* @
|
|
381
|
+
* @beta
|
|
367
382
|
*/
|
|
368
383
|
export type Outcome = (typeof Outcome)[keyof typeof Outcome];
|
|
369
384
|
/**
|
|
370
385
|
* The programming language of the code.
|
|
371
386
|
*
|
|
372
|
-
* @
|
|
387
|
+
* @beta
|
|
373
388
|
*/
|
|
374
389
|
export declare const Language: {
|
|
375
390
|
UNSPECIFIED: string;
|
|
@@ -378,6 +393,6 @@ export declare const Language: {
|
|
|
378
393
|
/**
|
|
379
394
|
* The programming language of the code.
|
|
380
395
|
*
|
|
381
|
-
* @
|
|
396
|
+
* @beta
|
|
382
397
|
*/
|
|
383
398
|
export type Language = (typeof Language)[keyof typeof Language];
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
* See the License for the specific language governing permissions and
|
|
15
15
|
* limitations under the License.
|
|
16
16
|
*/
|
|
17
|
-
import { Tool, GenerationConfig, Citation, FinishReason, GroundingMetadata, PromptFeedback, SafetyRating, UsageMetadata } from '../public-types';
|
|
17
|
+
import { Tool, GenerationConfig, Citation, FinishReason, GroundingMetadata, PromptFeedback, SafetyRating, UsageMetadata, URLContextMetadata } from '../public-types';
|
|
18
18
|
import { Content, Part } from './content';
|
|
19
19
|
/**
|
|
20
20
|
* @internal
|
|
@@ -47,6 +47,7 @@ export interface GoogleAIGenerateContentCandidate {
|
|
|
47
47
|
safetyRatings?: SafetyRating[];
|
|
48
48
|
citationMetadata?: GoogleAICitationMetadata;
|
|
49
49
|
groundingMetadata?: GroundingMetadata;
|
|
50
|
+
urlContextMetadata?: URLContextMetadata;
|
|
50
51
|
}
|
|
51
52
|
/**
|
|
52
53
|
* @internal
|
|
@@ -18,7 +18,7 @@ import { ImagenImageFormat } from '../../requests/imagen-image-format';
|
|
|
18
18
|
/**
|
|
19
19
|
* Parameters for configuring an {@link ImagenModel}.
|
|
20
20
|
*
|
|
21
|
-
* @
|
|
21
|
+
* @public
|
|
22
22
|
*/
|
|
23
23
|
export interface ImagenModelParams {
|
|
24
24
|
/**
|
|
@@ -46,7 +46,7 @@ export interface ImagenModelParams {
|
|
|
46
46
|
* See the {@link http://firebase.google.com/docs/vertex-ai/generate-images-imagen | documentation} for
|
|
47
47
|
* more details.
|
|
48
48
|
*
|
|
49
|
-
* @
|
|
49
|
+
* @public
|
|
50
50
|
*/
|
|
51
51
|
export interface ImagenGenerationConfig {
|
|
52
52
|
/**
|
|
@@ -104,7 +104,7 @@ export interface ImagenGenerationConfig {
|
|
|
104
104
|
* and the {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters | Responsible AI and usage guidelines}
|
|
105
105
|
* for more details.
|
|
106
106
|
*
|
|
107
|
-
* @
|
|
107
|
+
* @public
|
|
108
108
|
*/
|
|
109
109
|
export declare const ImagenSafetyFilterLevel: {
|
|
110
110
|
/**
|
|
@@ -137,7 +137,7 @@ export declare const ImagenSafetyFilterLevel: {
|
|
|
137
137
|
* and the {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters | Responsible AI and usage guidelines}
|
|
138
138
|
* for more details.
|
|
139
139
|
*
|
|
140
|
-
* @
|
|
140
|
+
* @public
|
|
141
141
|
*/
|
|
142
142
|
export type ImagenSafetyFilterLevel = (typeof ImagenSafetyFilterLevel)[keyof typeof ImagenSafetyFilterLevel];
|
|
143
143
|
/**
|
|
@@ -146,7 +146,7 @@ export type ImagenSafetyFilterLevel = (typeof ImagenSafetyFilterLevel)[keyof typ
|
|
|
146
146
|
* See the <a href="http://firebase.google.com/docs/vertex-ai/generate-images">personGeneration</a>
|
|
147
147
|
* documentation for more details.
|
|
148
148
|
*
|
|
149
|
-
* @
|
|
149
|
+
* @public
|
|
150
150
|
*/
|
|
151
151
|
export declare const ImagenPersonFilterLevel: {
|
|
152
152
|
/**
|
|
@@ -176,7 +176,7 @@ export declare const ImagenPersonFilterLevel: {
|
|
|
176
176
|
* See the <a href="http://firebase.google.com/docs/vertex-ai/generate-images">personGeneration</a>
|
|
177
177
|
* documentation for more details.
|
|
178
178
|
*
|
|
179
|
-
* @
|
|
179
|
+
* @public
|
|
180
180
|
*/
|
|
181
181
|
export type ImagenPersonFilterLevel = (typeof ImagenPersonFilterLevel)[keyof typeof ImagenPersonFilterLevel];
|
|
182
182
|
/**
|
|
@@ -185,7 +185,7 @@ export type ImagenPersonFilterLevel = (typeof ImagenPersonFilterLevel)[keyof typ
|
|
|
185
185
|
* See the {@link http://firebase.google.com/docs/vertex-ai/generate-images | documentation }
|
|
186
186
|
* for more details.
|
|
187
187
|
*
|
|
188
|
-
* @
|
|
188
|
+
* @public
|
|
189
189
|
*/
|
|
190
190
|
export interface ImagenSafetySettings {
|
|
191
191
|
/**
|
|
@@ -207,7 +207,7 @@ export interface ImagenSafetySettings {
|
|
|
207
207
|
* See the {@link http://firebase.google.com/docs/vertex-ai/generate-images | documentation }
|
|
208
208
|
* for more details and examples of the supported aspect ratios.
|
|
209
209
|
*
|
|
210
|
-
* @
|
|
210
|
+
* @public
|
|
211
211
|
*/
|
|
212
212
|
export declare const ImagenAspectRatio: {
|
|
213
213
|
/**
|
|
@@ -240,6 +240,6 @@ export declare const ImagenAspectRatio: {
|
|
|
240
240
|
* See the {@link http://firebase.google.com/docs/vertex-ai/generate-images | documentation }
|
|
241
241
|
* for more details and examples of the supported aspect ratios.
|
|
242
242
|
*
|
|
243
|
-
* @
|
|
243
|
+
* @public
|
|
244
244
|
*/
|
|
245
245
|
export type ImagenAspectRatio = (typeof ImagenAspectRatio)[keyof typeof ImagenAspectRatio];
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
/**
|
|
18
18
|
* An image generated by Imagen, represented as inline data.
|
|
19
19
|
*
|
|
20
|
-
* @
|
|
20
|
+
* @public
|
|
21
21
|
*/
|
|
22
22
|
export interface ImagenInlineImage {
|
|
23
23
|
/**
|
|
@@ -35,7 +35,7 @@ export interface ImagenInlineImage {
|
|
|
35
35
|
* An image generated by Imagen, stored in a Cloud Storage for Firebase bucket.
|
|
36
36
|
*
|
|
37
37
|
* This feature is not available yet.
|
|
38
|
-
* @
|
|
38
|
+
* @public
|
|
39
39
|
*/
|
|
40
40
|
export interface ImagenGCSImage {
|
|
41
41
|
/**
|
|
@@ -54,7 +54,7 @@ export interface ImagenGCSImage {
|
|
|
54
54
|
/**
|
|
55
55
|
* The response from a request to generate images with Imagen.
|
|
56
56
|
*
|
|
57
|
-
* @
|
|
57
|
+
* @public
|
|
58
58
|
*/
|
|
59
59
|
export interface ImagenGenerationResponse<T extends ImagenInlineImage | ImagenGCSImage> {
|
|
60
60
|
/**
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
* See the License for the specific language governing permissions and
|
|
15
15
|
* limitations under the License.
|
|
16
16
|
*/
|
|
17
|
-
import { Content, GenerativeContentBlob, Part } from './content';
|
|
17
|
+
import { Content, FunctionResponse, GenerativeContentBlob, Part } from './content';
|
|
18
18
|
import { LiveGenerationConfig, Tool, ToolConfig } from './requests';
|
|
19
19
|
/**
|
|
20
20
|
* User input that is sent to the model.
|
|
@@ -34,7 +34,21 @@ export interface _LiveClientContent {
|
|
|
34
34
|
*/
|
|
35
35
|
export interface _LiveClientRealtimeInput {
|
|
36
36
|
realtimeInput: {
|
|
37
|
-
|
|
37
|
+
text?: string;
|
|
38
|
+
audio?: GenerativeContentBlob;
|
|
39
|
+
video?: GenerativeContentBlob;
|
|
40
|
+
/**
|
|
41
|
+
* @deprecated Use `text`, `audio`, and `video` instead.
|
|
42
|
+
*/
|
|
43
|
+
mediaChunks?: GenerativeContentBlob[];
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Function responses that are sent to the model in real time.
|
|
48
|
+
*/
|
|
49
|
+
export interface _LiveClientToolResponse {
|
|
50
|
+
toolResponse: {
|
|
51
|
+
functionResponses: FunctionResponse[];
|
|
38
52
|
};
|
|
39
53
|
}
|
|
40
54
|
/**
|
|
@@ -218,7 +218,7 @@ export interface RequestOptions {
|
|
|
218
218
|
* Defines a tool that model can call to access external knowledge.
|
|
219
219
|
* @public
|
|
220
220
|
*/
|
|
221
|
-
export type Tool = FunctionDeclarationsTool | GoogleSearchTool | CodeExecutionTool;
|
|
221
|
+
export type Tool = FunctionDeclarationsTool | GoogleSearchTool | CodeExecutionTool | URLContextTool;
|
|
222
222
|
/**
|
|
223
223
|
* Structured representation of a function declaration as defined by the
|
|
224
224
|
* {@link https://spec.openapis.org/oas/v3.0.3 | OpenAPI 3.0 specification}.
|
|
@@ -273,7 +273,7 @@ export interface GoogleSearchTool {
|
|
|
273
273
|
/**
|
|
274
274
|
* A tool that enables the model to use code execution.
|
|
275
275
|
*
|
|
276
|
-
* @
|
|
276
|
+
* @beta
|
|
277
277
|
*/
|
|
278
278
|
export interface CodeExecutionTool {
|
|
279
279
|
/**
|
|
@@ -291,6 +291,26 @@ export interface CodeExecutionTool {
|
|
|
291
291
|
*/
|
|
292
292
|
export interface GoogleSearch {
|
|
293
293
|
}
|
|
294
|
+
/**
|
|
295
|
+
* A tool that allows you to provide additional context to the models in the form of public web
|
|
296
|
+
* URLs. By including URLs in your request, the Gemini model will access the content from those
|
|
297
|
+
* pages to inform and enhance its response.
|
|
298
|
+
*
|
|
299
|
+
* @beta
|
|
300
|
+
*/
|
|
301
|
+
export interface URLContextTool {
|
|
302
|
+
/**
|
|
303
|
+
* Specifies the URL Context configuration.
|
|
304
|
+
*/
|
|
305
|
+
urlContext: URLContext;
|
|
306
|
+
}
|
|
307
|
+
/**
|
|
308
|
+
* Specifies the URL Context configuration.
|
|
309
|
+
*
|
|
310
|
+
* @beta
|
|
311
|
+
*/
|
|
312
|
+
export interface URLContext {
|
|
313
|
+
}
|
|
294
314
|
/**
|
|
295
315
|
* A `FunctionDeclarationsTool` is a piece of code that enables the system to
|
|
296
316
|
* interact with external systems to perform an action, or set of actions,
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* limitations under the License.
|
|
16
16
|
*/
|
|
17
17
|
import { Content, FunctionCall, InlineDataPart } from './content';
|
|
18
|
-
import { BlockReason, FinishReason, HarmCategory, HarmProbability, HarmSeverity, Modality } from './enums';
|
|
18
|
+
import { BlockReason, FinishReason, HarmCategory, HarmProbability, HarmSeverity, InferenceSource, Modality } from './enums';
|
|
19
19
|
/**
|
|
20
20
|
* Result object returned from {@link GenerativeModel.generateContent} call.
|
|
21
21
|
*
|
|
@@ -76,6 +76,12 @@ export interface EnhancedGenerateContentResponse extends GenerateContentResponse
|
|
|
76
76
|
* set to `true`.
|
|
77
77
|
*/
|
|
78
78
|
thoughtSummary: () => string | undefined;
|
|
79
|
+
/**
|
|
80
|
+
* Indicates whether inference happened on-device or in-cloud.
|
|
81
|
+
*
|
|
82
|
+
* @beta
|
|
83
|
+
*/
|
|
84
|
+
inferenceSource?: InferenceSource;
|
|
79
85
|
}
|
|
80
86
|
/**
|
|
81
87
|
* Individual response from {@link GenerativeModel.generateContent} and
|
|
@@ -102,8 +108,16 @@ export interface UsageMetadata {
|
|
|
102
108
|
*/
|
|
103
109
|
thoughtsTokenCount?: number;
|
|
104
110
|
totalTokenCount: number;
|
|
111
|
+
/**
|
|
112
|
+
* The number of tokens used by tools.
|
|
113
|
+
*/
|
|
114
|
+
toolUsePromptTokenCount?: number;
|
|
105
115
|
promptTokensDetails?: ModalityTokenCount[];
|
|
106
116
|
candidatesTokensDetails?: ModalityTokenCount[];
|
|
117
|
+
/**
|
|
118
|
+
* A list of tokens used by tools, broken down by modality.
|
|
119
|
+
*/
|
|
120
|
+
toolUsePromptTokensDetails?: ModalityTokenCount[];
|
|
107
121
|
}
|
|
108
122
|
/**
|
|
109
123
|
* Represents token counting info for a single modality.
|
|
@@ -143,6 +157,7 @@ export interface GenerateContentCandidate {
|
|
|
143
157
|
safetyRatings?: SafetyRating[];
|
|
144
158
|
citationMetadata?: CitationMetadata;
|
|
145
159
|
groundingMetadata?: GroundingMetadata;
|
|
160
|
+
urlContextMetadata?: URLContextMetadata;
|
|
146
161
|
}
|
|
147
162
|
/**
|
|
148
163
|
* Citation metadata that may be found on a {@link GenerateContentCandidate}.
|
|
@@ -323,6 +338,89 @@ export interface Segment {
|
|
|
323
338
|
*/
|
|
324
339
|
text: string;
|
|
325
340
|
}
|
|
341
|
+
/**
|
|
342
|
+
* Metadata related to {@link URLContextTool}.
|
|
343
|
+
*
|
|
344
|
+
* @beta
|
|
345
|
+
*/
|
|
346
|
+
export interface URLContextMetadata {
|
|
347
|
+
/**
|
|
348
|
+
* List of URL metadata used to provide context to the Gemini model.
|
|
349
|
+
*/
|
|
350
|
+
urlMetadata: URLMetadata[];
|
|
351
|
+
}
|
|
352
|
+
/**
|
|
353
|
+
* Metadata for a single URL retrieved by the {@link URLContextTool} tool.
|
|
354
|
+
*
|
|
355
|
+
* @beta
|
|
356
|
+
*/
|
|
357
|
+
export interface URLMetadata {
|
|
358
|
+
/**
|
|
359
|
+
* The retrieved URL.
|
|
360
|
+
*/
|
|
361
|
+
retrievedUrl?: string;
|
|
362
|
+
/**
|
|
363
|
+
* The status of the URL retrieval.
|
|
364
|
+
*/
|
|
365
|
+
urlRetrievalStatus?: URLRetrievalStatus;
|
|
366
|
+
}
|
|
367
|
+
/**
|
|
368
|
+
* The status of a URL retrieval.
|
|
369
|
+
*
|
|
370
|
+
* @remarks
|
|
371
|
+
* <b>URL_RETRIEVAL_STATUS_UNSPECIFIED:</b> Unspecified retrieval status.
|
|
372
|
+
* <br/>
|
|
373
|
+
* <b>URL_RETRIEVAL_STATUS_SUCCESS:</b> The URL retrieval was successful.
|
|
374
|
+
* <br/>
|
|
375
|
+
* <b>URL_RETRIEVAL_STATUS_ERROR:</b> The URL retrieval failed.
|
|
376
|
+
* <br/>
|
|
377
|
+
* <b>URL_RETRIEVAL_STATUS_PAYWALL:</b> The URL retrieval failed because the content is behind a paywall.
|
|
378
|
+
* <br/>
|
|
379
|
+
* <b>URL_RETRIEVAL_STATUS_UNSAFE:</b> The URL retrieval failed because the content is unsafe.
|
|
380
|
+
* <br/>
|
|
381
|
+
*
|
|
382
|
+
* @beta
|
|
383
|
+
*/
|
|
384
|
+
export declare const URLRetrievalStatus: {
|
|
385
|
+
/**
|
|
386
|
+
* Unspecified retrieval status.
|
|
387
|
+
*/
|
|
388
|
+
URL_RETRIEVAL_STATUS_UNSPECIFIED: string;
|
|
389
|
+
/**
|
|
390
|
+
* The URL retrieval was successful.
|
|
391
|
+
*/
|
|
392
|
+
URL_RETRIEVAL_STATUS_SUCCESS: string;
|
|
393
|
+
/**
|
|
394
|
+
* The URL retrieval failed.
|
|
395
|
+
*/
|
|
396
|
+
URL_RETRIEVAL_STATUS_ERROR: string;
|
|
397
|
+
/**
|
|
398
|
+
* The URL retrieval failed because the content is behind a paywall.
|
|
399
|
+
*/
|
|
400
|
+
URL_RETRIEVAL_STATUS_PAYWALL: string;
|
|
401
|
+
/**
|
|
402
|
+
* The URL retrieval failed because the content is unsafe.
|
|
403
|
+
*/
|
|
404
|
+
URL_RETRIEVAL_STATUS_UNSAFE: string;
|
|
405
|
+
};
|
|
406
|
+
/**
|
|
407
|
+
* The status of a URL retrieval.
|
|
408
|
+
*
|
|
409
|
+
* @remarks
|
|
410
|
+
* <b>URL_RETRIEVAL_STATUS_UNSPECIFIED:</b> Unspecified retrieval status.
|
|
411
|
+
* <br/>
|
|
412
|
+
* <b>URL_RETRIEVAL_STATUS_SUCCESS:</b> The URL retrieval was successful.
|
|
413
|
+
* <br/>
|
|
414
|
+
* <b>URL_RETRIEVAL_STATUS_ERROR:</b> The URL retrieval failed.
|
|
415
|
+
* <br/>
|
|
416
|
+
* <b>URL_RETRIEVAL_STATUS_PAYWALL:</b> The URL retrieval failed because the content is behind a paywall.
|
|
417
|
+
* <br/>
|
|
418
|
+
* <b>URL_RETRIEVAL_STATUS_UNSAFE:</b> The URL retrieval failed because the content is unsafe.
|
|
419
|
+
* <br/>
|
|
420
|
+
*
|
|
421
|
+
* @beta
|
|
422
|
+
*/
|
|
423
|
+
export type URLRetrievalStatus = (typeof URLRetrievalStatus)[keyof typeof URLRetrievalStatus];
|
|
326
424
|
/**
|
|
327
425
|
* @public
|
|
328
426
|
*/
|