@firebase/ai 2.11.1 → 2.12.0-20260505164105
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-public.d.ts +358 -9
- package/dist/ai.d.ts +386 -12
- package/dist/esm/index.esm.js +375 -223
- package/dist/esm/index.esm.js.map +1 -1
- package/dist/esm/src/methods/live-session.d.ts +40 -5
- package/dist/esm/src/models/live-generative-model.d.ts +6 -4
- package/dist/esm/src/models/template-generative-model.d.ts +7 -3
- package/dist/esm/src/types/enums.d.ts +80 -0
- package/dist/esm/src/types/live-responses.d.ts +3 -1
- package/dist/esm/src/types/requests.d.ts +148 -2
- package/dist/esm/src/types/responses.d.ts +69 -2
- package/dist/index.cjs.js +376 -222
- package/dist/index.cjs.js.map +1 -1
- package/dist/index.node.cjs.js +376 -222
- package/dist/index.node.cjs.js.map +1 -1
- package/dist/index.node.mjs +375 -223
- package/dist/index.node.mjs.map +1 -1
- package/dist/src/methods/live-session.d.ts +40 -5
- package/dist/src/models/live-generative-model.d.ts +6 -4
- package/dist/src/models/template-generative-model.d.ts +7 -3
- package/dist/src/types/enums.d.ts +80 -0
- package/dist/src/types/live-responses.d.ts +3 -1
- package/dist/src/types/requests.d.ts +148 -2
- package/dist/src/types/responses.d.ts +69 -2
- package/package.json +9 -9
package/dist/index.node.cjs.js
CHANGED
|
@@ -8,7 +8,7 @@ var util = require('@firebase/util');
|
|
|
8
8
|
var logger$1 = require('@firebase/logger');
|
|
9
9
|
|
|
10
10
|
var name = "@firebase/ai";
|
|
11
|
-
var version = "2.
|
|
11
|
+
var version = "2.12.0-20260505164105";
|
|
12
12
|
|
|
13
13
|
/**
|
|
14
14
|
* @license
|
|
@@ -281,7 +281,78 @@ const FinishReason = {
|
|
|
281
281
|
/**
|
|
282
282
|
* The function call generated by the model was invalid.
|
|
283
283
|
*/
|
|
284
|
-
MALFORMED_FUNCTION_CALL: 'MALFORMED_FUNCTION_CALL'
|
|
284
|
+
MALFORMED_FUNCTION_CALL: 'MALFORMED_FUNCTION_CALL',
|
|
285
|
+
/**
|
|
286
|
+
* Token generation stopped because generated images contain safety violations.
|
|
287
|
+
*/
|
|
288
|
+
IMAGE_SAFETY: 'IMAGE_SAFETY',
|
|
289
|
+
/**
|
|
290
|
+
* Image generation stopped because generated images have other prohibited content.
|
|
291
|
+
*/
|
|
292
|
+
IMAGE_PROHIBITED_CONTENT: 'IMAGE_PROHIBITED_CONTENT',
|
|
293
|
+
/**
|
|
294
|
+
* Image generation stopped because of other miscellaneous issue.
|
|
295
|
+
*/
|
|
296
|
+
IMAGE_OTHER: 'IMAGE_OTHER',
|
|
297
|
+
/**
|
|
298
|
+
* The model was expected to generate an image, but none was generated.
|
|
299
|
+
*/
|
|
300
|
+
NO_IMAGE: 'NO_IMAGE',
|
|
301
|
+
/**
|
|
302
|
+
* Image generation stopped due to recitation.
|
|
303
|
+
*/
|
|
304
|
+
IMAGE_RECITATION: 'IMAGE_RECITATION',
|
|
305
|
+
/**
|
|
306
|
+
* The response candidate content was flagged for using an unsupported language.
|
|
307
|
+
*/
|
|
308
|
+
LANGUAGE: 'LANGUAGE',
|
|
309
|
+
/**
|
|
310
|
+
* Model generated a tool call but no tools were enabled in the request.
|
|
311
|
+
*/
|
|
312
|
+
UNEXPECTED_TOOL_CALL: 'UNEXPECTED_TOOL_CALL',
|
|
313
|
+
/**
|
|
314
|
+
* Model called too many tools consecutively, thus the system exited execution.
|
|
315
|
+
*/
|
|
316
|
+
TOO_MANY_TOOL_CALLS: 'TOO_MANY_TOOL_CALLS',
|
|
317
|
+
/**
|
|
318
|
+
* Request has at least one thought signature missing.
|
|
319
|
+
*/
|
|
320
|
+
MISSING_THOUGHT_SIGNATURE: 'MISSING_THOUGHT_SIGNATURE',
|
|
321
|
+
/**
|
|
322
|
+
* Finished due to malformed response.
|
|
323
|
+
*/
|
|
324
|
+
MALFORMED_RESPONSE: 'MALFORMED_RESPONSE'
|
|
325
|
+
};
|
|
326
|
+
/**
|
|
327
|
+
* Aspect ratios for generated images.
|
|
328
|
+
* @public
|
|
329
|
+
*/
|
|
330
|
+
/* eslint-disable camelcase */
|
|
331
|
+
const ImageConfigAspectRatio = {
|
|
332
|
+
SQUARE_1x1: '1:1',
|
|
333
|
+
PORTRAIT_9x16: '9:16',
|
|
334
|
+
LANDSCAPE_16x9: '16:9',
|
|
335
|
+
PORTRAIT_3x4: '3:4',
|
|
336
|
+
LANDSCAPE_4x3: '4:3',
|
|
337
|
+
PORTRAIT_2x3: '2:3',
|
|
338
|
+
LANDSCAPE_3x2: '3:2',
|
|
339
|
+
PORTRAIT_4x5: '4:5',
|
|
340
|
+
LANDSCAPE_5x4: '5:4',
|
|
341
|
+
PORTRAIT_1x4: '1:4',
|
|
342
|
+
LANDSCAPE_4x1: '4:1',
|
|
343
|
+
PORTRAIT_1x8: '1:8',
|
|
344
|
+
LANDSCAPE_8x1: '8:1',
|
|
345
|
+
ULTRAWIDE_21x9: '21:9'
|
|
346
|
+
};
|
|
347
|
+
/**
|
|
348
|
+
* Sizes for generated images. For example, '1K' is 1024px, '2K' is 2048px, and '4K' is 4096px.
|
|
349
|
+
* @public
|
|
350
|
+
*/
|
|
351
|
+
const ImageConfigImageSize = {
|
|
352
|
+
SIZE_512: '512',
|
|
353
|
+
SIZE_1K: '1K',
|
|
354
|
+
SIZE_2K: '2K',
|
|
355
|
+
SIZE_4K: '4K'
|
|
285
356
|
};
|
|
286
357
|
/**
|
|
287
358
|
* @public
|
|
@@ -491,7 +562,8 @@ const LiveResponseType = {
|
|
|
491
562
|
SERVER_CONTENT: 'serverContent',
|
|
492
563
|
TOOL_CALL: 'toolCall',
|
|
493
564
|
TOOL_CALL_CANCELLATION: 'toolCallCancellation',
|
|
494
|
-
GOING_AWAY_NOTICE: 'goingAwayNotice'
|
|
565
|
+
GOING_AWAY_NOTICE: 'goingAwayNotice',
|
|
566
|
+
SESSION_RESUMPTION_UPDATE: 'sessionResumptionUpdate'
|
|
495
567
|
};
|
|
496
568
|
|
|
497
569
|
/**
|
|
@@ -1526,7 +1598,24 @@ function getInlineDataParts(response) {
|
|
|
1526
1598
|
return undefined;
|
|
1527
1599
|
}
|
|
1528
1600
|
}
|
|
1529
|
-
const badFinishReasons = [
|
|
1601
|
+
const badFinishReasons = [
|
|
1602
|
+
FinishReason.RECITATION,
|
|
1603
|
+
FinishReason.SAFETY,
|
|
1604
|
+
FinishReason.BLOCKLIST,
|
|
1605
|
+
FinishReason.PROHIBITED_CONTENT,
|
|
1606
|
+
FinishReason.SPII,
|
|
1607
|
+
FinishReason.MALFORMED_FUNCTION_CALL,
|
|
1608
|
+
FinishReason.IMAGE_SAFETY,
|
|
1609
|
+
FinishReason.IMAGE_PROHIBITED_CONTENT,
|
|
1610
|
+
FinishReason.IMAGE_OTHER,
|
|
1611
|
+
FinishReason.NO_IMAGE,
|
|
1612
|
+
FinishReason.IMAGE_RECITATION,
|
|
1613
|
+
FinishReason.LANGUAGE,
|
|
1614
|
+
FinishReason.UNEXPECTED_TOOL_CALL,
|
|
1615
|
+
FinishReason.TOO_MANY_TOOL_CALLS,
|
|
1616
|
+
FinishReason.MISSING_THOUGHT_SIGNATURE,
|
|
1617
|
+
FinishReason.MALFORMED_RESPONSE
|
|
1618
|
+
];
|
|
1530
1619
|
function hadBadFinishReason(candidate) {
|
|
1531
1620
|
return (!!candidate.finishReason &&
|
|
1532
1621
|
badFinishReasons.some(reason => reason === candidate.finishReason));
|
|
@@ -2928,6 +3017,153 @@ function validateGenerationConfig(generationConfig) {
|
|
|
2928
3017
|
}
|
|
2929
3018
|
}
|
|
2930
3019
|
|
|
3020
|
+
/**
|
|
3021
|
+
* @license
|
|
3022
|
+
* Copyright 2025 Google LLC
|
|
3023
|
+
*
|
|
3024
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
3025
|
+
* you may not use this file except in compliance with the License.
|
|
3026
|
+
* You may obtain a copy of the License at
|
|
3027
|
+
*
|
|
3028
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
3029
|
+
*
|
|
3030
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
3031
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
3032
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
3033
|
+
* See the License for the specific language governing permissions and
|
|
3034
|
+
* limitations under the License.
|
|
3035
|
+
*/
|
|
3036
|
+
/**
|
|
3037
|
+
* A wrapper for the native `WebSocket` available in both Browsers and Node >= 22.
|
|
3038
|
+
*
|
|
3039
|
+
* @internal
|
|
3040
|
+
*/
|
|
3041
|
+
class WebSocketHandlerImpl {
|
|
3042
|
+
constructor() {
|
|
3043
|
+
if (typeof WebSocket === 'undefined') {
|
|
3044
|
+
throw new AIError(AIErrorCode.UNSUPPORTED, 'The WebSocket API is not available in this environment. ' +
|
|
3045
|
+
'The "Live" feature is not supported here. It is supported in ' +
|
|
3046
|
+
'modern browser windows, Web Workers with WebSocket support, and Node >= 22.');
|
|
3047
|
+
}
|
|
3048
|
+
}
|
|
3049
|
+
connect(url) {
|
|
3050
|
+
return new Promise((resolve, reject) => {
|
|
3051
|
+
this.ws = new WebSocket(url);
|
|
3052
|
+
this.ws.binaryType = 'blob'; // Only important to set in Node
|
|
3053
|
+
this.ws.addEventListener('open', () => resolve(), { once: true });
|
|
3054
|
+
this.ws.addEventListener('error', () => reject(new AIError(AIErrorCode.FETCH_ERROR, `Error event raised on WebSocket`)), { once: true });
|
|
3055
|
+
this.ws.addEventListener('close', (closeEvent) => {
|
|
3056
|
+
if (closeEvent.reason) {
|
|
3057
|
+
logger.warn(`WebSocket connection closed by server. Reason: '${closeEvent.reason}'`);
|
|
3058
|
+
}
|
|
3059
|
+
});
|
|
3060
|
+
});
|
|
3061
|
+
}
|
|
3062
|
+
send(data) {
|
|
3063
|
+
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
|
|
3064
|
+
throw new AIError(AIErrorCode.REQUEST_ERROR, 'WebSocket is not open.');
|
|
3065
|
+
}
|
|
3066
|
+
this.ws.send(data);
|
|
3067
|
+
}
|
|
3068
|
+
async *listen() {
|
|
3069
|
+
if (!this.ws) {
|
|
3070
|
+
throw new AIError(AIErrorCode.REQUEST_ERROR, 'WebSocket is not connected.');
|
|
3071
|
+
}
|
|
3072
|
+
const messageQueue = [];
|
|
3073
|
+
const errorQueue = [];
|
|
3074
|
+
let resolvePromise = null;
|
|
3075
|
+
let isClosed = false;
|
|
3076
|
+
const messageListener = async (event) => {
|
|
3077
|
+
let data;
|
|
3078
|
+
if (event.data instanceof Blob) {
|
|
3079
|
+
data = await event.data.text();
|
|
3080
|
+
}
|
|
3081
|
+
else if (typeof event.data === 'string') {
|
|
3082
|
+
data = event.data;
|
|
3083
|
+
}
|
|
3084
|
+
else {
|
|
3085
|
+
errorQueue.push(new AIError(AIErrorCode.PARSE_FAILED, `Failed to parse WebSocket response. Expected data to be a Blob or string, but was ${typeof event.data}.`));
|
|
3086
|
+
if (resolvePromise) {
|
|
3087
|
+
resolvePromise();
|
|
3088
|
+
resolvePromise = null;
|
|
3089
|
+
}
|
|
3090
|
+
return;
|
|
3091
|
+
}
|
|
3092
|
+
try {
|
|
3093
|
+
const obj = JSON.parse(data);
|
|
3094
|
+
messageQueue.push(obj);
|
|
3095
|
+
}
|
|
3096
|
+
catch (e) {
|
|
3097
|
+
const err = e;
|
|
3098
|
+
errorQueue.push(new AIError(AIErrorCode.PARSE_FAILED, `Error parsing WebSocket message to JSON: ${err.message}`));
|
|
3099
|
+
}
|
|
3100
|
+
if (resolvePromise) {
|
|
3101
|
+
resolvePromise();
|
|
3102
|
+
resolvePromise = null;
|
|
3103
|
+
}
|
|
3104
|
+
};
|
|
3105
|
+
const errorListener = () => {
|
|
3106
|
+
errorQueue.push(new AIError(AIErrorCode.FETCH_ERROR, 'WebSocket connection error.'));
|
|
3107
|
+
if (resolvePromise) {
|
|
3108
|
+
resolvePromise();
|
|
3109
|
+
resolvePromise = null;
|
|
3110
|
+
}
|
|
3111
|
+
};
|
|
3112
|
+
const closeListener = (event) => {
|
|
3113
|
+
if (event.reason) {
|
|
3114
|
+
logger.warn(`WebSocket connection closed by the server with reason: ${event.reason}`);
|
|
3115
|
+
}
|
|
3116
|
+
isClosed = true;
|
|
3117
|
+
if (resolvePromise) {
|
|
3118
|
+
resolvePromise();
|
|
3119
|
+
resolvePromise = null;
|
|
3120
|
+
}
|
|
3121
|
+
// Clean up listeners to prevent memory leaks
|
|
3122
|
+
this.ws?.removeEventListener('message', messageListener);
|
|
3123
|
+
this.ws?.removeEventListener('close', closeListener);
|
|
3124
|
+
this.ws?.removeEventListener('error', errorListener);
|
|
3125
|
+
};
|
|
3126
|
+
this.ws.addEventListener('message', messageListener);
|
|
3127
|
+
this.ws.addEventListener('close', closeListener);
|
|
3128
|
+
this.ws.addEventListener('error', errorListener);
|
|
3129
|
+
while (!isClosed) {
|
|
3130
|
+
if (errorQueue.length > 0) {
|
|
3131
|
+
const error = errorQueue.shift();
|
|
3132
|
+
throw error;
|
|
3133
|
+
}
|
|
3134
|
+
if (messageQueue.length > 0) {
|
|
3135
|
+
yield messageQueue.shift();
|
|
3136
|
+
}
|
|
3137
|
+
else {
|
|
3138
|
+
await new Promise(resolve => {
|
|
3139
|
+
resolvePromise = resolve;
|
|
3140
|
+
});
|
|
3141
|
+
}
|
|
3142
|
+
}
|
|
3143
|
+
// If the loop terminated because isClosed is true, check for any final errors
|
|
3144
|
+
if (errorQueue.length > 0) {
|
|
3145
|
+
const error = errorQueue.shift();
|
|
3146
|
+
throw error;
|
|
3147
|
+
}
|
|
3148
|
+
}
|
|
3149
|
+
close(code, reason) {
|
|
3150
|
+
return new Promise(resolve => {
|
|
3151
|
+
if (!this.ws) {
|
|
3152
|
+
return resolve();
|
|
3153
|
+
}
|
|
3154
|
+
this.ws.addEventListener('close', () => resolve(), { once: true });
|
|
3155
|
+
// Calling 'close' during these states results in an error.
|
|
3156
|
+
if (this.ws.readyState === WebSocket.CLOSED ||
|
|
3157
|
+
this.ws.readyState === WebSocket.CONNECTING) {
|
|
3158
|
+
return resolve();
|
|
3159
|
+
}
|
|
3160
|
+
if (this.ws.readyState !== WebSocket.CLOSING) {
|
|
3161
|
+
this.ws.close(code, reason);
|
|
3162
|
+
}
|
|
3163
|
+
});
|
|
3164
|
+
}
|
|
3165
|
+
}
|
|
3166
|
+
|
|
2931
3167
|
/**
|
|
2932
3168
|
* @license
|
|
2933
3169
|
* Copyright 2025 Google LLC
|
|
@@ -2955,9 +3191,10 @@ class LiveSession {
|
|
|
2955
3191
|
/**
|
|
2956
3192
|
* @internal
|
|
2957
3193
|
*/
|
|
2958
|
-
constructor(
|
|
2959
|
-
this.
|
|
2960
|
-
this.
|
|
3194
|
+
constructor(_setupMessage, _apiSettings, _sessionResumption, webSocketHandler) {
|
|
3195
|
+
this._setupMessage = _setupMessage;
|
|
3196
|
+
this._apiSettings = _apiSettings;
|
|
3197
|
+
this._sessionResumption = _sessionResumption;
|
|
2961
3198
|
/**
|
|
2962
3199
|
* Indicates whether this Live session is closed.
|
|
2963
3200
|
*
|
|
@@ -2970,6 +3207,64 @@ class LiveSession {
|
|
|
2970
3207
|
* @beta
|
|
2971
3208
|
*/
|
|
2972
3209
|
this.inConversation = false;
|
|
3210
|
+
/**
|
|
3211
|
+
* Generator yielding WebSocket messages from the server.
|
|
3212
|
+
*/
|
|
3213
|
+
this._serverMessages = null;
|
|
3214
|
+
this._webSocketHandler = webSocketHandler || new WebSocketHandlerImpl();
|
|
3215
|
+
this.connectionPromise = this._connectSession(this._sessionResumption);
|
|
3216
|
+
}
|
|
3217
|
+
/**
|
|
3218
|
+
* Initializes connection to the WebSocket. Should be called immediately
|
|
3219
|
+
* after instantiation.
|
|
3220
|
+
*
|
|
3221
|
+
* @internal
|
|
3222
|
+
*/
|
|
3223
|
+
async _connectSession(sessionResumption) {
|
|
3224
|
+
const url = new WebSocketUrl(this._apiSettings);
|
|
3225
|
+
await this._webSocketHandler.connect(url.toString());
|
|
3226
|
+
try {
|
|
3227
|
+
// Begin listening for server messages, and begin the handshake by sending the 'setupMessage'
|
|
3228
|
+
this._serverMessages = this._webSocketHandler.listen();
|
|
3229
|
+
const setupMessage = { ...this._setupMessage };
|
|
3230
|
+
if (sessionResumption) {
|
|
3231
|
+
setupMessage.setup.sessionResumption = sessionResumption;
|
|
3232
|
+
}
|
|
3233
|
+
this._webSocketHandler.send(JSON.stringify(setupMessage));
|
|
3234
|
+
// Verify we received the handshake response 'setupComplete'
|
|
3235
|
+
const firstMessage = (await this._serverMessages.next()).value;
|
|
3236
|
+
if (!firstMessage ||
|
|
3237
|
+
!(typeof firstMessage === 'object') ||
|
|
3238
|
+
!('setupComplete' in firstMessage)) {
|
|
3239
|
+
await this._webSocketHandler.close(1011, 'Handshake failure');
|
|
3240
|
+
throw new AIError(AIErrorCode.RESPONSE_ERROR, 'Server connection handshake failed. The server did not respond with a setupComplete message.');
|
|
3241
|
+
}
|
|
3242
|
+
this.isClosed = false;
|
|
3243
|
+
}
|
|
3244
|
+
catch (e) {
|
|
3245
|
+
// Ensure connection is closed on any setup error
|
|
3246
|
+
await this._webSocketHandler.close();
|
|
3247
|
+
throw e;
|
|
3248
|
+
}
|
|
3249
|
+
}
|
|
3250
|
+
/**
|
|
3251
|
+
* Resumes an existing live session with the server.
|
|
3252
|
+
*
|
|
3253
|
+
* This closes the current WebSocket connection and establishes a new one using
|
|
3254
|
+
* the same configuration (URI, headers, model, system instruction, tools, etc.)
|
|
3255
|
+
* as the original session.
|
|
3256
|
+
*
|
|
3257
|
+
* @param sessionResumption - The configuration for session resumption, such as the handle to the previous session state to restore.
|
|
3258
|
+
* @throws If the session resumption configuration is unsupported.
|
|
3259
|
+
*
|
|
3260
|
+
* @beta
|
|
3261
|
+
*/
|
|
3262
|
+
async resumeSession(sessionResumption) {
|
|
3263
|
+
if (!this._sessionResumption) {
|
|
3264
|
+
throw new AIError(AIErrorCode.UNSUPPORTED, 'Cannot resume session: no sessionResumption config provided');
|
|
3265
|
+
}
|
|
3266
|
+
await this.close();
|
|
3267
|
+
await this._connectSession(sessionResumption);
|
|
2973
3268
|
}
|
|
2974
3269
|
/**
|
|
2975
3270
|
* Sends content to the server.
|
|
@@ -2991,7 +3286,7 @@ class LiveSession {
|
|
|
2991
3286
|
turnComplete
|
|
2992
3287
|
}
|
|
2993
3288
|
};
|
|
2994
|
-
this.
|
|
3289
|
+
this._webSocketHandler.send(JSON.stringify(message));
|
|
2995
3290
|
}
|
|
2996
3291
|
/**
|
|
2997
3292
|
* Sends text to the server in realtime.
|
|
@@ -3015,7 +3310,7 @@ class LiveSession {
|
|
|
3015
3310
|
text
|
|
3016
3311
|
}
|
|
3017
3312
|
};
|
|
3018
|
-
this.
|
|
3313
|
+
this._webSocketHandler.send(JSON.stringify(message));
|
|
3019
3314
|
}
|
|
3020
3315
|
/**
|
|
3021
3316
|
* Sends audio data to the server in realtime.
|
|
@@ -3044,7 +3339,7 @@ class LiveSession {
|
|
|
3044
3339
|
audio: blob
|
|
3045
3340
|
}
|
|
3046
3341
|
};
|
|
3047
|
-
this.
|
|
3342
|
+
this._webSocketHandler.send(JSON.stringify(message));
|
|
3048
3343
|
}
|
|
3049
3344
|
/**
|
|
3050
3345
|
* Sends video data to the server in realtime.
|
|
@@ -3072,7 +3367,7 @@ class LiveSession {
|
|
|
3072
3367
|
video: blob
|
|
3073
3368
|
}
|
|
3074
3369
|
};
|
|
3075
|
-
this.
|
|
3370
|
+
this._webSocketHandler.send(JSON.stringify(message));
|
|
3076
3371
|
}
|
|
3077
3372
|
/**
|
|
3078
3373
|
* Sends function responses to the server.
|
|
@@ -3091,7 +3386,7 @@ class LiveSession {
|
|
|
3091
3386
|
functionResponses
|
|
3092
3387
|
}
|
|
3093
3388
|
};
|
|
3094
|
-
this.
|
|
3389
|
+
this._webSocketHandler.send(JSON.stringify(message));
|
|
3095
3390
|
}
|
|
3096
3391
|
/**
|
|
3097
3392
|
* Yields messages received from the server.
|
|
@@ -3106,42 +3401,50 @@ class LiveSession {
|
|
|
3106
3401
|
if (this.isClosed) {
|
|
3107
3402
|
throw new AIError(AIErrorCode.SESSION_CLOSED, 'Cannot read from a Live session that is closed. Try starting a new Live session.');
|
|
3108
3403
|
}
|
|
3109
|
-
|
|
3110
|
-
|
|
3111
|
-
if (
|
|
3112
|
-
|
|
3113
|
-
|
|
3114
|
-
|
|
3115
|
-
|
|
3116
|
-
|
|
3117
|
-
|
|
3118
|
-
|
|
3119
|
-
|
|
3120
|
-
|
|
3121
|
-
|
|
3122
|
-
|
|
3123
|
-
|
|
3124
|
-
|
|
3125
|
-
|
|
3126
|
-
|
|
3127
|
-
|
|
3128
|
-
|
|
3129
|
-
|
|
3130
|
-
|
|
3131
|
-
|
|
3132
|
-
|
|
3133
|
-
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
|
|
3404
|
+
if (this._serverMessages) {
|
|
3405
|
+
for await (const message of this._serverMessages) {
|
|
3406
|
+
if (message && typeof message === 'object') {
|
|
3407
|
+
if (LiveResponseType.SERVER_CONTENT in message) {
|
|
3408
|
+
yield {
|
|
3409
|
+
type: 'serverContent',
|
|
3410
|
+
...message
|
|
3411
|
+
.serverContent
|
|
3412
|
+
};
|
|
3413
|
+
}
|
|
3414
|
+
else if (LiveResponseType.TOOL_CALL in message) {
|
|
3415
|
+
yield {
|
|
3416
|
+
type: 'toolCall',
|
|
3417
|
+
...message
|
|
3418
|
+
.toolCall
|
|
3419
|
+
};
|
|
3420
|
+
}
|
|
3421
|
+
else if (LiveResponseType.TOOL_CALL_CANCELLATION in message) {
|
|
3422
|
+
yield {
|
|
3423
|
+
type: 'toolCallCancellation',
|
|
3424
|
+
...message.toolCallCancellation
|
|
3425
|
+
};
|
|
3426
|
+
}
|
|
3427
|
+
else if ('goAway' in message) {
|
|
3428
|
+
const notice = message.goAway;
|
|
3429
|
+
yield {
|
|
3430
|
+
type: LiveResponseType.GOING_AWAY_NOTICE,
|
|
3431
|
+
timeLeft: parseDuration(notice.timeLeft)
|
|
3432
|
+
};
|
|
3433
|
+
}
|
|
3434
|
+
else if (LiveResponseType.SESSION_RESUMPTION_UPDATE in message) {
|
|
3435
|
+
yield {
|
|
3436
|
+
type: LiveResponseType.SESSION_RESUMPTION_UPDATE,
|
|
3437
|
+
...message.sessionResumptionUpdate
|
|
3438
|
+
};
|
|
3439
|
+
}
|
|
3440
|
+
else {
|
|
3441
|
+
logger.warn(`Received an unknown message type from the server: ${JSON.stringify(message)}`);
|
|
3442
|
+
}
|
|
3137
3443
|
}
|
|
3138
3444
|
else {
|
|
3139
|
-
logger.warn(`Received an
|
|
3445
|
+
logger.warn(`Received an invalid message from the server: ${JSON.stringify(message)}`);
|
|
3140
3446
|
}
|
|
3141
3447
|
}
|
|
3142
|
-
else {
|
|
3143
|
-
logger.warn(`Received an invalid message from the server: ${JSON.stringify(message)}`);
|
|
3144
|
-
}
|
|
3145
3448
|
}
|
|
3146
3449
|
}
|
|
3147
3450
|
/**
|
|
@@ -3153,7 +3456,7 @@ class LiveSession {
|
|
|
3153
3456
|
async close() {
|
|
3154
3457
|
if (!this.isClosed) {
|
|
3155
3458
|
this.isClosed = true;
|
|
3156
|
-
await this.
|
|
3459
|
+
await this._webSocketHandler.close(1000, 'Client closed session.');
|
|
3157
3460
|
}
|
|
3158
3461
|
}
|
|
3159
3462
|
/**
|
|
@@ -3176,7 +3479,7 @@ class LiveSession {
|
|
|
3176
3479
|
const message = {
|
|
3177
3480
|
realtimeInput: { mediaChunks: [mediaChunk] }
|
|
3178
3481
|
};
|
|
3179
|
-
this.
|
|
3482
|
+
this._webSocketHandler.send(JSON.stringify(message));
|
|
3180
3483
|
});
|
|
3181
3484
|
}
|
|
3182
3485
|
/**
|
|
@@ -3256,6 +3559,7 @@ class LiveGenerativeModel extends AIModel {
|
|
|
3256
3559
|
*/
|
|
3257
3560
|
constructor(ai, modelParams,
|
|
3258
3561
|
/**
|
|
3562
|
+
* For testing injection
|
|
3259
3563
|
* @internal
|
|
3260
3564
|
*/
|
|
3261
3565
|
_webSocketHandler) {
|
|
@@ -3274,9 +3578,7 @@ class LiveGenerativeModel extends AIModel {
|
|
|
3274
3578
|
*
|
|
3275
3579
|
* @beta
|
|
3276
3580
|
*/
|
|
3277
|
-
async connect() {
|
|
3278
|
-
const url = new WebSocketUrl(this._apiSettings);
|
|
3279
|
-
await this._webSocketHandler.connect(url.toString());
|
|
3581
|
+
async connect(sessionResumption) {
|
|
3280
3582
|
let fullModelPath;
|
|
3281
3583
|
if (this._apiSettings.backend.backendType === BackendType.GOOGLE_AI) {
|
|
3282
3584
|
fullModelPath = `projects/${this._apiSettings.project}/${this.model}`;
|
|
@@ -3287,36 +3589,24 @@ class LiveGenerativeModel extends AIModel {
|
|
|
3287
3589
|
// inputAudioTranscription and outputAudioTranscription are on the generation config in the public API,
|
|
3288
3590
|
// but the backend expects them to be in the `setup` message.
|
|
3289
3591
|
const { inputAudioTranscription, outputAudioTranscription, ...generationConfig } = this.generationConfig;
|
|
3592
|
+
const contextWindowCompression = generationConfig.contextWindowCompression;
|
|
3593
|
+
delete generationConfig.contextWindowCompression;
|
|
3290
3594
|
const setupMessage = {
|
|
3291
3595
|
setup: {
|
|
3292
3596
|
model: fullModelPath,
|
|
3293
3597
|
generationConfig,
|
|
3598
|
+
contextWindowCompression,
|
|
3294
3599
|
tools: this.tools,
|
|
3295
3600
|
toolConfig: this.toolConfig,
|
|
3296
3601
|
systemInstruction: this.systemInstruction,
|
|
3297
3602
|
inputAudioTranscription,
|
|
3298
|
-
outputAudioTranscription
|
|
3603
|
+
outputAudioTranscription,
|
|
3604
|
+
sessionResumption
|
|
3299
3605
|
}
|
|
3300
3606
|
};
|
|
3301
|
-
|
|
3302
|
-
|
|
3303
|
-
|
|
3304
|
-
this._webSocketHandler.send(JSON.stringify(setupMessage));
|
|
3305
|
-
// Verify we received the handshake response 'setupComplete'
|
|
3306
|
-
const firstMessage = (await serverMessages.next()).value;
|
|
3307
|
-
if (!firstMessage ||
|
|
3308
|
-
!(typeof firstMessage === 'object') ||
|
|
3309
|
-
!('setupComplete' in firstMessage)) {
|
|
3310
|
-
await this._webSocketHandler.close(1011, 'Handshake failure');
|
|
3311
|
-
throw new AIError(AIErrorCode.RESPONSE_ERROR, 'Server connection handshake failed. The server did not respond with a setupComplete message.');
|
|
3312
|
-
}
|
|
3313
|
-
return new LiveSession(this._webSocketHandler, serverMessages);
|
|
3314
|
-
}
|
|
3315
|
-
catch (e) {
|
|
3316
|
-
// Ensure connection is closed on any setup error
|
|
3317
|
-
await this._webSocketHandler.close();
|
|
3318
|
-
throw e;
|
|
3319
|
-
}
|
|
3607
|
+
const session = new LiveSession(setupMessage, this._apiSettings, sessionResumption, this._webSocketHandler);
|
|
3608
|
+
await session.connectionPromise;
|
|
3609
|
+
return session;
|
|
3320
3610
|
}
|
|
3321
3611
|
}
|
|
3322
3612
|
|
|
@@ -3457,153 +3747,6 @@ class ImagenModel extends AIModel {
|
|
|
3457
3747
|
}
|
|
3458
3748
|
}
|
|
3459
3749
|
|
|
3460
|
-
/**
|
|
3461
|
-
* @license
|
|
3462
|
-
* Copyright 2025 Google LLC
|
|
3463
|
-
*
|
|
3464
|
-
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
3465
|
-
* you may not use this file except in compliance with the License.
|
|
3466
|
-
* You may obtain a copy of the License at
|
|
3467
|
-
*
|
|
3468
|
-
* http://www.apache.org/licenses/LICENSE-2.0
|
|
3469
|
-
*
|
|
3470
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
3471
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
3472
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
3473
|
-
* See the License for the specific language governing permissions and
|
|
3474
|
-
* limitations under the License.
|
|
3475
|
-
*/
|
|
3476
|
-
/**
|
|
3477
|
-
* A wrapper for the native `WebSocket` available in both Browsers and Node >= 22.
|
|
3478
|
-
*
|
|
3479
|
-
* @internal
|
|
3480
|
-
*/
|
|
3481
|
-
class WebSocketHandlerImpl {
|
|
3482
|
-
constructor() {
|
|
3483
|
-
if (typeof WebSocket === 'undefined') {
|
|
3484
|
-
throw new AIError(AIErrorCode.UNSUPPORTED, 'The WebSocket API is not available in this environment. ' +
|
|
3485
|
-
'The "Live" feature is not supported here. It is supported in ' +
|
|
3486
|
-
'modern browser windows, Web Workers with WebSocket support, and Node >= 22.');
|
|
3487
|
-
}
|
|
3488
|
-
}
|
|
3489
|
-
connect(url) {
|
|
3490
|
-
return new Promise((resolve, reject) => {
|
|
3491
|
-
this.ws = new WebSocket(url);
|
|
3492
|
-
this.ws.binaryType = 'blob'; // Only important to set in Node
|
|
3493
|
-
this.ws.addEventListener('open', () => resolve(), { once: true });
|
|
3494
|
-
this.ws.addEventListener('error', () => reject(new AIError(AIErrorCode.FETCH_ERROR, `Error event raised on WebSocket`)), { once: true });
|
|
3495
|
-
this.ws.addEventListener('close', (closeEvent) => {
|
|
3496
|
-
if (closeEvent.reason) {
|
|
3497
|
-
logger.warn(`WebSocket connection closed by server. Reason: '${closeEvent.reason}'`);
|
|
3498
|
-
}
|
|
3499
|
-
});
|
|
3500
|
-
});
|
|
3501
|
-
}
|
|
3502
|
-
send(data) {
|
|
3503
|
-
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
|
|
3504
|
-
throw new AIError(AIErrorCode.REQUEST_ERROR, 'WebSocket is not open.');
|
|
3505
|
-
}
|
|
3506
|
-
this.ws.send(data);
|
|
3507
|
-
}
|
|
3508
|
-
async *listen() {
|
|
3509
|
-
if (!this.ws) {
|
|
3510
|
-
throw new AIError(AIErrorCode.REQUEST_ERROR, 'WebSocket is not connected.');
|
|
3511
|
-
}
|
|
3512
|
-
const messageQueue = [];
|
|
3513
|
-
const errorQueue = [];
|
|
3514
|
-
let resolvePromise = null;
|
|
3515
|
-
let isClosed = false;
|
|
3516
|
-
const messageListener = async (event) => {
|
|
3517
|
-
let data;
|
|
3518
|
-
if (event.data instanceof Blob) {
|
|
3519
|
-
data = await event.data.text();
|
|
3520
|
-
}
|
|
3521
|
-
else if (typeof event.data === 'string') {
|
|
3522
|
-
data = event.data;
|
|
3523
|
-
}
|
|
3524
|
-
else {
|
|
3525
|
-
errorQueue.push(new AIError(AIErrorCode.PARSE_FAILED, `Failed to parse WebSocket response. Expected data to be a Blob or string, but was ${typeof event.data}.`));
|
|
3526
|
-
if (resolvePromise) {
|
|
3527
|
-
resolvePromise();
|
|
3528
|
-
resolvePromise = null;
|
|
3529
|
-
}
|
|
3530
|
-
return;
|
|
3531
|
-
}
|
|
3532
|
-
try {
|
|
3533
|
-
const obj = JSON.parse(data);
|
|
3534
|
-
messageQueue.push(obj);
|
|
3535
|
-
}
|
|
3536
|
-
catch (e) {
|
|
3537
|
-
const err = e;
|
|
3538
|
-
errorQueue.push(new AIError(AIErrorCode.PARSE_FAILED, `Error parsing WebSocket message to JSON: ${err.message}`));
|
|
3539
|
-
}
|
|
3540
|
-
if (resolvePromise) {
|
|
3541
|
-
resolvePromise();
|
|
3542
|
-
resolvePromise = null;
|
|
3543
|
-
}
|
|
3544
|
-
};
|
|
3545
|
-
const errorListener = () => {
|
|
3546
|
-
errorQueue.push(new AIError(AIErrorCode.FETCH_ERROR, 'WebSocket connection error.'));
|
|
3547
|
-
if (resolvePromise) {
|
|
3548
|
-
resolvePromise();
|
|
3549
|
-
resolvePromise = null;
|
|
3550
|
-
}
|
|
3551
|
-
};
|
|
3552
|
-
const closeListener = (event) => {
|
|
3553
|
-
if (event.reason) {
|
|
3554
|
-
logger.warn(`WebSocket connection closed by the server with reason: ${event.reason}`);
|
|
3555
|
-
}
|
|
3556
|
-
isClosed = true;
|
|
3557
|
-
if (resolvePromise) {
|
|
3558
|
-
resolvePromise();
|
|
3559
|
-
resolvePromise = null;
|
|
3560
|
-
}
|
|
3561
|
-
// Clean up listeners to prevent memory leaks
|
|
3562
|
-
this.ws?.removeEventListener('message', messageListener);
|
|
3563
|
-
this.ws?.removeEventListener('close', closeListener);
|
|
3564
|
-
this.ws?.removeEventListener('error', errorListener);
|
|
3565
|
-
};
|
|
3566
|
-
this.ws.addEventListener('message', messageListener);
|
|
3567
|
-
this.ws.addEventListener('close', closeListener);
|
|
3568
|
-
this.ws.addEventListener('error', errorListener);
|
|
3569
|
-
while (!isClosed) {
|
|
3570
|
-
if (errorQueue.length > 0) {
|
|
3571
|
-
const error = errorQueue.shift();
|
|
3572
|
-
throw error;
|
|
3573
|
-
}
|
|
3574
|
-
if (messageQueue.length > 0) {
|
|
3575
|
-
yield messageQueue.shift();
|
|
3576
|
-
}
|
|
3577
|
-
else {
|
|
3578
|
-
await new Promise(resolve => {
|
|
3579
|
-
resolvePromise = resolve;
|
|
3580
|
-
});
|
|
3581
|
-
}
|
|
3582
|
-
}
|
|
3583
|
-
// If the loop terminated because isClosed is true, check for any final errors
|
|
3584
|
-
if (errorQueue.length > 0) {
|
|
3585
|
-
const error = errorQueue.shift();
|
|
3586
|
-
throw error;
|
|
3587
|
-
}
|
|
3588
|
-
}
|
|
3589
|
-
close(code, reason) {
|
|
3590
|
-
return new Promise(resolve => {
|
|
3591
|
-
if (!this.ws) {
|
|
3592
|
-
return resolve();
|
|
3593
|
-
}
|
|
3594
|
-
this.ws.addEventListener('close', () => resolve(), { once: true });
|
|
3595
|
-
// Calling 'close' during these states results in an error.
|
|
3596
|
-
if (this.ws.readyState === WebSocket.CLOSED ||
|
|
3597
|
-
this.ws.readyState === WebSocket.CONNECTING) {
|
|
3598
|
-
return resolve();
|
|
3599
|
-
}
|
|
3600
|
-
if (this.ws.readyState !== WebSocket.CLOSING) {
|
|
3601
|
-
this.ws.close(code, reason);
|
|
3602
|
-
}
|
|
3603
|
-
});
|
|
3604
|
-
}
|
|
3605
|
-
}
|
|
3606
|
-
|
|
3607
3750
|
/**
|
|
3608
3751
|
* @license
|
|
3609
3752
|
* Copyright 2026 Google LLC
|
|
@@ -3748,11 +3891,16 @@ class TemplateGenerativeModel {
|
|
|
3748
3891
|
* @param templateId - The ID of the server-side template to execute.
|
|
3749
3892
|
* @param templateVariables - A key-value map of variables to populate the
|
|
3750
3893
|
* template with.
|
|
3894
|
+
* @param singleRequestOptions - Optional. Options to use for this request.
|
|
3895
|
+
* @param templateToolConfig - Optional. Configuration for tools to use with this request.
|
|
3751
3896
|
*
|
|
3752
3897
|
* @beta
|
|
3753
3898
|
*/
|
|
3754
|
-
async generateContent(templateId, templateVariables, singleRequestOptions) {
|
|
3755
|
-
return templateGenerateContent(this._apiSettings, templateId, {
|
|
3899
|
+
async generateContent(templateId, templateVariables, singleRequestOptions, templateToolConfig) {
|
|
3900
|
+
return templateGenerateContent(this._apiSettings, templateId, {
|
|
3901
|
+
inputs: templateVariables,
|
|
3902
|
+
...(templateToolConfig && { toolConfig: templateToolConfig })
|
|
3903
|
+
}, {
|
|
3756
3904
|
...this.requestOptions,
|
|
3757
3905
|
...singleRequestOptions
|
|
3758
3906
|
});
|
|
@@ -3766,11 +3914,16 @@ class TemplateGenerativeModel {
|
|
|
3766
3914
|
* @param templateId - The ID of the server-side template to execute.
|
|
3767
3915
|
* @param templateVariables - A key-value map of variables to populate the
|
|
3768
3916
|
* template with.
|
|
3917
|
+
* @param singleRequestOptions - Optional.Options to use for this request.
|
|
3918
|
+
* @param templateToolConfig - Optional. Configuration for tools to use with this request.
|
|
3769
3919
|
*
|
|
3770
3920
|
* @beta
|
|
3771
3921
|
*/
|
|
3772
|
-
async generateContentStream(templateId, templateVariables, singleRequestOptions) {
|
|
3773
|
-
return templateGenerateContentStream(this._apiSettings, templateId, {
|
|
3922
|
+
async generateContentStream(templateId, templateVariables, singleRequestOptions, templateToolConfig) {
|
|
3923
|
+
return templateGenerateContentStream(this._apiSettings, templateId, {
|
|
3924
|
+
inputs: templateVariables,
|
|
3925
|
+
...(templateToolConfig && { toolConfig: templateToolConfig })
|
|
3926
|
+
}, {
|
|
3774
3927
|
...this.requestOptions,
|
|
3775
3928
|
...singleRequestOptions
|
|
3776
3929
|
});
|
|
@@ -4663,8 +4816,7 @@ function getLiveGenerativeModel(ai, modelParams) {
|
|
|
4663
4816
|
if (!modelParams.model) {
|
|
4664
4817
|
throw new AIError(AIErrorCode.NO_MODEL, `Must provide a model name for getLiveGenerativeModel. Example: getLiveGenerativeModel(ai, { model: 'my-model-name' })`);
|
|
4665
4818
|
}
|
|
4666
|
-
|
|
4667
|
-
return new LiveGenerativeModel(ai, modelParams, webSocketHandler);
|
|
4819
|
+
return new LiveGenerativeModel(ai, modelParams);
|
|
4668
4820
|
}
|
|
4669
4821
|
/**
|
|
4670
4822
|
* Returns a {@link TemplateGenerativeModel} class for executing server-side
|
|
@@ -4727,6 +4879,8 @@ exports.HarmBlockThreshold = HarmBlockThreshold;
|
|
|
4727
4879
|
exports.HarmCategory = HarmCategory;
|
|
4728
4880
|
exports.HarmProbability = HarmProbability;
|
|
4729
4881
|
exports.HarmSeverity = HarmSeverity;
|
|
4882
|
+
exports.ImageConfigAspectRatio = ImageConfigAspectRatio;
|
|
4883
|
+
exports.ImageConfigImageSize = ImageConfigImageSize;
|
|
4730
4884
|
exports.ImagenAspectRatio = ImagenAspectRatio;
|
|
4731
4885
|
exports.ImagenImageFormat = ImagenImageFormat;
|
|
4732
4886
|
exports.ImagenModel = ImagenModel;
|