@firebase/ai 2.11.1 → 2.12.0-20260505164105
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-public.d.ts +358 -9
- package/dist/ai.d.ts +386 -12
- package/dist/esm/index.esm.js +375 -223
- package/dist/esm/index.esm.js.map +1 -1
- package/dist/esm/src/methods/live-session.d.ts +40 -5
- package/dist/esm/src/models/live-generative-model.d.ts +6 -4
- package/dist/esm/src/models/template-generative-model.d.ts +7 -3
- package/dist/esm/src/types/enums.d.ts +80 -0
- package/dist/esm/src/types/live-responses.d.ts +3 -1
- package/dist/esm/src/types/requests.d.ts +148 -2
- package/dist/esm/src/types/responses.d.ts +69 -2
- package/dist/index.cjs.js +376 -222
- package/dist/index.cjs.js.map +1 -1
- package/dist/index.node.cjs.js +376 -222
- package/dist/index.node.cjs.js.map +1 -1
- package/dist/index.node.mjs +375 -223
- package/dist/index.node.mjs.map +1 -1
- package/dist/src/methods/live-session.d.ts +40 -5
- package/dist/src/models/live-generative-model.d.ts +6 -4
- package/dist/src/models/template-generative-model.d.ts +7 -3
- package/dist/src/types/enums.d.ts +80 -0
- package/dist/src/types/live-responses.d.ts +3 -1
- package/dist/src/types/requests.d.ts +148 -2
- package/dist/src/types/responses.d.ts +69 -2
- package/package.json +9 -9
package/dist/index.cjs.js
CHANGED
|
@@ -8,7 +8,7 @@ var util = require('@firebase/util');
|
|
|
8
8
|
var logger$1 = require('@firebase/logger');
|
|
9
9
|
|
|
10
10
|
var name = "@firebase/ai";
|
|
11
|
-
var version = "2.
|
|
11
|
+
var version = "2.12.0-20260505164105";
|
|
12
12
|
|
|
13
13
|
/**
|
|
14
14
|
* @license
|
|
@@ -281,7 +281,78 @@ const FinishReason = {
|
|
|
281
281
|
/**
|
|
282
282
|
* The function call generated by the model was invalid.
|
|
283
283
|
*/
|
|
284
|
-
MALFORMED_FUNCTION_CALL: 'MALFORMED_FUNCTION_CALL'
|
|
284
|
+
MALFORMED_FUNCTION_CALL: 'MALFORMED_FUNCTION_CALL',
|
|
285
|
+
/**
|
|
286
|
+
* Token generation stopped because generated images contain safety violations.
|
|
287
|
+
*/
|
|
288
|
+
IMAGE_SAFETY: 'IMAGE_SAFETY',
|
|
289
|
+
/**
|
|
290
|
+
* Image generation stopped because generated images have other prohibited content.
|
|
291
|
+
*/
|
|
292
|
+
IMAGE_PROHIBITED_CONTENT: 'IMAGE_PROHIBITED_CONTENT',
|
|
293
|
+
/**
|
|
294
|
+
* Image generation stopped because of other miscellaneous issue.
|
|
295
|
+
*/
|
|
296
|
+
IMAGE_OTHER: 'IMAGE_OTHER',
|
|
297
|
+
/**
|
|
298
|
+
* The model was expected to generate an image, but none was generated.
|
|
299
|
+
*/
|
|
300
|
+
NO_IMAGE: 'NO_IMAGE',
|
|
301
|
+
/**
|
|
302
|
+
* Image generation stopped due to recitation.
|
|
303
|
+
*/
|
|
304
|
+
IMAGE_RECITATION: 'IMAGE_RECITATION',
|
|
305
|
+
/**
|
|
306
|
+
* The response candidate content was flagged for using an unsupported language.
|
|
307
|
+
*/
|
|
308
|
+
LANGUAGE: 'LANGUAGE',
|
|
309
|
+
/**
|
|
310
|
+
* Model generated a tool call but no tools were enabled in the request.
|
|
311
|
+
*/
|
|
312
|
+
UNEXPECTED_TOOL_CALL: 'UNEXPECTED_TOOL_CALL',
|
|
313
|
+
/**
|
|
314
|
+
* Model called too many tools consecutively, thus the system exited execution.
|
|
315
|
+
*/
|
|
316
|
+
TOO_MANY_TOOL_CALLS: 'TOO_MANY_TOOL_CALLS',
|
|
317
|
+
/**
|
|
318
|
+
* Request has at least one thought signature missing.
|
|
319
|
+
*/
|
|
320
|
+
MISSING_THOUGHT_SIGNATURE: 'MISSING_THOUGHT_SIGNATURE',
|
|
321
|
+
/**
|
|
322
|
+
* Finished due to malformed response.
|
|
323
|
+
*/
|
|
324
|
+
MALFORMED_RESPONSE: 'MALFORMED_RESPONSE'
|
|
325
|
+
};
|
|
326
|
+
/**
|
|
327
|
+
* Aspect ratios for generated images.
|
|
328
|
+
* @public
|
|
329
|
+
*/
|
|
330
|
+
/* eslint-disable camelcase */
|
|
331
|
+
const ImageConfigAspectRatio = {
|
|
332
|
+
SQUARE_1x1: '1:1',
|
|
333
|
+
PORTRAIT_9x16: '9:16',
|
|
334
|
+
LANDSCAPE_16x9: '16:9',
|
|
335
|
+
PORTRAIT_3x4: '3:4',
|
|
336
|
+
LANDSCAPE_4x3: '4:3',
|
|
337
|
+
PORTRAIT_2x3: '2:3',
|
|
338
|
+
LANDSCAPE_3x2: '3:2',
|
|
339
|
+
PORTRAIT_4x5: '4:5',
|
|
340
|
+
LANDSCAPE_5x4: '5:4',
|
|
341
|
+
PORTRAIT_1x4: '1:4',
|
|
342
|
+
LANDSCAPE_4x1: '4:1',
|
|
343
|
+
PORTRAIT_1x8: '1:8',
|
|
344
|
+
LANDSCAPE_8x1: '8:1',
|
|
345
|
+
ULTRAWIDE_21x9: '21:9'
|
|
346
|
+
};
|
|
347
|
+
/**
|
|
348
|
+
* Sizes for generated images. For example, '1K' is 1024px, '2K' is 2048px, and '4K' is 4096px.
|
|
349
|
+
* @public
|
|
350
|
+
*/
|
|
351
|
+
const ImageConfigImageSize = {
|
|
352
|
+
SIZE_512: '512',
|
|
353
|
+
SIZE_1K: '1K',
|
|
354
|
+
SIZE_2K: '2K',
|
|
355
|
+
SIZE_4K: '4K'
|
|
285
356
|
};
|
|
286
357
|
/**
|
|
287
358
|
* @public
|
|
@@ -491,7 +562,8 @@ const LiveResponseType = {
|
|
|
491
562
|
SERVER_CONTENT: 'serverContent',
|
|
492
563
|
TOOL_CALL: 'toolCall',
|
|
493
564
|
TOOL_CALL_CANCELLATION: 'toolCallCancellation',
|
|
494
|
-
GOING_AWAY_NOTICE: 'goingAwayNotice'
|
|
565
|
+
GOING_AWAY_NOTICE: 'goingAwayNotice',
|
|
566
|
+
SESSION_RESUMPTION_UPDATE: 'sessionResumptionUpdate'
|
|
495
567
|
};
|
|
496
568
|
|
|
497
569
|
/**
|
|
@@ -1834,7 +1906,24 @@ function getInlineDataParts(response) {
|
|
|
1834
1906
|
return undefined;
|
|
1835
1907
|
}
|
|
1836
1908
|
}
|
|
1837
|
-
const badFinishReasons = [
|
|
1909
|
+
const badFinishReasons = [
|
|
1910
|
+
FinishReason.RECITATION,
|
|
1911
|
+
FinishReason.SAFETY,
|
|
1912
|
+
FinishReason.BLOCKLIST,
|
|
1913
|
+
FinishReason.PROHIBITED_CONTENT,
|
|
1914
|
+
FinishReason.SPII,
|
|
1915
|
+
FinishReason.MALFORMED_FUNCTION_CALL,
|
|
1916
|
+
FinishReason.IMAGE_SAFETY,
|
|
1917
|
+
FinishReason.IMAGE_PROHIBITED_CONTENT,
|
|
1918
|
+
FinishReason.IMAGE_OTHER,
|
|
1919
|
+
FinishReason.NO_IMAGE,
|
|
1920
|
+
FinishReason.IMAGE_RECITATION,
|
|
1921
|
+
FinishReason.LANGUAGE,
|
|
1922
|
+
FinishReason.UNEXPECTED_TOOL_CALL,
|
|
1923
|
+
FinishReason.TOO_MANY_TOOL_CALLS,
|
|
1924
|
+
FinishReason.MISSING_THOUGHT_SIGNATURE,
|
|
1925
|
+
FinishReason.MALFORMED_RESPONSE
|
|
1926
|
+
];
|
|
1838
1927
|
function hadBadFinishReason(candidate) {
|
|
1839
1928
|
return (!!candidate.finishReason &&
|
|
1840
1929
|
badFinishReasons.some(reason => reason === candidate.finishReason));
|
|
@@ -3236,6 +3325,153 @@ function validateGenerationConfig(generationConfig) {
|
|
|
3236
3325
|
}
|
|
3237
3326
|
}
|
|
3238
3327
|
|
|
3328
|
+
/**
|
|
3329
|
+
* @license
|
|
3330
|
+
* Copyright 2025 Google LLC
|
|
3331
|
+
*
|
|
3332
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
3333
|
+
* you may not use this file except in compliance with the License.
|
|
3334
|
+
* You may obtain a copy of the License at
|
|
3335
|
+
*
|
|
3336
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
3337
|
+
*
|
|
3338
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
3339
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
3340
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
3341
|
+
* See the License for the specific language governing permissions and
|
|
3342
|
+
* limitations under the License.
|
|
3343
|
+
*/
|
|
3344
|
+
/**
|
|
3345
|
+
* A wrapper for the native `WebSocket` available in both Browsers and Node >= 22.
|
|
3346
|
+
*
|
|
3347
|
+
* @internal
|
|
3348
|
+
*/
|
|
3349
|
+
class WebSocketHandlerImpl {
|
|
3350
|
+
constructor() {
|
|
3351
|
+
if (typeof WebSocket === 'undefined') {
|
|
3352
|
+
throw new AIError(AIErrorCode.UNSUPPORTED, 'The WebSocket API is not available in this environment. ' +
|
|
3353
|
+
'The "Live" feature is not supported here. It is supported in ' +
|
|
3354
|
+
'modern browser windows, Web Workers with WebSocket support, and Node >= 22.');
|
|
3355
|
+
}
|
|
3356
|
+
}
|
|
3357
|
+
connect(url) {
|
|
3358
|
+
return new Promise((resolve, reject) => {
|
|
3359
|
+
this.ws = new WebSocket(url);
|
|
3360
|
+
this.ws.binaryType = 'blob'; // Only important to set in Node
|
|
3361
|
+
this.ws.addEventListener('open', () => resolve(), { once: true });
|
|
3362
|
+
this.ws.addEventListener('error', () => reject(new AIError(AIErrorCode.FETCH_ERROR, `Error event raised on WebSocket`)), { once: true });
|
|
3363
|
+
this.ws.addEventListener('close', (closeEvent) => {
|
|
3364
|
+
if (closeEvent.reason) {
|
|
3365
|
+
logger.warn(`WebSocket connection closed by server. Reason: '${closeEvent.reason}'`);
|
|
3366
|
+
}
|
|
3367
|
+
});
|
|
3368
|
+
});
|
|
3369
|
+
}
|
|
3370
|
+
send(data) {
|
|
3371
|
+
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
|
|
3372
|
+
throw new AIError(AIErrorCode.REQUEST_ERROR, 'WebSocket is not open.');
|
|
3373
|
+
}
|
|
3374
|
+
this.ws.send(data);
|
|
3375
|
+
}
|
|
3376
|
+
async *listen() {
|
|
3377
|
+
if (!this.ws) {
|
|
3378
|
+
throw new AIError(AIErrorCode.REQUEST_ERROR, 'WebSocket is not connected.');
|
|
3379
|
+
}
|
|
3380
|
+
const messageQueue = [];
|
|
3381
|
+
const errorQueue = [];
|
|
3382
|
+
let resolvePromise = null;
|
|
3383
|
+
let isClosed = false;
|
|
3384
|
+
const messageListener = async (event) => {
|
|
3385
|
+
let data;
|
|
3386
|
+
if (event.data instanceof Blob) {
|
|
3387
|
+
data = await event.data.text();
|
|
3388
|
+
}
|
|
3389
|
+
else if (typeof event.data === 'string') {
|
|
3390
|
+
data = event.data;
|
|
3391
|
+
}
|
|
3392
|
+
else {
|
|
3393
|
+
errorQueue.push(new AIError(AIErrorCode.PARSE_FAILED, `Failed to parse WebSocket response. Expected data to be a Blob or string, but was ${typeof event.data}.`));
|
|
3394
|
+
if (resolvePromise) {
|
|
3395
|
+
resolvePromise();
|
|
3396
|
+
resolvePromise = null;
|
|
3397
|
+
}
|
|
3398
|
+
return;
|
|
3399
|
+
}
|
|
3400
|
+
try {
|
|
3401
|
+
const obj = JSON.parse(data);
|
|
3402
|
+
messageQueue.push(obj);
|
|
3403
|
+
}
|
|
3404
|
+
catch (e) {
|
|
3405
|
+
const err = e;
|
|
3406
|
+
errorQueue.push(new AIError(AIErrorCode.PARSE_FAILED, `Error parsing WebSocket message to JSON: ${err.message}`));
|
|
3407
|
+
}
|
|
3408
|
+
if (resolvePromise) {
|
|
3409
|
+
resolvePromise();
|
|
3410
|
+
resolvePromise = null;
|
|
3411
|
+
}
|
|
3412
|
+
};
|
|
3413
|
+
const errorListener = () => {
|
|
3414
|
+
errorQueue.push(new AIError(AIErrorCode.FETCH_ERROR, 'WebSocket connection error.'));
|
|
3415
|
+
if (resolvePromise) {
|
|
3416
|
+
resolvePromise();
|
|
3417
|
+
resolvePromise = null;
|
|
3418
|
+
}
|
|
3419
|
+
};
|
|
3420
|
+
const closeListener = (event) => {
|
|
3421
|
+
if (event.reason) {
|
|
3422
|
+
logger.warn(`WebSocket connection closed by the server with reason: ${event.reason}`);
|
|
3423
|
+
}
|
|
3424
|
+
isClosed = true;
|
|
3425
|
+
if (resolvePromise) {
|
|
3426
|
+
resolvePromise();
|
|
3427
|
+
resolvePromise = null;
|
|
3428
|
+
}
|
|
3429
|
+
// Clean up listeners to prevent memory leaks
|
|
3430
|
+
this.ws?.removeEventListener('message', messageListener);
|
|
3431
|
+
this.ws?.removeEventListener('close', closeListener);
|
|
3432
|
+
this.ws?.removeEventListener('error', errorListener);
|
|
3433
|
+
};
|
|
3434
|
+
this.ws.addEventListener('message', messageListener);
|
|
3435
|
+
this.ws.addEventListener('close', closeListener);
|
|
3436
|
+
this.ws.addEventListener('error', errorListener);
|
|
3437
|
+
while (!isClosed) {
|
|
3438
|
+
if (errorQueue.length > 0) {
|
|
3439
|
+
const error = errorQueue.shift();
|
|
3440
|
+
throw error;
|
|
3441
|
+
}
|
|
3442
|
+
if (messageQueue.length > 0) {
|
|
3443
|
+
yield messageQueue.shift();
|
|
3444
|
+
}
|
|
3445
|
+
else {
|
|
3446
|
+
await new Promise(resolve => {
|
|
3447
|
+
resolvePromise = resolve;
|
|
3448
|
+
});
|
|
3449
|
+
}
|
|
3450
|
+
}
|
|
3451
|
+
// If the loop terminated because isClosed is true, check for any final errors
|
|
3452
|
+
if (errorQueue.length > 0) {
|
|
3453
|
+
const error = errorQueue.shift();
|
|
3454
|
+
throw error;
|
|
3455
|
+
}
|
|
3456
|
+
}
|
|
3457
|
+
close(code, reason) {
|
|
3458
|
+
return new Promise(resolve => {
|
|
3459
|
+
if (!this.ws) {
|
|
3460
|
+
return resolve();
|
|
3461
|
+
}
|
|
3462
|
+
this.ws.addEventListener('close', () => resolve(), { once: true });
|
|
3463
|
+
// Calling 'close' during these states results in an error.
|
|
3464
|
+
if (this.ws.readyState === WebSocket.CLOSED ||
|
|
3465
|
+
this.ws.readyState === WebSocket.CONNECTING) {
|
|
3466
|
+
return resolve();
|
|
3467
|
+
}
|
|
3468
|
+
if (this.ws.readyState !== WebSocket.CLOSING) {
|
|
3469
|
+
this.ws.close(code, reason);
|
|
3470
|
+
}
|
|
3471
|
+
});
|
|
3472
|
+
}
|
|
3473
|
+
}
|
|
3474
|
+
|
|
3239
3475
|
/**
|
|
3240
3476
|
* @license
|
|
3241
3477
|
* Copyright 2025 Google LLC
|
|
@@ -3263,9 +3499,10 @@ class LiveSession {
|
|
|
3263
3499
|
/**
|
|
3264
3500
|
* @internal
|
|
3265
3501
|
*/
|
|
3266
|
-
constructor(
|
|
3267
|
-
this.
|
|
3268
|
-
this.
|
|
3502
|
+
constructor(_setupMessage, _apiSettings, _sessionResumption, webSocketHandler) {
|
|
3503
|
+
this._setupMessage = _setupMessage;
|
|
3504
|
+
this._apiSettings = _apiSettings;
|
|
3505
|
+
this._sessionResumption = _sessionResumption;
|
|
3269
3506
|
/**
|
|
3270
3507
|
* Indicates whether this Live session is closed.
|
|
3271
3508
|
*
|
|
@@ -3278,6 +3515,64 @@ class LiveSession {
|
|
|
3278
3515
|
* @beta
|
|
3279
3516
|
*/
|
|
3280
3517
|
this.inConversation = false;
|
|
3518
|
+
/**
|
|
3519
|
+
* Generator yielding WebSocket messages from the server.
|
|
3520
|
+
*/
|
|
3521
|
+
this._serverMessages = null;
|
|
3522
|
+
this._webSocketHandler = webSocketHandler || new WebSocketHandlerImpl();
|
|
3523
|
+
this.connectionPromise = this._connectSession(this._sessionResumption);
|
|
3524
|
+
}
|
|
3525
|
+
/**
|
|
3526
|
+
* Initializes connection to the WebSocket. Should be called immediately
|
|
3527
|
+
* after instantiation.
|
|
3528
|
+
*
|
|
3529
|
+
* @internal
|
|
3530
|
+
*/
|
|
3531
|
+
async _connectSession(sessionResumption) {
|
|
3532
|
+
const url = new WebSocketUrl(this._apiSettings);
|
|
3533
|
+
await this._webSocketHandler.connect(url.toString());
|
|
3534
|
+
try {
|
|
3535
|
+
// Begin listening for server messages, and begin the handshake by sending the 'setupMessage'
|
|
3536
|
+
this._serverMessages = this._webSocketHandler.listen();
|
|
3537
|
+
const setupMessage = { ...this._setupMessage };
|
|
3538
|
+
if (sessionResumption) {
|
|
3539
|
+
setupMessage.setup.sessionResumption = sessionResumption;
|
|
3540
|
+
}
|
|
3541
|
+
this._webSocketHandler.send(JSON.stringify(setupMessage));
|
|
3542
|
+
// Verify we received the handshake response 'setupComplete'
|
|
3543
|
+
const firstMessage = (await this._serverMessages.next()).value;
|
|
3544
|
+
if (!firstMessage ||
|
|
3545
|
+
!(typeof firstMessage === 'object') ||
|
|
3546
|
+
!('setupComplete' in firstMessage)) {
|
|
3547
|
+
await this._webSocketHandler.close(1011, 'Handshake failure');
|
|
3548
|
+
throw new AIError(AIErrorCode.RESPONSE_ERROR, 'Server connection handshake failed. The server did not respond with a setupComplete message.');
|
|
3549
|
+
}
|
|
3550
|
+
this.isClosed = false;
|
|
3551
|
+
}
|
|
3552
|
+
catch (e) {
|
|
3553
|
+
// Ensure connection is closed on any setup error
|
|
3554
|
+
await this._webSocketHandler.close();
|
|
3555
|
+
throw e;
|
|
3556
|
+
}
|
|
3557
|
+
}
|
|
3558
|
+
/**
|
|
3559
|
+
* Resumes an existing live session with the server.
|
|
3560
|
+
*
|
|
3561
|
+
* This closes the current WebSocket connection and establishes a new one using
|
|
3562
|
+
* the same configuration (URI, headers, model, system instruction, tools, etc.)
|
|
3563
|
+
* as the original session.
|
|
3564
|
+
*
|
|
3565
|
+
* @param sessionResumption - The configuration for session resumption, such as the handle to the previous session state to restore.
|
|
3566
|
+
* @throws If the session resumption configuration is unsupported.
|
|
3567
|
+
*
|
|
3568
|
+
* @beta
|
|
3569
|
+
*/
|
|
3570
|
+
async resumeSession(sessionResumption) {
|
|
3571
|
+
if (!this._sessionResumption) {
|
|
3572
|
+
throw new AIError(AIErrorCode.UNSUPPORTED, 'Cannot resume session: no sessionResumption config provided');
|
|
3573
|
+
}
|
|
3574
|
+
await this.close();
|
|
3575
|
+
await this._connectSession(sessionResumption);
|
|
3281
3576
|
}
|
|
3282
3577
|
/**
|
|
3283
3578
|
* Sends content to the server.
|
|
@@ -3299,7 +3594,7 @@ class LiveSession {
|
|
|
3299
3594
|
turnComplete
|
|
3300
3595
|
}
|
|
3301
3596
|
};
|
|
3302
|
-
this.
|
|
3597
|
+
this._webSocketHandler.send(JSON.stringify(message));
|
|
3303
3598
|
}
|
|
3304
3599
|
/**
|
|
3305
3600
|
* Sends text to the server in realtime.
|
|
@@ -3323,7 +3618,7 @@ class LiveSession {
|
|
|
3323
3618
|
text
|
|
3324
3619
|
}
|
|
3325
3620
|
};
|
|
3326
|
-
this.
|
|
3621
|
+
this._webSocketHandler.send(JSON.stringify(message));
|
|
3327
3622
|
}
|
|
3328
3623
|
/**
|
|
3329
3624
|
* Sends audio data to the server in realtime.
|
|
@@ -3352,7 +3647,7 @@ class LiveSession {
|
|
|
3352
3647
|
audio: blob
|
|
3353
3648
|
}
|
|
3354
3649
|
};
|
|
3355
|
-
this.
|
|
3650
|
+
this._webSocketHandler.send(JSON.stringify(message));
|
|
3356
3651
|
}
|
|
3357
3652
|
/**
|
|
3358
3653
|
* Sends video data to the server in realtime.
|
|
@@ -3380,7 +3675,7 @@ class LiveSession {
|
|
|
3380
3675
|
video: blob
|
|
3381
3676
|
}
|
|
3382
3677
|
};
|
|
3383
|
-
this.
|
|
3678
|
+
this._webSocketHandler.send(JSON.stringify(message));
|
|
3384
3679
|
}
|
|
3385
3680
|
/**
|
|
3386
3681
|
* Sends function responses to the server.
|
|
@@ -3399,7 +3694,7 @@ class LiveSession {
|
|
|
3399
3694
|
functionResponses
|
|
3400
3695
|
}
|
|
3401
3696
|
};
|
|
3402
|
-
this.
|
|
3697
|
+
this._webSocketHandler.send(JSON.stringify(message));
|
|
3403
3698
|
}
|
|
3404
3699
|
/**
|
|
3405
3700
|
* Yields messages received from the server.
|
|
@@ -3414,42 +3709,50 @@ class LiveSession {
|
|
|
3414
3709
|
if (this.isClosed) {
|
|
3415
3710
|
throw new AIError(AIErrorCode.SESSION_CLOSED, 'Cannot read from a Live session that is closed. Try starting a new Live session.');
|
|
3416
3711
|
}
|
|
3417
|
-
|
|
3418
|
-
|
|
3419
|
-
if (
|
|
3420
|
-
|
|
3421
|
-
|
|
3422
|
-
|
|
3423
|
-
|
|
3424
|
-
|
|
3425
|
-
|
|
3426
|
-
|
|
3427
|
-
|
|
3428
|
-
|
|
3429
|
-
|
|
3430
|
-
|
|
3431
|
-
|
|
3432
|
-
|
|
3433
|
-
|
|
3434
|
-
|
|
3435
|
-
|
|
3436
|
-
|
|
3437
|
-
|
|
3438
|
-
|
|
3439
|
-
|
|
3440
|
-
|
|
3441
|
-
|
|
3442
|
-
|
|
3443
|
-
|
|
3444
|
-
|
|
3712
|
+
if (this._serverMessages) {
|
|
3713
|
+
for await (const message of this._serverMessages) {
|
|
3714
|
+
if (message && typeof message === 'object') {
|
|
3715
|
+
if (LiveResponseType.SERVER_CONTENT in message) {
|
|
3716
|
+
yield {
|
|
3717
|
+
type: 'serverContent',
|
|
3718
|
+
...message
|
|
3719
|
+
.serverContent
|
|
3720
|
+
};
|
|
3721
|
+
}
|
|
3722
|
+
else if (LiveResponseType.TOOL_CALL in message) {
|
|
3723
|
+
yield {
|
|
3724
|
+
type: 'toolCall',
|
|
3725
|
+
...message
|
|
3726
|
+
.toolCall
|
|
3727
|
+
};
|
|
3728
|
+
}
|
|
3729
|
+
else if (LiveResponseType.TOOL_CALL_CANCELLATION in message) {
|
|
3730
|
+
yield {
|
|
3731
|
+
type: 'toolCallCancellation',
|
|
3732
|
+
...message.toolCallCancellation
|
|
3733
|
+
};
|
|
3734
|
+
}
|
|
3735
|
+
else if ('goAway' in message) {
|
|
3736
|
+
const notice = message.goAway;
|
|
3737
|
+
yield {
|
|
3738
|
+
type: LiveResponseType.GOING_AWAY_NOTICE,
|
|
3739
|
+
timeLeft: parseDuration(notice.timeLeft)
|
|
3740
|
+
};
|
|
3741
|
+
}
|
|
3742
|
+
else if (LiveResponseType.SESSION_RESUMPTION_UPDATE in message) {
|
|
3743
|
+
yield {
|
|
3744
|
+
type: LiveResponseType.SESSION_RESUMPTION_UPDATE,
|
|
3745
|
+
...message.sessionResumptionUpdate
|
|
3746
|
+
};
|
|
3747
|
+
}
|
|
3748
|
+
else {
|
|
3749
|
+
logger.warn(`Received an unknown message type from the server: ${JSON.stringify(message)}`);
|
|
3750
|
+
}
|
|
3445
3751
|
}
|
|
3446
3752
|
else {
|
|
3447
|
-
logger.warn(`Received an
|
|
3753
|
+
logger.warn(`Received an invalid message from the server: ${JSON.stringify(message)}`);
|
|
3448
3754
|
}
|
|
3449
3755
|
}
|
|
3450
|
-
else {
|
|
3451
|
-
logger.warn(`Received an invalid message from the server: ${JSON.stringify(message)}`);
|
|
3452
|
-
}
|
|
3453
3756
|
}
|
|
3454
3757
|
}
|
|
3455
3758
|
/**
|
|
@@ -3461,7 +3764,7 @@ class LiveSession {
|
|
|
3461
3764
|
async close() {
|
|
3462
3765
|
if (!this.isClosed) {
|
|
3463
3766
|
this.isClosed = true;
|
|
3464
|
-
await this.
|
|
3767
|
+
await this._webSocketHandler.close(1000, 'Client closed session.');
|
|
3465
3768
|
}
|
|
3466
3769
|
}
|
|
3467
3770
|
/**
|
|
@@ -3484,7 +3787,7 @@ class LiveSession {
|
|
|
3484
3787
|
const message = {
|
|
3485
3788
|
realtimeInput: { mediaChunks: [mediaChunk] }
|
|
3486
3789
|
};
|
|
3487
|
-
this.
|
|
3790
|
+
this._webSocketHandler.send(JSON.stringify(message));
|
|
3488
3791
|
});
|
|
3489
3792
|
}
|
|
3490
3793
|
/**
|
|
@@ -3564,6 +3867,7 @@ class LiveGenerativeModel extends AIModel {
|
|
|
3564
3867
|
*/
|
|
3565
3868
|
constructor(ai, modelParams,
|
|
3566
3869
|
/**
|
|
3870
|
+
* For testing injection
|
|
3567
3871
|
* @internal
|
|
3568
3872
|
*/
|
|
3569
3873
|
_webSocketHandler) {
|
|
@@ -3582,9 +3886,7 @@ class LiveGenerativeModel extends AIModel {
|
|
|
3582
3886
|
*
|
|
3583
3887
|
* @beta
|
|
3584
3888
|
*/
|
|
3585
|
-
async connect() {
|
|
3586
|
-
const url = new WebSocketUrl(this._apiSettings);
|
|
3587
|
-
await this._webSocketHandler.connect(url.toString());
|
|
3889
|
+
async connect(sessionResumption) {
|
|
3588
3890
|
let fullModelPath;
|
|
3589
3891
|
if (this._apiSettings.backend.backendType === BackendType.GOOGLE_AI) {
|
|
3590
3892
|
fullModelPath = `projects/${this._apiSettings.project}/${this.model}`;
|
|
@@ -3595,36 +3897,24 @@ class LiveGenerativeModel extends AIModel {
|
|
|
3595
3897
|
// inputAudioTranscription and outputAudioTranscription are on the generation config in the public API,
|
|
3596
3898
|
// but the backend expects them to be in the `setup` message.
|
|
3597
3899
|
const { inputAudioTranscription, outputAudioTranscription, ...generationConfig } = this.generationConfig;
|
|
3900
|
+
const contextWindowCompression = generationConfig.contextWindowCompression;
|
|
3901
|
+
delete generationConfig.contextWindowCompression;
|
|
3598
3902
|
const setupMessage = {
|
|
3599
3903
|
setup: {
|
|
3600
3904
|
model: fullModelPath,
|
|
3601
3905
|
generationConfig,
|
|
3906
|
+
contextWindowCompression,
|
|
3602
3907
|
tools: this.tools,
|
|
3603
3908
|
toolConfig: this.toolConfig,
|
|
3604
3909
|
systemInstruction: this.systemInstruction,
|
|
3605
3910
|
inputAudioTranscription,
|
|
3606
|
-
outputAudioTranscription
|
|
3911
|
+
outputAudioTranscription,
|
|
3912
|
+
sessionResumption
|
|
3607
3913
|
}
|
|
3608
3914
|
};
|
|
3609
|
-
|
|
3610
|
-
|
|
3611
|
-
|
|
3612
|
-
this._webSocketHandler.send(JSON.stringify(setupMessage));
|
|
3613
|
-
// Verify we received the handshake response 'setupComplete'
|
|
3614
|
-
const firstMessage = (await serverMessages.next()).value;
|
|
3615
|
-
if (!firstMessage ||
|
|
3616
|
-
!(typeof firstMessage === 'object') ||
|
|
3617
|
-
!('setupComplete' in firstMessage)) {
|
|
3618
|
-
await this._webSocketHandler.close(1011, 'Handshake failure');
|
|
3619
|
-
throw new AIError(AIErrorCode.RESPONSE_ERROR, 'Server connection handshake failed. The server did not respond with a setupComplete message.');
|
|
3620
|
-
}
|
|
3621
|
-
return new LiveSession(this._webSocketHandler, serverMessages);
|
|
3622
|
-
}
|
|
3623
|
-
catch (e) {
|
|
3624
|
-
// Ensure connection is closed on any setup error
|
|
3625
|
-
await this._webSocketHandler.close();
|
|
3626
|
-
throw e;
|
|
3627
|
-
}
|
|
3915
|
+
const session = new LiveSession(setupMessage, this._apiSettings, sessionResumption, this._webSocketHandler);
|
|
3916
|
+
await session.connectionPromise;
|
|
3917
|
+
return session;
|
|
3628
3918
|
}
|
|
3629
3919
|
}
|
|
3630
3920
|
|
|
@@ -3765,153 +4055,6 @@ class ImagenModel extends AIModel {
|
|
|
3765
4055
|
}
|
|
3766
4056
|
}
|
|
3767
4057
|
|
|
3768
|
-
/**
|
|
3769
|
-
* @license
|
|
3770
|
-
* Copyright 2025 Google LLC
|
|
3771
|
-
*
|
|
3772
|
-
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
3773
|
-
* you may not use this file except in compliance with the License.
|
|
3774
|
-
* You may obtain a copy of the License at
|
|
3775
|
-
*
|
|
3776
|
-
* http://www.apache.org/licenses/LICENSE-2.0
|
|
3777
|
-
*
|
|
3778
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
3779
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
3780
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
3781
|
-
* See the License for the specific language governing permissions and
|
|
3782
|
-
* limitations under the License.
|
|
3783
|
-
*/
|
|
3784
|
-
/**
|
|
3785
|
-
* A wrapper for the native `WebSocket` available in both Browsers and Node >= 22.
|
|
3786
|
-
*
|
|
3787
|
-
* @internal
|
|
3788
|
-
*/
|
|
3789
|
-
class WebSocketHandlerImpl {
|
|
3790
|
-
constructor() {
|
|
3791
|
-
if (typeof WebSocket === 'undefined') {
|
|
3792
|
-
throw new AIError(AIErrorCode.UNSUPPORTED, 'The WebSocket API is not available in this environment. ' +
|
|
3793
|
-
'The "Live" feature is not supported here. It is supported in ' +
|
|
3794
|
-
'modern browser windows, Web Workers with WebSocket support, and Node >= 22.');
|
|
3795
|
-
}
|
|
3796
|
-
}
|
|
3797
|
-
connect(url) {
|
|
3798
|
-
return new Promise((resolve, reject) => {
|
|
3799
|
-
this.ws = new WebSocket(url);
|
|
3800
|
-
this.ws.binaryType = 'blob'; // Only important to set in Node
|
|
3801
|
-
this.ws.addEventListener('open', () => resolve(), { once: true });
|
|
3802
|
-
this.ws.addEventListener('error', () => reject(new AIError(AIErrorCode.FETCH_ERROR, `Error event raised on WebSocket`)), { once: true });
|
|
3803
|
-
this.ws.addEventListener('close', (closeEvent) => {
|
|
3804
|
-
if (closeEvent.reason) {
|
|
3805
|
-
logger.warn(`WebSocket connection closed by server. Reason: '${closeEvent.reason}'`);
|
|
3806
|
-
}
|
|
3807
|
-
});
|
|
3808
|
-
});
|
|
3809
|
-
}
|
|
3810
|
-
send(data) {
|
|
3811
|
-
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
|
|
3812
|
-
throw new AIError(AIErrorCode.REQUEST_ERROR, 'WebSocket is not open.');
|
|
3813
|
-
}
|
|
3814
|
-
this.ws.send(data);
|
|
3815
|
-
}
|
|
3816
|
-
async *listen() {
|
|
3817
|
-
if (!this.ws) {
|
|
3818
|
-
throw new AIError(AIErrorCode.REQUEST_ERROR, 'WebSocket is not connected.');
|
|
3819
|
-
}
|
|
3820
|
-
const messageQueue = [];
|
|
3821
|
-
const errorQueue = [];
|
|
3822
|
-
let resolvePromise = null;
|
|
3823
|
-
let isClosed = false;
|
|
3824
|
-
const messageListener = async (event) => {
|
|
3825
|
-
let data;
|
|
3826
|
-
if (event.data instanceof Blob) {
|
|
3827
|
-
data = await event.data.text();
|
|
3828
|
-
}
|
|
3829
|
-
else if (typeof event.data === 'string') {
|
|
3830
|
-
data = event.data;
|
|
3831
|
-
}
|
|
3832
|
-
else {
|
|
3833
|
-
errorQueue.push(new AIError(AIErrorCode.PARSE_FAILED, `Failed to parse WebSocket response. Expected data to be a Blob or string, but was ${typeof event.data}.`));
|
|
3834
|
-
if (resolvePromise) {
|
|
3835
|
-
resolvePromise();
|
|
3836
|
-
resolvePromise = null;
|
|
3837
|
-
}
|
|
3838
|
-
return;
|
|
3839
|
-
}
|
|
3840
|
-
try {
|
|
3841
|
-
const obj = JSON.parse(data);
|
|
3842
|
-
messageQueue.push(obj);
|
|
3843
|
-
}
|
|
3844
|
-
catch (e) {
|
|
3845
|
-
const err = e;
|
|
3846
|
-
errorQueue.push(new AIError(AIErrorCode.PARSE_FAILED, `Error parsing WebSocket message to JSON: ${err.message}`));
|
|
3847
|
-
}
|
|
3848
|
-
if (resolvePromise) {
|
|
3849
|
-
resolvePromise();
|
|
3850
|
-
resolvePromise = null;
|
|
3851
|
-
}
|
|
3852
|
-
};
|
|
3853
|
-
const errorListener = () => {
|
|
3854
|
-
errorQueue.push(new AIError(AIErrorCode.FETCH_ERROR, 'WebSocket connection error.'));
|
|
3855
|
-
if (resolvePromise) {
|
|
3856
|
-
resolvePromise();
|
|
3857
|
-
resolvePromise = null;
|
|
3858
|
-
}
|
|
3859
|
-
};
|
|
3860
|
-
const closeListener = (event) => {
|
|
3861
|
-
if (event.reason) {
|
|
3862
|
-
logger.warn(`WebSocket connection closed by the server with reason: ${event.reason}`);
|
|
3863
|
-
}
|
|
3864
|
-
isClosed = true;
|
|
3865
|
-
if (resolvePromise) {
|
|
3866
|
-
resolvePromise();
|
|
3867
|
-
resolvePromise = null;
|
|
3868
|
-
}
|
|
3869
|
-
// Clean up listeners to prevent memory leaks
|
|
3870
|
-
this.ws?.removeEventListener('message', messageListener);
|
|
3871
|
-
this.ws?.removeEventListener('close', closeListener);
|
|
3872
|
-
this.ws?.removeEventListener('error', errorListener);
|
|
3873
|
-
};
|
|
3874
|
-
this.ws.addEventListener('message', messageListener);
|
|
3875
|
-
this.ws.addEventListener('close', closeListener);
|
|
3876
|
-
this.ws.addEventListener('error', errorListener);
|
|
3877
|
-
while (!isClosed) {
|
|
3878
|
-
if (errorQueue.length > 0) {
|
|
3879
|
-
const error = errorQueue.shift();
|
|
3880
|
-
throw error;
|
|
3881
|
-
}
|
|
3882
|
-
if (messageQueue.length > 0) {
|
|
3883
|
-
yield messageQueue.shift();
|
|
3884
|
-
}
|
|
3885
|
-
else {
|
|
3886
|
-
await new Promise(resolve => {
|
|
3887
|
-
resolvePromise = resolve;
|
|
3888
|
-
});
|
|
3889
|
-
}
|
|
3890
|
-
}
|
|
3891
|
-
// If the loop terminated because isClosed is true, check for any final errors
|
|
3892
|
-
if (errorQueue.length > 0) {
|
|
3893
|
-
const error = errorQueue.shift();
|
|
3894
|
-
throw error;
|
|
3895
|
-
}
|
|
3896
|
-
}
|
|
3897
|
-
close(code, reason) {
|
|
3898
|
-
return new Promise(resolve => {
|
|
3899
|
-
if (!this.ws) {
|
|
3900
|
-
return resolve();
|
|
3901
|
-
}
|
|
3902
|
-
this.ws.addEventListener('close', () => resolve(), { once: true });
|
|
3903
|
-
// Calling 'close' during these states results in an error.
|
|
3904
|
-
if (this.ws.readyState === WebSocket.CLOSED ||
|
|
3905
|
-
this.ws.readyState === WebSocket.CONNECTING) {
|
|
3906
|
-
return resolve();
|
|
3907
|
-
}
|
|
3908
|
-
if (this.ws.readyState !== WebSocket.CLOSING) {
|
|
3909
|
-
this.ws.close(code, reason);
|
|
3910
|
-
}
|
|
3911
|
-
});
|
|
3912
|
-
}
|
|
3913
|
-
}
|
|
3914
|
-
|
|
3915
4058
|
/**
|
|
3916
4059
|
* @license
|
|
3917
4060
|
* Copyright 2026 Google LLC
|
|
@@ -4056,11 +4199,16 @@ class TemplateGenerativeModel {
|
|
|
4056
4199
|
* @param templateId - The ID of the server-side template to execute.
|
|
4057
4200
|
* @param templateVariables - A key-value map of variables to populate the
|
|
4058
4201
|
* template with.
|
|
4202
|
+
* @param singleRequestOptions - Optional. Options to use for this request.
|
|
4203
|
+
* @param templateToolConfig - Optional. Configuration for tools to use with this request.
|
|
4059
4204
|
*
|
|
4060
4205
|
* @beta
|
|
4061
4206
|
*/
|
|
4062
|
-
async generateContent(templateId, templateVariables, singleRequestOptions) {
|
|
4063
|
-
return templateGenerateContent(this._apiSettings, templateId, {
|
|
4207
|
+
async generateContent(templateId, templateVariables, singleRequestOptions, templateToolConfig) {
|
|
4208
|
+
return templateGenerateContent(this._apiSettings, templateId, {
|
|
4209
|
+
inputs: templateVariables,
|
|
4210
|
+
...(templateToolConfig && { toolConfig: templateToolConfig })
|
|
4211
|
+
}, {
|
|
4064
4212
|
...this.requestOptions,
|
|
4065
4213
|
...singleRequestOptions
|
|
4066
4214
|
});
|
|
@@ -4074,11 +4222,16 @@ class TemplateGenerativeModel {
|
|
|
4074
4222
|
* @param templateId - The ID of the server-side template to execute.
|
|
4075
4223
|
* @param templateVariables - A key-value map of variables to populate the
|
|
4076
4224
|
* template with.
|
|
4225
|
+
* @param singleRequestOptions - Optional.Options to use for this request.
|
|
4226
|
+
* @param templateToolConfig - Optional. Configuration for tools to use with this request.
|
|
4077
4227
|
*
|
|
4078
4228
|
* @beta
|
|
4079
4229
|
*/
|
|
4080
|
-
async generateContentStream(templateId, templateVariables, singleRequestOptions) {
|
|
4081
|
-
return templateGenerateContentStream(this._apiSettings, templateId, {
|
|
4230
|
+
async generateContentStream(templateId, templateVariables, singleRequestOptions, templateToolConfig) {
|
|
4231
|
+
return templateGenerateContentStream(this._apiSettings, templateId, {
|
|
4232
|
+
inputs: templateVariables,
|
|
4233
|
+
...(templateToolConfig && { toolConfig: templateToolConfig })
|
|
4234
|
+
}, {
|
|
4082
4235
|
...this.requestOptions,
|
|
4083
4236
|
...singleRequestOptions
|
|
4084
4237
|
});
|
|
@@ -4971,8 +5124,7 @@ function getLiveGenerativeModel(ai, modelParams) {
|
|
|
4971
5124
|
if (!modelParams.model) {
|
|
4972
5125
|
throw new AIError(AIErrorCode.NO_MODEL, `Must provide a model name for getLiveGenerativeModel. Example: getLiveGenerativeModel(ai, { model: 'my-model-name' })`);
|
|
4973
5126
|
}
|
|
4974
|
-
|
|
4975
|
-
return new LiveGenerativeModel(ai, modelParams, webSocketHandler);
|
|
5127
|
+
return new LiveGenerativeModel(ai, modelParams);
|
|
4976
5128
|
}
|
|
4977
5129
|
/**
|
|
4978
5130
|
* Returns a {@link TemplateGenerativeModel} class for executing server-side
|
|
@@ -5035,6 +5187,8 @@ exports.HarmBlockThreshold = HarmBlockThreshold;
|
|
|
5035
5187
|
exports.HarmCategory = HarmCategory;
|
|
5036
5188
|
exports.HarmProbability = HarmProbability;
|
|
5037
5189
|
exports.HarmSeverity = HarmSeverity;
|
|
5190
|
+
exports.ImageConfigAspectRatio = ImageConfigAspectRatio;
|
|
5191
|
+
exports.ImageConfigImageSize = ImageConfigImageSize;
|
|
5038
5192
|
exports.ImagenAspectRatio = ImagenAspectRatio;
|
|
5039
5193
|
exports.ImagenImageFormat = ImagenImageFormat;
|
|
5040
5194
|
exports.ImagenModel = ImagenModel;
|