@firebase/ai 2.11.1 → 2.12.0-20260505164105
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-public.d.ts +358 -9
- package/dist/ai.d.ts +386 -12
- package/dist/esm/index.esm.js +375 -223
- package/dist/esm/index.esm.js.map +1 -1
- package/dist/esm/src/methods/live-session.d.ts +40 -5
- package/dist/esm/src/models/live-generative-model.d.ts +6 -4
- package/dist/esm/src/models/template-generative-model.d.ts +7 -3
- package/dist/esm/src/types/enums.d.ts +80 -0
- package/dist/esm/src/types/live-responses.d.ts +3 -1
- package/dist/esm/src/types/requests.d.ts +148 -2
- package/dist/esm/src/types/responses.d.ts +69 -2
- package/dist/index.cjs.js +376 -222
- package/dist/index.cjs.js.map +1 -1
- package/dist/index.node.cjs.js +376 -222
- package/dist/index.node.cjs.js.map +1 -1
- package/dist/index.node.mjs +375 -223
- package/dist/index.node.mjs.map +1 -1
- package/dist/src/methods/live-session.d.ts +40 -5
- package/dist/src/models/live-generative-model.d.ts +6 -4
- package/dist/src/models/template-generative-model.d.ts +7 -3
- package/dist/src/types/enums.d.ts +80 -0
- package/dist/src/types/live-responses.d.ts +3 -1
- package/dist/src/types/requests.d.ts +148 -2
- package/dist/src/types/responses.d.ts +69 -2
- package/package.json +9 -9
package/dist/esm/index.esm.js
CHANGED
|
@@ -4,7 +4,7 @@ import { FirebaseError, Deferred, getModularInstance } from '@firebase/util';
|
|
|
4
4
|
import { Logger } from '@firebase/logger';
|
|
5
5
|
|
|
6
6
|
var name = "@firebase/ai";
|
|
7
|
-
var version = "2.
|
|
7
|
+
var version = "2.12.0-20260505164105";
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
10
|
* @license
|
|
@@ -277,7 +277,78 @@ const FinishReason = {
|
|
|
277
277
|
/**
|
|
278
278
|
* The function call generated by the model was invalid.
|
|
279
279
|
*/
|
|
280
|
-
MALFORMED_FUNCTION_CALL: 'MALFORMED_FUNCTION_CALL'
|
|
280
|
+
MALFORMED_FUNCTION_CALL: 'MALFORMED_FUNCTION_CALL',
|
|
281
|
+
/**
|
|
282
|
+
* Token generation stopped because generated images contain safety violations.
|
|
283
|
+
*/
|
|
284
|
+
IMAGE_SAFETY: 'IMAGE_SAFETY',
|
|
285
|
+
/**
|
|
286
|
+
* Image generation stopped because generated images have other prohibited content.
|
|
287
|
+
*/
|
|
288
|
+
IMAGE_PROHIBITED_CONTENT: 'IMAGE_PROHIBITED_CONTENT',
|
|
289
|
+
/**
|
|
290
|
+
* Image generation stopped because of other miscellaneous issue.
|
|
291
|
+
*/
|
|
292
|
+
IMAGE_OTHER: 'IMAGE_OTHER',
|
|
293
|
+
/**
|
|
294
|
+
* The model was expected to generate an image, but none was generated.
|
|
295
|
+
*/
|
|
296
|
+
NO_IMAGE: 'NO_IMAGE',
|
|
297
|
+
/**
|
|
298
|
+
* Image generation stopped due to recitation.
|
|
299
|
+
*/
|
|
300
|
+
IMAGE_RECITATION: 'IMAGE_RECITATION',
|
|
301
|
+
/**
|
|
302
|
+
* The response candidate content was flagged for using an unsupported language.
|
|
303
|
+
*/
|
|
304
|
+
LANGUAGE: 'LANGUAGE',
|
|
305
|
+
/**
|
|
306
|
+
* Model generated a tool call but no tools were enabled in the request.
|
|
307
|
+
*/
|
|
308
|
+
UNEXPECTED_TOOL_CALL: 'UNEXPECTED_TOOL_CALL',
|
|
309
|
+
/**
|
|
310
|
+
* Model called too many tools consecutively, thus the system exited execution.
|
|
311
|
+
*/
|
|
312
|
+
TOO_MANY_TOOL_CALLS: 'TOO_MANY_TOOL_CALLS',
|
|
313
|
+
/**
|
|
314
|
+
* Request has at least one thought signature missing.
|
|
315
|
+
*/
|
|
316
|
+
MISSING_THOUGHT_SIGNATURE: 'MISSING_THOUGHT_SIGNATURE',
|
|
317
|
+
/**
|
|
318
|
+
* Finished due to malformed response.
|
|
319
|
+
*/
|
|
320
|
+
MALFORMED_RESPONSE: 'MALFORMED_RESPONSE'
|
|
321
|
+
};
|
|
322
|
+
/**
|
|
323
|
+
* Aspect ratios for generated images.
|
|
324
|
+
* @public
|
|
325
|
+
*/
|
|
326
|
+
/* eslint-disable camelcase */
|
|
327
|
+
const ImageConfigAspectRatio = {
|
|
328
|
+
SQUARE_1x1: '1:1',
|
|
329
|
+
PORTRAIT_9x16: '9:16',
|
|
330
|
+
LANDSCAPE_16x9: '16:9',
|
|
331
|
+
PORTRAIT_3x4: '3:4',
|
|
332
|
+
LANDSCAPE_4x3: '4:3',
|
|
333
|
+
PORTRAIT_2x3: '2:3',
|
|
334
|
+
LANDSCAPE_3x2: '3:2',
|
|
335
|
+
PORTRAIT_4x5: '4:5',
|
|
336
|
+
LANDSCAPE_5x4: '5:4',
|
|
337
|
+
PORTRAIT_1x4: '1:4',
|
|
338
|
+
LANDSCAPE_4x1: '4:1',
|
|
339
|
+
PORTRAIT_1x8: '1:8',
|
|
340
|
+
LANDSCAPE_8x1: '8:1',
|
|
341
|
+
ULTRAWIDE_21x9: '21:9'
|
|
342
|
+
};
|
|
343
|
+
/**
|
|
344
|
+
* Sizes for generated images. For example, '1K' is 1024px, '2K' is 2048px, and '4K' is 4096px.
|
|
345
|
+
* @public
|
|
346
|
+
*/
|
|
347
|
+
const ImageConfigImageSize = {
|
|
348
|
+
SIZE_512: '512',
|
|
349
|
+
SIZE_1K: '1K',
|
|
350
|
+
SIZE_2K: '2K',
|
|
351
|
+
SIZE_4K: '4K'
|
|
281
352
|
};
|
|
282
353
|
/**
|
|
283
354
|
* @public
|
|
@@ -487,7 +558,8 @@ const LiveResponseType = {
|
|
|
487
558
|
SERVER_CONTENT: 'serverContent',
|
|
488
559
|
TOOL_CALL: 'toolCall',
|
|
489
560
|
TOOL_CALL_CANCELLATION: 'toolCallCancellation',
|
|
490
|
-
GOING_AWAY_NOTICE: 'goingAwayNotice'
|
|
561
|
+
GOING_AWAY_NOTICE: 'goingAwayNotice',
|
|
562
|
+
SESSION_RESUMPTION_UPDATE: 'sessionResumptionUpdate'
|
|
491
563
|
};
|
|
492
564
|
|
|
493
565
|
/**
|
|
@@ -1830,7 +1902,24 @@ function getInlineDataParts(response) {
|
|
|
1830
1902
|
return undefined;
|
|
1831
1903
|
}
|
|
1832
1904
|
}
|
|
1833
|
-
const badFinishReasons = [
|
|
1905
|
+
const badFinishReasons = [
|
|
1906
|
+
FinishReason.RECITATION,
|
|
1907
|
+
FinishReason.SAFETY,
|
|
1908
|
+
FinishReason.BLOCKLIST,
|
|
1909
|
+
FinishReason.PROHIBITED_CONTENT,
|
|
1910
|
+
FinishReason.SPII,
|
|
1911
|
+
FinishReason.MALFORMED_FUNCTION_CALL,
|
|
1912
|
+
FinishReason.IMAGE_SAFETY,
|
|
1913
|
+
FinishReason.IMAGE_PROHIBITED_CONTENT,
|
|
1914
|
+
FinishReason.IMAGE_OTHER,
|
|
1915
|
+
FinishReason.NO_IMAGE,
|
|
1916
|
+
FinishReason.IMAGE_RECITATION,
|
|
1917
|
+
FinishReason.LANGUAGE,
|
|
1918
|
+
FinishReason.UNEXPECTED_TOOL_CALL,
|
|
1919
|
+
FinishReason.TOO_MANY_TOOL_CALLS,
|
|
1920
|
+
FinishReason.MISSING_THOUGHT_SIGNATURE,
|
|
1921
|
+
FinishReason.MALFORMED_RESPONSE
|
|
1922
|
+
];
|
|
1834
1923
|
function hadBadFinishReason(candidate) {
|
|
1835
1924
|
return (!!candidate.finishReason &&
|
|
1836
1925
|
badFinishReasons.some(reason => reason === candidate.finishReason));
|
|
@@ -3232,6 +3321,153 @@ function validateGenerationConfig(generationConfig) {
|
|
|
3232
3321
|
}
|
|
3233
3322
|
}
|
|
3234
3323
|
|
|
3324
|
+
/**
|
|
3325
|
+
* @license
|
|
3326
|
+
* Copyright 2025 Google LLC
|
|
3327
|
+
*
|
|
3328
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
3329
|
+
* you may not use this file except in compliance with the License.
|
|
3330
|
+
* You may obtain a copy of the License at
|
|
3331
|
+
*
|
|
3332
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
3333
|
+
*
|
|
3334
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
3335
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
3336
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
3337
|
+
* See the License for the specific language governing permissions and
|
|
3338
|
+
* limitations under the License.
|
|
3339
|
+
*/
|
|
3340
|
+
/**
|
|
3341
|
+
* A wrapper for the native `WebSocket` available in both Browsers and Node >= 22.
|
|
3342
|
+
*
|
|
3343
|
+
* @internal
|
|
3344
|
+
*/
|
|
3345
|
+
class WebSocketHandlerImpl {
|
|
3346
|
+
constructor() {
|
|
3347
|
+
if (typeof WebSocket === 'undefined') {
|
|
3348
|
+
throw new AIError(AIErrorCode.UNSUPPORTED, 'The WebSocket API is not available in this environment. ' +
|
|
3349
|
+
'The "Live" feature is not supported here. It is supported in ' +
|
|
3350
|
+
'modern browser windows, Web Workers with WebSocket support, and Node >= 22.');
|
|
3351
|
+
}
|
|
3352
|
+
}
|
|
3353
|
+
connect(url) {
|
|
3354
|
+
return new Promise((resolve, reject) => {
|
|
3355
|
+
this.ws = new WebSocket(url);
|
|
3356
|
+
this.ws.binaryType = 'blob'; // Only important to set in Node
|
|
3357
|
+
this.ws.addEventListener('open', () => resolve(), { once: true });
|
|
3358
|
+
this.ws.addEventListener('error', () => reject(new AIError(AIErrorCode.FETCH_ERROR, `Error event raised on WebSocket`)), { once: true });
|
|
3359
|
+
this.ws.addEventListener('close', (closeEvent) => {
|
|
3360
|
+
if (closeEvent.reason) {
|
|
3361
|
+
logger.warn(`WebSocket connection closed by server. Reason: '${closeEvent.reason}'`);
|
|
3362
|
+
}
|
|
3363
|
+
});
|
|
3364
|
+
});
|
|
3365
|
+
}
|
|
3366
|
+
send(data) {
|
|
3367
|
+
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
|
|
3368
|
+
throw new AIError(AIErrorCode.REQUEST_ERROR, 'WebSocket is not open.');
|
|
3369
|
+
}
|
|
3370
|
+
this.ws.send(data);
|
|
3371
|
+
}
|
|
3372
|
+
async *listen() {
|
|
3373
|
+
if (!this.ws) {
|
|
3374
|
+
throw new AIError(AIErrorCode.REQUEST_ERROR, 'WebSocket is not connected.');
|
|
3375
|
+
}
|
|
3376
|
+
const messageQueue = [];
|
|
3377
|
+
const errorQueue = [];
|
|
3378
|
+
let resolvePromise = null;
|
|
3379
|
+
let isClosed = false;
|
|
3380
|
+
const messageListener = async (event) => {
|
|
3381
|
+
let data;
|
|
3382
|
+
if (event.data instanceof Blob) {
|
|
3383
|
+
data = await event.data.text();
|
|
3384
|
+
}
|
|
3385
|
+
else if (typeof event.data === 'string') {
|
|
3386
|
+
data = event.data;
|
|
3387
|
+
}
|
|
3388
|
+
else {
|
|
3389
|
+
errorQueue.push(new AIError(AIErrorCode.PARSE_FAILED, `Failed to parse WebSocket response. Expected data to be a Blob or string, but was ${typeof event.data}.`));
|
|
3390
|
+
if (resolvePromise) {
|
|
3391
|
+
resolvePromise();
|
|
3392
|
+
resolvePromise = null;
|
|
3393
|
+
}
|
|
3394
|
+
return;
|
|
3395
|
+
}
|
|
3396
|
+
try {
|
|
3397
|
+
const obj = JSON.parse(data);
|
|
3398
|
+
messageQueue.push(obj);
|
|
3399
|
+
}
|
|
3400
|
+
catch (e) {
|
|
3401
|
+
const err = e;
|
|
3402
|
+
errorQueue.push(new AIError(AIErrorCode.PARSE_FAILED, `Error parsing WebSocket message to JSON: ${err.message}`));
|
|
3403
|
+
}
|
|
3404
|
+
if (resolvePromise) {
|
|
3405
|
+
resolvePromise();
|
|
3406
|
+
resolvePromise = null;
|
|
3407
|
+
}
|
|
3408
|
+
};
|
|
3409
|
+
const errorListener = () => {
|
|
3410
|
+
errorQueue.push(new AIError(AIErrorCode.FETCH_ERROR, 'WebSocket connection error.'));
|
|
3411
|
+
if (resolvePromise) {
|
|
3412
|
+
resolvePromise();
|
|
3413
|
+
resolvePromise = null;
|
|
3414
|
+
}
|
|
3415
|
+
};
|
|
3416
|
+
const closeListener = (event) => {
|
|
3417
|
+
if (event.reason) {
|
|
3418
|
+
logger.warn(`WebSocket connection closed by the server with reason: ${event.reason}`);
|
|
3419
|
+
}
|
|
3420
|
+
isClosed = true;
|
|
3421
|
+
if (resolvePromise) {
|
|
3422
|
+
resolvePromise();
|
|
3423
|
+
resolvePromise = null;
|
|
3424
|
+
}
|
|
3425
|
+
// Clean up listeners to prevent memory leaks
|
|
3426
|
+
this.ws?.removeEventListener('message', messageListener);
|
|
3427
|
+
this.ws?.removeEventListener('close', closeListener);
|
|
3428
|
+
this.ws?.removeEventListener('error', errorListener);
|
|
3429
|
+
};
|
|
3430
|
+
this.ws.addEventListener('message', messageListener);
|
|
3431
|
+
this.ws.addEventListener('close', closeListener);
|
|
3432
|
+
this.ws.addEventListener('error', errorListener);
|
|
3433
|
+
while (!isClosed) {
|
|
3434
|
+
if (errorQueue.length > 0) {
|
|
3435
|
+
const error = errorQueue.shift();
|
|
3436
|
+
throw error;
|
|
3437
|
+
}
|
|
3438
|
+
if (messageQueue.length > 0) {
|
|
3439
|
+
yield messageQueue.shift();
|
|
3440
|
+
}
|
|
3441
|
+
else {
|
|
3442
|
+
await new Promise(resolve => {
|
|
3443
|
+
resolvePromise = resolve;
|
|
3444
|
+
});
|
|
3445
|
+
}
|
|
3446
|
+
}
|
|
3447
|
+
// If the loop terminated because isClosed is true, check for any final errors
|
|
3448
|
+
if (errorQueue.length > 0) {
|
|
3449
|
+
const error = errorQueue.shift();
|
|
3450
|
+
throw error;
|
|
3451
|
+
}
|
|
3452
|
+
}
|
|
3453
|
+
close(code, reason) {
|
|
3454
|
+
return new Promise(resolve => {
|
|
3455
|
+
if (!this.ws) {
|
|
3456
|
+
return resolve();
|
|
3457
|
+
}
|
|
3458
|
+
this.ws.addEventListener('close', () => resolve(), { once: true });
|
|
3459
|
+
// Calling 'close' during these states results in an error.
|
|
3460
|
+
if (this.ws.readyState === WebSocket.CLOSED ||
|
|
3461
|
+
this.ws.readyState === WebSocket.CONNECTING) {
|
|
3462
|
+
return resolve();
|
|
3463
|
+
}
|
|
3464
|
+
if (this.ws.readyState !== WebSocket.CLOSING) {
|
|
3465
|
+
this.ws.close(code, reason);
|
|
3466
|
+
}
|
|
3467
|
+
});
|
|
3468
|
+
}
|
|
3469
|
+
}
|
|
3470
|
+
|
|
3235
3471
|
/**
|
|
3236
3472
|
* @license
|
|
3237
3473
|
* Copyright 2025 Google LLC
|
|
@@ -3259,9 +3495,10 @@ class LiveSession {
|
|
|
3259
3495
|
/**
|
|
3260
3496
|
* @internal
|
|
3261
3497
|
*/
|
|
3262
|
-
constructor(
|
|
3263
|
-
this.
|
|
3264
|
-
this.
|
|
3498
|
+
constructor(_setupMessage, _apiSettings, _sessionResumption, webSocketHandler) {
|
|
3499
|
+
this._setupMessage = _setupMessage;
|
|
3500
|
+
this._apiSettings = _apiSettings;
|
|
3501
|
+
this._sessionResumption = _sessionResumption;
|
|
3265
3502
|
/**
|
|
3266
3503
|
* Indicates whether this Live session is closed.
|
|
3267
3504
|
*
|
|
@@ -3274,6 +3511,64 @@ class LiveSession {
|
|
|
3274
3511
|
* @beta
|
|
3275
3512
|
*/
|
|
3276
3513
|
this.inConversation = false;
|
|
3514
|
+
/**
|
|
3515
|
+
* Generator yielding WebSocket messages from the server.
|
|
3516
|
+
*/
|
|
3517
|
+
this._serverMessages = null;
|
|
3518
|
+
this._webSocketHandler = webSocketHandler || new WebSocketHandlerImpl();
|
|
3519
|
+
this.connectionPromise = this._connectSession(this._sessionResumption);
|
|
3520
|
+
}
|
|
3521
|
+
/**
|
|
3522
|
+
* Initializes connection to the WebSocket. Should be called immediately
|
|
3523
|
+
* after instantiation.
|
|
3524
|
+
*
|
|
3525
|
+
* @internal
|
|
3526
|
+
*/
|
|
3527
|
+
async _connectSession(sessionResumption) {
|
|
3528
|
+
const url = new WebSocketUrl(this._apiSettings);
|
|
3529
|
+
await this._webSocketHandler.connect(url.toString());
|
|
3530
|
+
try {
|
|
3531
|
+
// Begin listening for server messages, and begin the handshake by sending the 'setupMessage'
|
|
3532
|
+
this._serverMessages = this._webSocketHandler.listen();
|
|
3533
|
+
const setupMessage = { ...this._setupMessage };
|
|
3534
|
+
if (sessionResumption) {
|
|
3535
|
+
setupMessage.setup.sessionResumption = sessionResumption;
|
|
3536
|
+
}
|
|
3537
|
+
this._webSocketHandler.send(JSON.stringify(setupMessage));
|
|
3538
|
+
// Verify we received the handshake response 'setupComplete'
|
|
3539
|
+
const firstMessage = (await this._serverMessages.next()).value;
|
|
3540
|
+
if (!firstMessage ||
|
|
3541
|
+
!(typeof firstMessage === 'object') ||
|
|
3542
|
+
!('setupComplete' in firstMessage)) {
|
|
3543
|
+
await this._webSocketHandler.close(1011, 'Handshake failure');
|
|
3544
|
+
throw new AIError(AIErrorCode.RESPONSE_ERROR, 'Server connection handshake failed. The server did not respond with a setupComplete message.');
|
|
3545
|
+
}
|
|
3546
|
+
this.isClosed = false;
|
|
3547
|
+
}
|
|
3548
|
+
catch (e) {
|
|
3549
|
+
// Ensure connection is closed on any setup error
|
|
3550
|
+
await this._webSocketHandler.close();
|
|
3551
|
+
throw e;
|
|
3552
|
+
}
|
|
3553
|
+
}
|
|
3554
|
+
/**
|
|
3555
|
+
* Resumes an existing live session with the server.
|
|
3556
|
+
*
|
|
3557
|
+
* This closes the current WebSocket connection and establishes a new one using
|
|
3558
|
+
* the same configuration (URI, headers, model, system instruction, tools, etc.)
|
|
3559
|
+
* as the original session.
|
|
3560
|
+
*
|
|
3561
|
+
* @param sessionResumption - The configuration for session resumption, such as the handle to the previous session state to restore.
|
|
3562
|
+
* @throws If the session resumption configuration is unsupported.
|
|
3563
|
+
*
|
|
3564
|
+
* @beta
|
|
3565
|
+
*/
|
|
3566
|
+
async resumeSession(sessionResumption) {
|
|
3567
|
+
if (!this._sessionResumption) {
|
|
3568
|
+
throw new AIError(AIErrorCode.UNSUPPORTED, 'Cannot resume session: no sessionResumption config provided');
|
|
3569
|
+
}
|
|
3570
|
+
await this.close();
|
|
3571
|
+
await this._connectSession(sessionResumption);
|
|
3277
3572
|
}
|
|
3278
3573
|
/**
|
|
3279
3574
|
* Sends content to the server.
|
|
@@ -3295,7 +3590,7 @@ class LiveSession {
|
|
|
3295
3590
|
turnComplete
|
|
3296
3591
|
}
|
|
3297
3592
|
};
|
|
3298
|
-
this.
|
|
3593
|
+
this._webSocketHandler.send(JSON.stringify(message));
|
|
3299
3594
|
}
|
|
3300
3595
|
/**
|
|
3301
3596
|
* Sends text to the server in realtime.
|
|
@@ -3319,7 +3614,7 @@ class LiveSession {
|
|
|
3319
3614
|
text
|
|
3320
3615
|
}
|
|
3321
3616
|
};
|
|
3322
|
-
this.
|
|
3617
|
+
this._webSocketHandler.send(JSON.stringify(message));
|
|
3323
3618
|
}
|
|
3324
3619
|
/**
|
|
3325
3620
|
* Sends audio data to the server in realtime.
|
|
@@ -3348,7 +3643,7 @@ class LiveSession {
|
|
|
3348
3643
|
audio: blob
|
|
3349
3644
|
}
|
|
3350
3645
|
};
|
|
3351
|
-
this.
|
|
3646
|
+
this._webSocketHandler.send(JSON.stringify(message));
|
|
3352
3647
|
}
|
|
3353
3648
|
/**
|
|
3354
3649
|
* Sends video data to the server in realtime.
|
|
@@ -3376,7 +3671,7 @@ class LiveSession {
|
|
|
3376
3671
|
video: blob
|
|
3377
3672
|
}
|
|
3378
3673
|
};
|
|
3379
|
-
this.
|
|
3674
|
+
this._webSocketHandler.send(JSON.stringify(message));
|
|
3380
3675
|
}
|
|
3381
3676
|
/**
|
|
3382
3677
|
* Sends function responses to the server.
|
|
@@ -3395,7 +3690,7 @@ class LiveSession {
|
|
|
3395
3690
|
functionResponses
|
|
3396
3691
|
}
|
|
3397
3692
|
};
|
|
3398
|
-
this.
|
|
3693
|
+
this._webSocketHandler.send(JSON.stringify(message));
|
|
3399
3694
|
}
|
|
3400
3695
|
/**
|
|
3401
3696
|
* Yields messages received from the server.
|
|
@@ -3410,42 +3705,50 @@ class LiveSession {
|
|
|
3410
3705
|
if (this.isClosed) {
|
|
3411
3706
|
throw new AIError(AIErrorCode.SESSION_CLOSED, 'Cannot read from a Live session that is closed. Try starting a new Live session.');
|
|
3412
3707
|
}
|
|
3413
|
-
|
|
3414
|
-
|
|
3415
|
-
if (
|
|
3416
|
-
|
|
3417
|
-
|
|
3418
|
-
|
|
3419
|
-
|
|
3420
|
-
|
|
3421
|
-
|
|
3422
|
-
|
|
3423
|
-
|
|
3424
|
-
|
|
3425
|
-
|
|
3426
|
-
|
|
3427
|
-
|
|
3428
|
-
|
|
3429
|
-
|
|
3430
|
-
|
|
3431
|
-
|
|
3432
|
-
|
|
3433
|
-
|
|
3434
|
-
|
|
3435
|
-
|
|
3436
|
-
|
|
3437
|
-
|
|
3438
|
-
|
|
3439
|
-
|
|
3440
|
-
|
|
3708
|
+
if (this._serverMessages) {
|
|
3709
|
+
for await (const message of this._serverMessages) {
|
|
3710
|
+
if (message && typeof message === 'object') {
|
|
3711
|
+
if (LiveResponseType.SERVER_CONTENT in message) {
|
|
3712
|
+
yield {
|
|
3713
|
+
type: 'serverContent',
|
|
3714
|
+
...message
|
|
3715
|
+
.serverContent
|
|
3716
|
+
};
|
|
3717
|
+
}
|
|
3718
|
+
else if (LiveResponseType.TOOL_CALL in message) {
|
|
3719
|
+
yield {
|
|
3720
|
+
type: 'toolCall',
|
|
3721
|
+
...message
|
|
3722
|
+
.toolCall
|
|
3723
|
+
};
|
|
3724
|
+
}
|
|
3725
|
+
else if (LiveResponseType.TOOL_CALL_CANCELLATION in message) {
|
|
3726
|
+
yield {
|
|
3727
|
+
type: 'toolCallCancellation',
|
|
3728
|
+
...message.toolCallCancellation
|
|
3729
|
+
};
|
|
3730
|
+
}
|
|
3731
|
+
else if ('goAway' in message) {
|
|
3732
|
+
const notice = message.goAway;
|
|
3733
|
+
yield {
|
|
3734
|
+
type: LiveResponseType.GOING_AWAY_NOTICE,
|
|
3735
|
+
timeLeft: parseDuration(notice.timeLeft)
|
|
3736
|
+
};
|
|
3737
|
+
}
|
|
3738
|
+
else if (LiveResponseType.SESSION_RESUMPTION_UPDATE in message) {
|
|
3739
|
+
yield {
|
|
3740
|
+
type: LiveResponseType.SESSION_RESUMPTION_UPDATE,
|
|
3741
|
+
...message.sessionResumptionUpdate
|
|
3742
|
+
};
|
|
3743
|
+
}
|
|
3744
|
+
else {
|
|
3745
|
+
logger.warn(`Received an unknown message type from the server: ${JSON.stringify(message)}`);
|
|
3746
|
+
}
|
|
3441
3747
|
}
|
|
3442
3748
|
else {
|
|
3443
|
-
logger.warn(`Received an
|
|
3749
|
+
logger.warn(`Received an invalid message from the server: ${JSON.stringify(message)}`);
|
|
3444
3750
|
}
|
|
3445
3751
|
}
|
|
3446
|
-
else {
|
|
3447
|
-
logger.warn(`Received an invalid message from the server: ${JSON.stringify(message)}`);
|
|
3448
|
-
}
|
|
3449
3752
|
}
|
|
3450
3753
|
}
|
|
3451
3754
|
/**
|
|
@@ -3457,7 +3760,7 @@ class LiveSession {
|
|
|
3457
3760
|
async close() {
|
|
3458
3761
|
if (!this.isClosed) {
|
|
3459
3762
|
this.isClosed = true;
|
|
3460
|
-
await this.
|
|
3763
|
+
await this._webSocketHandler.close(1000, 'Client closed session.');
|
|
3461
3764
|
}
|
|
3462
3765
|
}
|
|
3463
3766
|
/**
|
|
@@ -3480,7 +3783,7 @@ class LiveSession {
|
|
|
3480
3783
|
const message = {
|
|
3481
3784
|
realtimeInput: { mediaChunks: [mediaChunk] }
|
|
3482
3785
|
};
|
|
3483
|
-
this.
|
|
3786
|
+
this._webSocketHandler.send(JSON.stringify(message));
|
|
3484
3787
|
});
|
|
3485
3788
|
}
|
|
3486
3789
|
/**
|
|
@@ -3560,6 +3863,7 @@ class LiveGenerativeModel extends AIModel {
|
|
|
3560
3863
|
*/
|
|
3561
3864
|
constructor(ai, modelParams,
|
|
3562
3865
|
/**
|
|
3866
|
+
* For testing injection
|
|
3563
3867
|
* @internal
|
|
3564
3868
|
*/
|
|
3565
3869
|
_webSocketHandler) {
|
|
@@ -3578,9 +3882,7 @@ class LiveGenerativeModel extends AIModel {
|
|
|
3578
3882
|
*
|
|
3579
3883
|
* @beta
|
|
3580
3884
|
*/
|
|
3581
|
-
async connect() {
|
|
3582
|
-
const url = new WebSocketUrl(this._apiSettings);
|
|
3583
|
-
await this._webSocketHandler.connect(url.toString());
|
|
3885
|
+
async connect(sessionResumption) {
|
|
3584
3886
|
let fullModelPath;
|
|
3585
3887
|
if (this._apiSettings.backend.backendType === BackendType.GOOGLE_AI) {
|
|
3586
3888
|
fullModelPath = `projects/${this._apiSettings.project}/${this.model}`;
|
|
@@ -3591,36 +3893,24 @@ class LiveGenerativeModel extends AIModel {
|
|
|
3591
3893
|
// inputAudioTranscription and outputAudioTranscription are on the generation config in the public API,
|
|
3592
3894
|
// but the backend expects them to be in the `setup` message.
|
|
3593
3895
|
const { inputAudioTranscription, outputAudioTranscription, ...generationConfig } = this.generationConfig;
|
|
3896
|
+
const contextWindowCompression = generationConfig.contextWindowCompression;
|
|
3897
|
+
delete generationConfig.contextWindowCompression;
|
|
3594
3898
|
const setupMessage = {
|
|
3595
3899
|
setup: {
|
|
3596
3900
|
model: fullModelPath,
|
|
3597
3901
|
generationConfig,
|
|
3902
|
+
contextWindowCompression,
|
|
3598
3903
|
tools: this.tools,
|
|
3599
3904
|
toolConfig: this.toolConfig,
|
|
3600
3905
|
systemInstruction: this.systemInstruction,
|
|
3601
3906
|
inputAudioTranscription,
|
|
3602
|
-
outputAudioTranscription
|
|
3907
|
+
outputAudioTranscription,
|
|
3908
|
+
sessionResumption
|
|
3603
3909
|
}
|
|
3604
3910
|
};
|
|
3605
|
-
|
|
3606
|
-
|
|
3607
|
-
|
|
3608
|
-
this._webSocketHandler.send(JSON.stringify(setupMessage));
|
|
3609
|
-
// Verify we received the handshake response 'setupComplete'
|
|
3610
|
-
const firstMessage = (await serverMessages.next()).value;
|
|
3611
|
-
if (!firstMessage ||
|
|
3612
|
-
!(typeof firstMessage === 'object') ||
|
|
3613
|
-
!('setupComplete' in firstMessage)) {
|
|
3614
|
-
await this._webSocketHandler.close(1011, 'Handshake failure');
|
|
3615
|
-
throw new AIError(AIErrorCode.RESPONSE_ERROR, 'Server connection handshake failed. The server did not respond with a setupComplete message.');
|
|
3616
|
-
}
|
|
3617
|
-
return new LiveSession(this._webSocketHandler, serverMessages);
|
|
3618
|
-
}
|
|
3619
|
-
catch (e) {
|
|
3620
|
-
// Ensure connection is closed on any setup error
|
|
3621
|
-
await this._webSocketHandler.close();
|
|
3622
|
-
throw e;
|
|
3623
|
-
}
|
|
3911
|
+
const session = new LiveSession(setupMessage, this._apiSettings, sessionResumption, this._webSocketHandler);
|
|
3912
|
+
await session.connectionPromise;
|
|
3913
|
+
return session;
|
|
3624
3914
|
}
|
|
3625
3915
|
}
|
|
3626
3916
|
|
|
@@ -3761,153 +4051,6 @@ class ImagenModel extends AIModel {
|
|
|
3761
4051
|
}
|
|
3762
4052
|
}
|
|
3763
4053
|
|
|
3764
|
-
/**
|
|
3765
|
-
* @license
|
|
3766
|
-
* Copyright 2025 Google LLC
|
|
3767
|
-
*
|
|
3768
|
-
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
3769
|
-
* you may not use this file except in compliance with the License.
|
|
3770
|
-
* You may obtain a copy of the License at
|
|
3771
|
-
*
|
|
3772
|
-
* http://www.apache.org/licenses/LICENSE-2.0
|
|
3773
|
-
*
|
|
3774
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
3775
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
3776
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
3777
|
-
* See the License for the specific language governing permissions and
|
|
3778
|
-
* limitations under the License.
|
|
3779
|
-
*/
|
|
3780
|
-
/**
|
|
3781
|
-
* A wrapper for the native `WebSocket` available in both Browsers and Node >= 22.
|
|
3782
|
-
*
|
|
3783
|
-
* @internal
|
|
3784
|
-
*/
|
|
3785
|
-
class WebSocketHandlerImpl {
|
|
3786
|
-
constructor() {
|
|
3787
|
-
if (typeof WebSocket === 'undefined') {
|
|
3788
|
-
throw new AIError(AIErrorCode.UNSUPPORTED, 'The WebSocket API is not available in this environment. ' +
|
|
3789
|
-
'The "Live" feature is not supported here. It is supported in ' +
|
|
3790
|
-
'modern browser windows, Web Workers with WebSocket support, and Node >= 22.');
|
|
3791
|
-
}
|
|
3792
|
-
}
|
|
3793
|
-
connect(url) {
|
|
3794
|
-
return new Promise((resolve, reject) => {
|
|
3795
|
-
this.ws = new WebSocket(url);
|
|
3796
|
-
this.ws.binaryType = 'blob'; // Only important to set in Node
|
|
3797
|
-
this.ws.addEventListener('open', () => resolve(), { once: true });
|
|
3798
|
-
this.ws.addEventListener('error', () => reject(new AIError(AIErrorCode.FETCH_ERROR, `Error event raised on WebSocket`)), { once: true });
|
|
3799
|
-
this.ws.addEventListener('close', (closeEvent) => {
|
|
3800
|
-
if (closeEvent.reason) {
|
|
3801
|
-
logger.warn(`WebSocket connection closed by server. Reason: '${closeEvent.reason}'`);
|
|
3802
|
-
}
|
|
3803
|
-
});
|
|
3804
|
-
});
|
|
3805
|
-
}
|
|
3806
|
-
send(data) {
|
|
3807
|
-
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
|
|
3808
|
-
throw new AIError(AIErrorCode.REQUEST_ERROR, 'WebSocket is not open.');
|
|
3809
|
-
}
|
|
3810
|
-
this.ws.send(data);
|
|
3811
|
-
}
|
|
3812
|
-
async *listen() {
|
|
3813
|
-
if (!this.ws) {
|
|
3814
|
-
throw new AIError(AIErrorCode.REQUEST_ERROR, 'WebSocket is not connected.');
|
|
3815
|
-
}
|
|
3816
|
-
const messageQueue = [];
|
|
3817
|
-
const errorQueue = [];
|
|
3818
|
-
let resolvePromise = null;
|
|
3819
|
-
let isClosed = false;
|
|
3820
|
-
const messageListener = async (event) => {
|
|
3821
|
-
let data;
|
|
3822
|
-
if (event.data instanceof Blob) {
|
|
3823
|
-
data = await event.data.text();
|
|
3824
|
-
}
|
|
3825
|
-
else if (typeof event.data === 'string') {
|
|
3826
|
-
data = event.data;
|
|
3827
|
-
}
|
|
3828
|
-
else {
|
|
3829
|
-
errorQueue.push(new AIError(AIErrorCode.PARSE_FAILED, `Failed to parse WebSocket response. Expected data to be a Blob or string, but was ${typeof event.data}.`));
|
|
3830
|
-
if (resolvePromise) {
|
|
3831
|
-
resolvePromise();
|
|
3832
|
-
resolvePromise = null;
|
|
3833
|
-
}
|
|
3834
|
-
return;
|
|
3835
|
-
}
|
|
3836
|
-
try {
|
|
3837
|
-
const obj = JSON.parse(data);
|
|
3838
|
-
messageQueue.push(obj);
|
|
3839
|
-
}
|
|
3840
|
-
catch (e) {
|
|
3841
|
-
const err = e;
|
|
3842
|
-
errorQueue.push(new AIError(AIErrorCode.PARSE_FAILED, `Error parsing WebSocket message to JSON: ${err.message}`));
|
|
3843
|
-
}
|
|
3844
|
-
if (resolvePromise) {
|
|
3845
|
-
resolvePromise();
|
|
3846
|
-
resolvePromise = null;
|
|
3847
|
-
}
|
|
3848
|
-
};
|
|
3849
|
-
const errorListener = () => {
|
|
3850
|
-
errorQueue.push(new AIError(AIErrorCode.FETCH_ERROR, 'WebSocket connection error.'));
|
|
3851
|
-
if (resolvePromise) {
|
|
3852
|
-
resolvePromise();
|
|
3853
|
-
resolvePromise = null;
|
|
3854
|
-
}
|
|
3855
|
-
};
|
|
3856
|
-
const closeListener = (event) => {
|
|
3857
|
-
if (event.reason) {
|
|
3858
|
-
logger.warn(`WebSocket connection closed by the server with reason: ${event.reason}`);
|
|
3859
|
-
}
|
|
3860
|
-
isClosed = true;
|
|
3861
|
-
if (resolvePromise) {
|
|
3862
|
-
resolvePromise();
|
|
3863
|
-
resolvePromise = null;
|
|
3864
|
-
}
|
|
3865
|
-
// Clean up listeners to prevent memory leaks
|
|
3866
|
-
this.ws?.removeEventListener('message', messageListener);
|
|
3867
|
-
this.ws?.removeEventListener('close', closeListener);
|
|
3868
|
-
this.ws?.removeEventListener('error', errorListener);
|
|
3869
|
-
};
|
|
3870
|
-
this.ws.addEventListener('message', messageListener);
|
|
3871
|
-
this.ws.addEventListener('close', closeListener);
|
|
3872
|
-
this.ws.addEventListener('error', errorListener);
|
|
3873
|
-
while (!isClosed) {
|
|
3874
|
-
if (errorQueue.length > 0) {
|
|
3875
|
-
const error = errorQueue.shift();
|
|
3876
|
-
throw error;
|
|
3877
|
-
}
|
|
3878
|
-
if (messageQueue.length > 0) {
|
|
3879
|
-
yield messageQueue.shift();
|
|
3880
|
-
}
|
|
3881
|
-
else {
|
|
3882
|
-
await new Promise(resolve => {
|
|
3883
|
-
resolvePromise = resolve;
|
|
3884
|
-
});
|
|
3885
|
-
}
|
|
3886
|
-
}
|
|
3887
|
-
// If the loop terminated because isClosed is true, check for any final errors
|
|
3888
|
-
if (errorQueue.length > 0) {
|
|
3889
|
-
const error = errorQueue.shift();
|
|
3890
|
-
throw error;
|
|
3891
|
-
}
|
|
3892
|
-
}
|
|
3893
|
-
close(code, reason) {
|
|
3894
|
-
return new Promise(resolve => {
|
|
3895
|
-
if (!this.ws) {
|
|
3896
|
-
return resolve();
|
|
3897
|
-
}
|
|
3898
|
-
this.ws.addEventListener('close', () => resolve(), { once: true });
|
|
3899
|
-
// Calling 'close' during these states results in an error.
|
|
3900
|
-
if (this.ws.readyState === WebSocket.CLOSED ||
|
|
3901
|
-
this.ws.readyState === WebSocket.CONNECTING) {
|
|
3902
|
-
return resolve();
|
|
3903
|
-
}
|
|
3904
|
-
if (this.ws.readyState !== WebSocket.CLOSING) {
|
|
3905
|
-
this.ws.close(code, reason);
|
|
3906
|
-
}
|
|
3907
|
-
});
|
|
3908
|
-
}
|
|
3909
|
-
}
|
|
3910
|
-
|
|
3911
4054
|
/**
|
|
3912
4055
|
* @license
|
|
3913
4056
|
* Copyright 2026 Google LLC
|
|
@@ -4052,11 +4195,16 @@ class TemplateGenerativeModel {
|
|
|
4052
4195
|
* @param templateId - The ID of the server-side template to execute.
|
|
4053
4196
|
* @param templateVariables - A key-value map of variables to populate the
|
|
4054
4197
|
* template with.
|
|
4198
|
+
* @param singleRequestOptions - Optional. Options to use for this request.
|
|
4199
|
+
* @param templateToolConfig - Optional. Configuration for tools to use with this request.
|
|
4055
4200
|
*
|
|
4056
4201
|
* @beta
|
|
4057
4202
|
*/
|
|
4058
|
-
async generateContent(templateId, templateVariables, singleRequestOptions) {
|
|
4059
|
-
return templateGenerateContent(this._apiSettings, templateId, {
|
|
4203
|
+
async generateContent(templateId, templateVariables, singleRequestOptions, templateToolConfig) {
|
|
4204
|
+
return templateGenerateContent(this._apiSettings, templateId, {
|
|
4205
|
+
inputs: templateVariables,
|
|
4206
|
+
...(templateToolConfig && { toolConfig: templateToolConfig })
|
|
4207
|
+
}, {
|
|
4060
4208
|
...this.requestOptions,
|
|
4061
4209
|
...singleRequestOptions
|
|
4062
4210
|
});
|
|
@@ -4070,11 +4218,16 @@ class TemplateGenerativeModel {
|
|
|
4070
4218
|
* @param templateId - The ID of the server-side template to execute.
|
|
4071
4219
|
* @param templateVariables - A key-value map of variables to populate the
|
|
4072
4220
|
* template with.
|
|
4221
|
+
* @param singleRequestOptions - Optional.Options to use for this request.
|
|
4222
|
+
* @param templateToolConfig - Optional. Configuration for tools to use with this request.
|
|
4073
4223
|
*
|
|
4074
4224
|
* @beta
|
|
4075
4225
|
*/
|
|
4076
|
-
async generateContentStream(templateId, templateVariables, singleRequestOptions) {
|
|
4077
|
-
return templateGenerateContentStream(this._apiSettings, templateId, {
|
|
4226
|
+
async generateContentStream(templateId, templateVariables, singleRequestOptions, templateToolConfig) {
|
|
4227
|
+
return templateGenerateContentStream(this._apiSettings, templateId, {
|
|
4228
|
+
inputs: templateVariables,
|
|
4229
|
+
...(templateToolConfig && { toolConfig: templateToolConfig })
|
|
4230
|
+
}, {
|
|
4078
4231
|
...this.requestOptions,
|
|
4079
4232
|
...singleRequestOptions
|
|
4080
4233
|
});
|
|
@@ -4967,8 +5120,7 @@ function getLiveGenerativeModel(ai, modelParams) {
|
|
|
4967
5120
|
if (!modelParams.model) {
|
|
4968
5121
|
throw new AIError(AIErrorCode.NO_MODEL, `Must provide a model name for getLiveGenerativeModel. Example: getLiveGenerativeModel(ai, { model: 'my-model-name' })`);
|
|
4969
5122
|
}
|
|
4970
|
-
|
|
4971
|
-
return new LiveGenerativeModel(ai, modelParams, webSocketHandler);
|
|
5123
|
+
return new LiveGenerativeModel(ai, modelParams);
|
|
4972
5124
|
}
|
|
4973
5125
|
/**
|
|
4974
5126
|
* Returns a {@link TemplateGenerativeModel} class for executing server-side
|
|
@@ -5011,5 +5163,5 @@ function registerAI() {
|
|
|
5011
5163
|
}
|
|
5012
5164
|
registerAI();
|
|
5013
5165
|
|
|
5014
|
-
export { AIError, AIErrorCode, AIModel, AnyOfSchema, ArraySchema, Backend, BackendType, BlockReason, BooleanSchema, ChatSession, ChatSessionBase, FinishReason, FunctionCallingMode, GenerativeModel, GoogleAIBackend, HarmBlockMethod, HarmBlockThreshold, HarmCategory, HarmProbability, HarmSeverity, ImagenAspectRatio, ImagenImageFormat, ImagenModel, ImagenPersonFilterLevel, ImagenSafetyFilterLevel, InferenceMode, InferenceSource, IntegerSchema, Language, LiveGenerativeModel, LiveResponseType, LiveSession, Modality, NumberSchema, ObjectSchema, Outcome, POSSIBLE_ROLES, ResponseModality, Schema, SchemaType, StringSchema, TemplateGenerativeModel, TemplateImagenModel, ThinkingLevel, URLRetrievalStatus, VertexAIBackend, getAI, getGenerativeModel, getImagenModel, getLiveGenerativeModel, getTemplateGenerativeModel, getTemplateImagenModel, startAudioConversation };
|
|
5166
|
+
export { AIError, AIErrorCode, AIModel, AnyOfSchema, ArraySchema, Backend, BackendType, BlockReason, BooleanSchema, ChatSession, ChatSessionBase, FinishReason, FunctionCallingMode, GenerativeModel, GoogleAIBackend, HarmBlockMethod, HarmBlockThreshold, HarmCategory, HarmProbability, HarmSeverity, ImageConfigAspectRatio, ImageConfigImageSize, ImagenAspectRatio, ImagenImageFormat, ImagenModel, ImagenPersonFilterLevel, ImagenSafetyFilterLevel, InferenceMode, InferenceSource, IntegerSchema, Language, LiveGenerativeModel, LiveResponseType, LiveSession, Modality, NumberSchema, ObjectSchema, Outcome, POSSIBLE_ROLES, ResponseModality, Schema, SchemaType, StringSchema, TemplateGenerativeModel, TemplateImagenModel, ThinkingLevel, URLRetrievalStatus, VertexAIBackend, getAI, getGenerativeModel, getImagenModel, getLiveGenerativeModel, getTemplateGenerativeModel, getTemplateImagenModel, startAudioConversation };
|
|
5015
5167
|
//# sourceMappingURL=index.esm.js.map
|