@firebase/ai 2.3.0 → 2.4.0-canary.22e0a1adb

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/dist/ai-public.d.ts +240 -35
  2. package/dist/ai.d.ts +241 -35
  3. package/dist/esm/index.esm.js +295 -92
  4. package/dist/esm/index.esm.js.map +1 -1
  5. package/dist/esm/src/api.d.ts +1 -1
  6. package/dist/esm/src/methods/chrome-adapter.d.ts +1 -1
  7. package/dist/esm/src/methods/live-session-helpers.d.ts +2 -2
  8. package/dist/esm/src/methods/live-session.d.ts +71 -7
  9. package/dist/esm/src/models/imagen-model.d.ts +2 -2
  10. package/dist/esm/src/requests/hybrid-helpers.d.ts +7 -2
  11. package/dist/esm/src/requests/imagen-image-format.d.ts +3 -3
  12. package/dist/esm/src/requests/response-helpers.d.ts +2 -2
  13. package/dist/esm/src/requests/stream-reader.d.ts +2 -1
  14. package/dist/esm/src/types/content.d.ts +4 -4
  15. package/dist/esm/src/types/enums.d.ts +19 -4
  16. package/dist/esm/src/types/googleai.d.ts +2 -1
  17. package/dist/esm/src/types/imagen/requests.d.ts +9 -9
  18. package/dist/esm/src/types/imagen/responses.d.ts +3 -3
  19. package/dist/esm/src/types/live-responses.d.ts +16 -2
  20. package/dist/esm/src/types/requests.d.ts +22 -2
  21. package/dist/esm/src/types/responses.d.ts +99 -1
  22. package/dist/index.cjs.js +296 -91
  23. package/dist/index.cjs.js.map +1 -1
  24. package/dist/index.node.cjs.js +276 -84
  25. package/dist/index.node.cjs.js.map +1 -1
  26. package/dist/index.node.mjs +275 -85
  27. package/dist/index.node.mjs.map +1 -1
  28. package/dist/src/api.d.ts +1 -1
  29. package/dist/src/methods/chrome-adapter.d.ts +1 -1
  30. package/dist/src/methods/live-session-helpers.d.ts +2 -2
  31. package/dist/src/methods/live-session.d.ts +71 -7
  32. package/dist/src/models/imagen-model.d.ts +2 -2
  33. package/dist/src/requests/hybrid-helpers.d.ts +7 -2
  34. package/dist/src/requests/imagen-image-format.d.ts +3 -3
  35. package/dist/src/requests/response-helpers.d.ts +2 -2
  36. package/dist/src/requests/stream-reader.d.ts +2 -1
  37. package/dist/src/types/content.d.ts +4 -4
  38. package/dist/src/types/enums.d.ts +19 -4
  39. package/dist/src/types/googleai.d.ts +2 -1
  40. package/dist/src/types/imagen/requests.d.ts +9 -9
  41. package/dist/src/types/imagen/responses.d.ts +3 -3
  42. package/dist/src/types/live-responses.d.ts +16 -2
  43. package/dist/src/types/requests.d.ts +22 -2
  44. package/dist/src/types/responses.d.ts +99 -1
  45. package/package.json +8 -8
@@ -4,7 +4,7 @@ import { FirebaseError, Deferred, getModularInstance } from '@firebase/util';
4
4
  import { Logger } from '@firebase/logger';
5
5
 
6
6
  var name = "@firebase/ai";
7
- var version = "2.3.0";
7
+ var version = "2.4.0-canary.22e0a1adb";
8
8
 
9
9
  /**
10
10
  * @license
@@ -323,10 +323,19 @@ const InferenceMode = {
323
323
  'ONLY_IN_CLOUD': 'only_in_cloud',
324
324
  'PREFER_IN_CLOUD': 'prefer_in_cloud'
325
325
  };
326
+ /**
327
+ * Indicates whether inference happened on-device or in-cloud.
328
+ *
329
+ * @beta
330
+ */
331
+ const InferenceSource = {
332
+ 'ON_DEVICE': 'on_device',
333
+ 'IN_CLOUD': 'in_cloud'
334
+ };
326
335
  /**
327
336
  * Represents the result of the code execution.
328
337
  *
329
- * @public
338
+ * @beta
330
339
  */
331
340
  const Outcome = {
332
341
  UNSPECIFIED: 'OUTCOME_UNSPECIFIED',
@@ -337,7 +346,7 @@ const Outcome = {
337
346
  /**
338
347
  * The programming language of the code.
339
348
  *
340
- * @public
349
+ * @beta
341
350
  */
342
351
  const Language = {
343
352
  UNSPECIFIED: 'LANGUAGE_UNSPECIFIED',
@@ -360,6 +369,45 @@ const Language = {
360
369
  * See the License for the specific language governing permissions and
361
370
  * limitations under the License.
362
371
  */
372
+ /**
373
+ * The status of a URL retrieval.
374
+ *
375
+ * @remarks
376
+ * <b>URL_RETRIEVAL_STATUS_UNSPECIFIED:</b> Unspecified retrieval status.
377
+ * <br/>
378
+ * <b>URL_RETRIEVAL_STATUS_SUCCESS:</b> The URL retrieval was successful.
379
+ * <br/>
380
+ * <b>URL_RETRIEVAL_STATUS_ERROR:</b> The URL retrieval failed.
381
+ * <br/>
382
+ * <b>URL_RETRIEVAL_STATUS_PAYWALL:</b> The URL retrieval failed because the content is behind a paywall.
383
+ * <br/>
384
+ * <b>URL_RETRIEVAL_STATUS_UNSAFE:</b> The URL retrieval failed because the content is unsafe.
385
+ * <br/>
386
+ *
387
+ * @beta
388
+ */
389
+ const URLRetrievalStatus = {
390
+ /**
391
+ * Unspecified retrieval status.
392
+ */
393
+ URL_RETRIEVAL_STATUS_UNSPECIFIED: 'URL_RETRIEVAL_STATUS_UNSPECIFIED',
394
+ /**
395
+ * The URL retrieval was successful.
396
+ */
397
+ URL_RETRIEVAL_STATUS_SUCCESS: 'URL_RETRIEVAL_STATUS_SUCCESS',
398
+ /**
399
+ * The URL retrieval failed.
400
+ */
401
+ URL_RETRIEVAL_STATUS_ERROR: 'URL_RETRIEVAL_STATUS_ERROR',
402
+ /**
403
+ * The URL retrieval failed because the content is behind a paywall.
404
+ */
405
+ URL_RETRIEVAL_STATUS_PAYWALL: 'URL_RETRIEVAL_STATUS_PAYWALL',
406
+ /**
407
+ * The URL retrieval failed because the content is unsafe.
408
+ */
409
+ URL_RETRIEVAL_STATUS_UNSAFE: 'URL_RETRIEVAL_STATUS_UNSAFE'
410
+ };
363
411
  /**
364
412
  * The types of responses that can be returned by {@link LiveSession.receive}.
365
413
  *
@@ -486,7 +534,7 @@ const SchemaType = {
486
534
  * and the {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters | Responsible AI and usage guidelines}
487
535
  * for more details.
488
536
  *
489
- * @beta
537
+ * @public
490
538
  */
491
539
  const ImagenSafetyFilterLevel = {
492
540
  /**
@@ -515,7 +563,7 @@ const ImagenSafetyFilterLevel = {
515
563
  * See the <a href="http://firebase.google.com/docs/vertex-ai/generate-images">personGeneration</a>
516
564
  * documentation for more details.
517
565
  *
518
- * @beta
566
+ * @public
519
567
  */
520
568
  const ImagenPersonFilterLevel = {
521
569
  /**
@@ -548,7 +596,7 @@ const ImagenPersonFilterLevel = {
548
596
  * See the {@link http://firebase.google.com/docs/vertex-ai/generate-images | documentation }
549
597
  * for more details and examples of the supported aspect ratios.
550
598
  *
551
- * @beta
599
+ * @public
552
600
  */
553
601
  const ImagenAspectRatio = {
554
602
  /**
@@ -1230,7 +1278,7 @@ function hasValidCandidates(response) {
1230
1278
  * Creates an EnhancedGenerateContentResponse object that has helper functions and
1231
1279
  * other modifications that improve usability.
1232
1280
  */
1233
- function createEnhancedContentResponse(response) {
1281
+ function createEnhancedContentResponse(response, inferenceSource = InferenceSource.IN_CLOUD) {
1234
1282
  /**
1235
1283
  * The Vertex AI backend omits default values.
1236
1284
  * This causes the `index` property to be omitted from the first candidate in the
@@ -1241,6 +1289,7 @@ function createEnhancedContentResponse(response) {
1241
1289
  response.candidates[0].index = 0;
1242
1290
  }
1243
1291
  const responseWithHelpers = addHelpers(response);
1292
+ responseWithHelpers.inferenceSource = inferenceSource;
1244
1293
  return responseWithHelpers;
1245
1294
  }
1246
1295
  /**
@@ -1553,7 +1602,7 @@ function mapGenerateContentCandidates(candidates) {
1553
1602
  // videoMetadata is not supported.
1554
1603
  // Throw early since developers may send a long video as input and only expect to pay
1555
1604
  // for inference on a small portion of the video.
1556
- if (candidate.content?.parts.some(part => part?.videoMetadata)) {
1605
+ if (candidate.content?.parts?.some(part => part?.videoMetadata)) {
1557
1606
  throw new AIError(AIErrorCode.UNSUPPORTED, 'Part.videoMetadata is not supported in the Gemini Developer API. Please remove this property.');
1558
1607
  }
1559
1608
  const mappedCandidate = {
@@ -1563,7 +1612,8 @@ function mapGenerateContentCandidates(candidates) {
1563
1612
  finishMessage: candidate.finishMessage,
1564
1613
  safetyRatings: mappedSafetyRatings,
1565
1614
  citationMetadata,
1566
- groundingMetadata: candidate.groundingMetadata
1615
+ groundingMetadata: candidate.groundingMetadata,
1616
+ urlContextMetadata: candidate.urlContextMetadata
1567
1617
  };
1568
1618
  mappedCandidates.push(mappedCandidate);
1569
1619
  });
@@ -1616,16 +1666,16 @@ const responseLineRE = /^data\: (.*)(?:\n\n|\r\r|\r\n\r\n)/;
1616
1666
  *
1617
1667
  * @param response - Response from a fetch call
1618
1668
  */
1619
- function processStream(response, apiSettings) {
1669
+ function processStream(response, apiSettings, inferenceSource) {
1620
1670
  const inputStream = response.body.pipeThrough(new TextDecoderStream('utf8', { fatal: true }));
1621
1671
  const responseStream = getResponseStream(inputStream);
1622
1672
  const [stream1, stream2] = responseStream.tee();
1623
1673
  return {
1624
- stream: generateResponseSequence(stream1, apiSettings),
1625
- response: getResponsePromise(stream2, apiSettings)
1674
+ stream: generateResponseSequence(stream1, apiSettings, inferenceSource),
1675
+ response: getResponsePromise(stream2, apiSettings, inferenceSource)
1626
1676
  };
1627
1677
  }
1628
- async function getResponsePromise(stream, apiSettings) {
1678
+ async function getResponsePromise(stream, apiSettings, inferenceSource) {
1629
1679
  const allResponses = [];
1630
1680
  const reader = stream.getReader();
1631
1681
  while (true) {
@@ -1635,12 +1685,12 @@ async function getResponsePromise(stream, apiSettings) {
1635
1685
  if (apiSettings.backend.backendType === BackendType.GOOGLE_AI) {
1636
1686
  generateContentResponse = mapGenerateContentResponse(generateContentResponse);
1637
1687
  }
1638
- return createEnhancedContentResponse(generateContentResponse);
1688
+ return createEnhancedContentResponse(generateContentResponse, inferenceSource);
1639
1689
  }
1640
1690
  allResponses.push(value);
1641
1691
  }
1642
1692
  }
1643
- async function* generateResponseSequence(stream, apiSettings) {
1693
+ async function* generateResponseSequence(stream, apiSettings, inferenceSource) {
1644
1694
  const reader = stream.getReader();
1645
1695
  while (true) {
1646
1696
  const { value, done } = await reader.read();
@@ -1649,10 +1699,18 @@ async function* generateResponseSequence(stream, apiSettings) {
1649
1699
  }
1650
1700
  let enhancedResponse;
1651
1701
  if (apiSettings.backend.backendType === BackendType.GOOGLE_AI) {
1652
- enhancedResponse = createEnhancedContentResponse(mapGenerateContentResponse(value));
1702
+ enhancedResponse = createEnhancedContentResponse(mapGenerateContentResponse(value), inferenceSource);
1653
1703
  }
1654
1704
  else {
1655
- enhancedResponse = createEnhancedContentResponse(value);
1705
+ enhancedResponse = createEnhancedContentResponse(value, inferenceSource);
1706
+ }
1707
+ const firstCandidate = enhancedResponse.candidates?.[0];
1708
+ // Don't yield a response with no useful data for the developer.
1709
+ if (!firstCandidate?.content?.parts &&
1710
+ !firstCandidate?.finishReason &&
1711
+ !firstCandidate?.citationMetadata &&
1712
+ !firstCandidate?.urlContextMetadata) {
1713
+ continue;
1656
1714
  }
1657
1715
  yield enhancedResponse;
1658
1716
  }
@@ -1733,36 +1791,43 @@ function aggregateResponses(responses) {
1733
1791
  candidate.safetyRatings;
1734
1792
  aggregatedResponse.candidates[i].groundingMetadata =
1735
1793
  candidate.groundingMetadata;
1794
+ // The urlContextMetadata object is defined in the first chunk of the response stream.
1795
+ // In all subsequent chunks, the urlContextMetadata object will be undefined. We need to
1796
+ // make sure that we don't overwrite the first value urlContextMetadata object with undefined.
1797
+ // FIXME: What happens if we receive a second, valid urlContextMetadata object?
1798
+ const urlContextMetadata = candidate.urlContextMetadata;
1799
+ if (typeof urlContextMetadata === 'object' &&
1800
+ urlContextMetadata !== null &&
1801
+ Object.keys(urlContextMetadata).length > 0) {
1802
+ aggregatedResponse.candidates[i].urlContextMetadata =
1803
+ urlContextMetadata;
1804
+ }
1736
1805
  /**
1737
1806
  * Candidates should always have content and parts, but this handles
1738
1807
  * possible malformed responses.
1739
1808
  */
1740
- if (candidate.content && candidate.content.parts) {
1809
+ if (candidate.content) {
1810
+ // Skip a candidate without parts.
1811
+ if (!candidate.content.parts) {
1812
+ continue;
1813
+ }
1741
1814
  if (!aggregatedResponse.candidates[i].content) {
1742
1815
  aggregatedResponse.candidates[i].content = {
1743
1816
  role: candidate.content.role || 'user',
1744
1817
  parts: []
1745
1818
  };
1746
1819
  }
1747
- const newPart = {};
1748
1820
  for (const part of candidate.content.parts) {
1749
- if (part.text !== undefined) {
1750
- // The backend can send empty text parts. If these are sent back
1751
- // (e.g. in chat history), the backend will respond with an error.
1752
- // To prevent this, ignore empty text parts.
1753
- if (part.text === '') {
1754
- continue;
1755
- }
1756
- newPart.text = part.text;
1821
+ const newPart = { ...part };
1822
+ // The backend can send empty text parts. If these are sent back
1823
+ // (e.g. in chat history), the backend will respond with an error.
1824
+ // To prevent this, ignore empty text parts.
1825
+ if (part.text === '') {
1826
+ continue;
1757
1827
  }
1758
- if (part.functionCall) {
1759
- newPart.functionCall = part.functionCall;
1828
+ if (Object.keys(newPart).length > 0) {
1829
+ aggregatedResponse.candidates[i].content.parts.push(newPart);
1760
1830
  }
1761
- if (Object.keys(newPart).length === 0) {
1762
- throw new AIError(AIErrorCode.INVALID_CONTENT, 'Part should have at least one property, but there are none. This is likely caused ' +
1763
- 'by a malformed response from the backend.');
1764
- }
1765
- aggregatedResponse.candidates[i].content.parts.push(newPart);
1766
1831
  }
1767
1832
  }
1768
1833
  }
@@ -1807,31 +1872,52 @@ const errorsCausingFallback = [
1807
1872
  */
1808
1873
  async function callCloudOrDevice(request, chromeAdapter, onDeviceCall, inCloudCall) {
1809
1874
  if (!chromeAdapter) {
1810
- return inCloudCall();
1875
+ return {
1876
+ response: await inCloudCall(),
1877
+ inferenceSource: InferenceSource.IN_CLOUD
1878
+ };
1811
1879
  }
1812
1880
  switch (chromeAdapter.mode) {
1813
1881
  case InferenceMode.ONLY_ON_DEVICE:
1814
1882
  if (await chromeAdapter.isAvailable(request)) {
1815
- return onDeviceCall();
1883
+ return {
1884
+ response: await onDeviceCall(),
1885
+ inferenceSource: InferenceSource.ON_DEVICE
1886
+ };
1816
1887
  }
1817
1888
  throw new AIError(AIErrorCode.UNSUPPORTED, 'Inference mode is ONLY_ON_DEVICE, but an on-device model is not available.');
1818
1889
  case InferenceMode.ONLY_IN_CLOUD:
1819
- return inCloudCall();
1890
+ return {
1891
+ response: await inCloudCall(),
1892
+ inferenceSource: InferenceSource.IN_CLOUD
1893
+ };
1820
1894
  case InferenceMode.PREFER_IN_CLOUD:
1821
1895
  try {
1822
- return await inCloudCall();
1896
+ return {
1897
+ response: await inCloudCall(),
1898
+ inferenceSource: InferenceSource.IN_CLOUD
1899
+ };
1823
1900
  }
1824
1901
  catch (e) {
1825
1902
  if (e instanceof AIError && errorsCausingFallback.includes(e.code)) {
1826
- return onDeviceCall();
1903
+ return {
1904
+ response: await onDeviceCall(),
1905
+ inferenceSource: InferenceSource.ON_DEVICE
1906
+ };
1827
1907
  }
1828
1908
  throw e;
1829
1909
  }
1830
1910
  case InferenceMode.PREFER_ON_DEVICE:
1831
1911
  if (await chromeAdapter.isAvailable(request)) {
1832
- return onDeviceCall();
1912
+ return {
1913
+ response: await onDeviceCall(),
1914
+ inferenceSource: InferenceSource.ON_DEVICE
1915
+ };
1833
1916
  }
1834
- return inCloudCall();
1917
+ return {
1918
+ response: await inCloudCall(),
1919
+ inferenceSource: InferenceSource.IN_CLOUD
1920
+ };
1835
1921
  default:
1836
1922
  throw new AIError(AIErrorCode.ERROR, `Unexpected infererence mode: ${chromeAdapter.mode}`);
1837
1923
  }
@@ -1861,8 +1947,8 @@ async function generateContentStreamOnCloud(apiSettings, model, params, requestO
1861
1947
  /* stream */ true, JSON.stringify(params), requestOptions);
1862
1948
  }
1863
1949
  async function generateContentStream(apiSettings, model, params, chromeAdapter, requestOptions) {
1864
- const response = await callCloudOrDevice(params, chromeAdapter, () => chromeAdapter.generateContentStream(params), () => generateContentStreamOnCloud(apiSettings, model, params, requestOptions));
1865
- return processStream(response, apiSettings); // TODO: Map streaming responses
1950
+ const callResult = await callCloudOrDevice(params, chromeAdapter, () => chromeAdapter.generateContentStream(params), () => generateContentStreamOnCloud(apiSettings, model, params, requestOptions));
1951
+ return processStream(callResult.response, apiSettings); // TODO: Map streaming responses
1866
1952
  }
1867
1953
  async function generateContentOnCloud(apiSettings, model, params, requestOptions) {
1868
1954
  if (apiSettings.backend.backendType === BackendType.GOOGLE_AI) {
@@ -1872,9 +1958,9 @@ async function generateContentOnCloud(apiSettings, model, params, requestOptions
1872
1958
  /* stream */ false, JSON.stringify(params), requestOptions);
1873
1959
  }
1874
1960
  async function generateContent(apiSettings, model, params, chromeAdapter, requestOptions) {
1875
- const response = await callCloudOrDevice(params, chromeAdapter, () => chromeAdapter.generateContent(params), () => generateContentOnCloud(apiSettings, model, params, requestOptions));
1876
- const generateContentResponse = await processGenerateContentResponse(response, apiSettings);
1877
- const enhancedResponse = createEnhancedContentResponse(generateContentResponse);
1961
+ const callResult = await callCloudOrDevice(params, chromeAdapter, () => chromeAdapter.generateContent(params), () => generateContentOnCloud(apiSettings, model, params, requestOptions));
1962
+ const generateContentResponse = await processGenerateContentResponse(callResult.response, apiSettings);
1963
+ const enhancedResponse = createEnhancedContentResponse(generateContentResponse, callResult.inferenceSource);
1878
1964
  return {
1879
1965
  response: enhancedResponse
1880
1966
  };
@@ -2448,56 +2534,104 @@ class LiveSession {
2448
2534
  this.webSocketHandler.send(JSON.stringify(message));
2449
2535
  }
2450
2536
  /**
2451
- * Sends realtime input to the server.
2537
+ * Sends text to the server in realtime.
2452
2538
  *
2453
- * @param mediaChunks - The media chunks to send.
2539
+ * @example
2540
+ * ```javascript
2541
+ * liveSession.sendTextRealtime("Hello, how are you?");
2542
+ * ```
2543
+ *
2544
+ * @param text - The text data to send.
2454
2545
  * @throws If this session has been closed.
2455
2546
  *
2456
2547
  * @beta
2457
2548
  */
2458
- async sendMediaChunks(mediaChunks) {
2549
+ async sendTextRealtime(text) {
2459
2550
  if (this.isClosed) {
2460
2551
  throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
2461
2552
  }
2462
- // The backend does not support sending more than one mediaChunk in one message.
2463
- // Work around this limitation by sending mediaChunks in separate messages.
2464
- mediaChunks.forEach(mediaChunk => {
2465
- const message = {
2466
- realtimeInput: { mediaChunks: [mediaChunk] }
2467
- };
2468
- this.webSocketHandler.send(JSON.stringify(message));
2469
- });
2553
+ const message = {
2554
+ realtimeInput: {
2555
+ text
2556
+ }
2557
+ };
2558
+ this.webSocketHandler.send(JSON.stringify(message));
2470
2559
  }
2471
2560
  /**
2472
- * Sends a stream of {@link GenerativeContentBlob}.
2561
+ * Sends audio data to the server in realtime.
2473
2562
  *
2474
- * @param mediaChunkStream - The stream of {@link GenerativeContentBlob} to send.
2563
+ * @remarks The server requires that the audio data is base64-encoded 16-bit PCM at 16kHz
2564
+ * little-endian.
2565
+ *
2566
+ * @example
2567
+ * ```javascript
2568
+ * // const pcmData = ... base64-encoded 16-bit PCM at 16kHz little-endian.
2569
+ * const blob = { mimeType: "audio/pcm", data: pcmData };
2570
+ * liveSession.sendAudioRealtime(blob);
2571
+ * ```
2572
+ *
2573
+ * @param blob - The base64-encoded PCM data to send to the server in realtime.
2475
2574
  * @throws If this session has been closed.
2476
2575
  *
2477
2576
  * @beta
2478
2577
  */
2479
- async sendMediaStream(mediaChunkStream) {
2578
+ async sendAudioRealtime(blob) {
2480
2579
  if (this.isClosed) {
2481
2580
  throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
2482
2581
  }
2483
- const reader = mediaChunkStream.getReader();
2484
- while (true) {
2485
- try {
2486
- const { done, value } = await reader.read();
2487
- if (done) {
2488
- break;
2489
- }
2490
- else if (!value) {
2491
- throw new Error('Missing chunk in reader, but reader is not done.');
2492
- }
2493
- await this.sendMediaChunks([value]);
2582
+ const message = {
2583
+ realtimeInput: {
2584
+ audio: blob
2494
2585
  }
2495
- catch (e) {
2496
- // Re-throw any errors that occur during stream consumption or sending.
2497
- const message = e instanceof Error ? e.message : 'Error processing media stream.';
2498
- throw new AIError(AIErrorCode.REQUEST_ERROR, message);
2586
+ };
2587
+ this.webSocketHandler.send(JSON.stringify(message));
2588
+ }
2589
+ /**
2590
+ * Sends video data to the server in realtime.
2591
+ *
2592
+ * @remarks The server requires that the video is sent as individual video frames at 1 FPS. It
2593
+ * is recommended to set `mimeType` to `image/jpeg`.
2594
+ *
2595
+ * @example
2596
+ * ```javascript
2597
+ * // const videoFrame = ... base64-encoded JPEG data
2598
+ * const blob = { mimeType: "image/jpeg", data: videoFrame };
2599
+ * liveSession.sendVideoRealtime(blob);
2600
+ * ```
2601
+ * @param blob - The base64-encoded video data to send to the server in realtime.
2602
+ * @throws If this session has been closed.
2603
+ *
2604
+ * @beta
2605
+ */
2606
+ async sendVideoRealtime(blob) {
2607
+ if (this.isClosed) {
2608
+ throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
2609
+ }
2610
+ const message = {
2611
+ realtimeInput: {
2612
+ video: blob
2499
2613
  }
2614
+ };
2615
+ this.webSocketHandler.send(JSON.stringify(message));
2616
+ }
2617
+ /**
2618
+ * Sends function responses to the server.
2619
+ *
2620
+ * @param functionResponses - The function responses to send.
2621
+ * @throws If this session has been closed.
2622
+ *
2623
+ * @beta
2624
+ */
2625
+ async sendFunctionResponses(functionResponses) {
2626
+ if (this.isClosed) {
2627
+ throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
2500
2628
  }
2629
+ const message = {
2630
+ toolResponse: {
2631
+ functionResponses
2632
+ }
2633
+ };
2634
+ this.webSocketHandler.send(JSON.stringify(message));
2501
2635
  }
2502
2636
  /**
2503
2637
  * Yields messages received from the server.
@@ -2555,6 +2689,62 @@ class LiveSession {
2555
2689
  await this.webSocketHandler.close(1000, 'Client closed session.');
2556
2690
  }
2557
2691
  }
2692
+ /**
2693
+ * Sends realtime input to the server.
2694
+ *
2695
+ * @deprecated Use `sendTextRealtime()`, `sendAudioRealtime()`, and `sendVideoRealtime()` instead.
2696
+ *
2697
+ * @param mediaChunks - The media chunks to send.
2698
+ * @throws If this session has been closed.
2699
+ *
2700
+ * @beta
2701
+ */
2702
+ async sendMediaChunks(mediaChunks) {
2703
+ if (this.isClosed) {
2704
+ throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
2705
+ }
2706
+ // The backend does not support sending more than one mediaChunk in one message.
2707
+ // Work around this limitation by sending mediaChunks in separate messages.
2708
+ mediaChunks.forEach(mediaChunk => {
2709
+ const message = {
2710
+ realtimeInput: { mediaChunks: [mediaChunk] }
2711
+ };
2712
+ this.webSocketHandler.send(JSON.stringify(message));
2713
+ });
2714
+ }
2715
+ /**
2716
+ * @deprecated Use `sendTextRealtime()`, `sendAudioRealtime()`, and `sendVideoRealtime()` instead.
2717
+ *
2718
+ * Sends a stream of {@link GenerativeContentBlob}.
2719
+ *
2720
+ * @param mediaChunkStream - The stream of {@link GenerativeContentBlob} to send.
2721
+ * @throws If this session has been closed.
2722
+ *
2723
+ * @beta
2724
+ */
2725
+ async sendMediaStream(mediaChunkStream) {
2726
+ if (this.isClosed) {
2727
+ throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
2728
+ }
2729
+ const reader = mediaChunkStream.getReader();
2730
+ while (true) {
2731
+ try {
2732
+ const { done, value } = await reader.read();
2733
+ if (done) {
2734
+ break;
2735
+ }
2736
+ else if (!value) {
2737
+ throw new Error('Missing chunk in reader, but reader is not done.');
2738
+ }
2739
+ await this.sendMediaChunks([value]);
2740
+ }
2741
+ catch (e) {
2742
+ // Re-throw any errors that occur during stream consumption or sending.
2743
+ const message = e instanceof Error ? e.message : 'Error processing media stream.';
2744
+ throw new AIError(AIErrorCode.REQUEST_ERROR, message);
2745
+ }
2746
+ }
2747
+ }
2558
2748
  }
2559
2749
 
2560
2750
  /**
@@ -2682,7 +2872,7 @@ class LiveGenerativeModel extends AIModel {
2682
2872
  * }
2683
2873
  * ```
2684
2874
  *
2685
- * @beta
2875
+ * @public
2686
2876
  */
2687
2877
  class ImagenModel extends AIModel {
2688
2878
  /**
@@ -2718,7 +2908,7 @@ class ImagenModel extends AIModel {
2718
2908
  * returned object will have a `filteredReason` property.
2719
2909
  * If all images are filtered, the `images` array will be empty.
2720
2910
  *
2721
- * @beta
2911
+ * @public
2722
2912
  */
2723
2913
  async generateImages(prompt) {
2724
2914
  const body = createPredictRequestBody(prompt, {
@@ -3181,7 +3371,7 @@ class AnyOfSchema extends Schema {
3181
3371
  * }
3182
3372
  * ```
3183
3373
  *
3184
- * @beta
3374
+ * @public
3185
3375
  */
3186
3376
  class ImagenImageFormat {
3187
3377
  constructor() {
@@ -3193,7 +3383,7 @@ class ImagenImageFormat {
3193
3383
  * @param compressionQuality - The level of compression (a number between 0 and 100).
3194
3384
  * @returns An {@link ImagenImageFormat} object for a JPEG image.
3195
3385
  *
3196
- * @beta
3386
+ * @public
3197
3387
  */
3198
3388
  static jpeg(compressionQuality) {
3199
3389
  if (compressionQuality &&
@@ -3207,7 +3397,7 @@ class ImagenImageFormat {
3207
3397
  *
3208
3398
  * @returns An {@link ImagenImageFormat} object for a PNG image.
3209
3399
  *
3210
- * @beta
3400
+ * @public
3211
3401
  */
3212
3402
  static png() {
3213
3403
  return { mimeType: 'image/png' };
@@ -3327,7 +3517,7 @@ class AudioConversationRunner {
3327
3517
  mimeType: 'audio/pcm',
3328
3518
  data: base64
3329
3519
  };
3330
- void this.liveSession.sendMediaChunks([chunk]);
3520
+ void this.liveSession.sendAudioRealtime(chunk);
3331
3521
  };
3332
3522
  }
3333
3523
  /**
@@ -3449,9 +3639,9 @@ class AudioConversationRunner {
3449
3639
  }
3450
3640
  else {
3451
3641
  try {
3452
- const resultPart = await this.options.functionCallingHandler(message.functionCalls);
3642
+ const functionResponse = await this.options.functionCallingHandler(message.functionCalls);
3453
3643
  if (!this.isStopped) {
3454
- void this.liveSession.send([resultPart]);
3644
+ void this.liveSession.sendFunctionResponses([functionResponse]);
3455
3645
  }
3456
3646
  }
3457
3647
  catch (e) {
@@ -3671,7 +3861,7 @@ function getGenerativeModel(ai, modelParams, requestOptions) {
3671
3861
  * @throws If the `apiKey` or `projectId` fields are missing in your
3672
3862
  * Firebase config.
3673
3863
  *
3674
- * @beta
3864
+ * @public
3675
3865
  */
3676
3866
  function getImagenModel(ai, modelParams, requestOptions) {
3677
3867
  if (!modelParams.model) {
@@ -3722,5 +3912,5 @@ function registerAI() {
3722
3912
  }
3723
3913
  registerAI();
3724
3914
 
3725
- export { AIError, AIErrorCode, AIModel, AnyOfSchema, ArraySchema, Backend, BackendType, BlockReason, BooleanSchema, ChatSession, FinishReason, FunctionCallingMode, GenerativeModel, GoogleAIBackend, HarmBlockMethod, HarmBlockThreshold, HarmCategory, HarmProbability, HarmSeverity, ImagenAspectRatio, ImagenImageFormat, ImagenModel, ImagenPersonFilterLevel, ImagenSafetyFilterLevel, InferenceMode, IntegerSchema, Language, LiveGenerativeModel, LiveResponseType, LiveSession, Modality, NumberSchema, ObjectSchema, Outcome, POSSIBLE_ROLES, ResponseModality, Schema, SchemaType, StringSchema, VertexAIBackend, getAI, getGenerativeModel, getImagenModel, getLiveGenerativeModel, startAudioConversation };
3915
+ export { AIError, AIErrorCode, AIModel, AnyOfSchema, ArraySchema, Backend, BackendType, BlockReason, BooleanSchema, ChatSession, FinishReason, FunctionCallingMode, GenerativeModel, GoogleAIBackend, HarmBlockMethod, HarmBlockThreshold, HarmCategory, HarmProbability, HarmSeverity, ImagenAspectRatio, ImagenImageFormat, ImagenModel, ImagenPersonFilterLevel, ImagenSafetyFilterLevel, InferenceMode, InferenceSource, IntegerSchema, Language, LiveGenerativeModel, LiveResponseType, LiveSession, Modality, NumberSchema, ObjectSchema, Outcome, POSSIBLE_ROLES, ResponseModality, Schema, SchemaType, StringSchema, URLRetrievalStatus, VertexAIBackend, getAI, getGenerativeModel, getImagenModel, getLiveGenerativeModel, startAudioConversation };
3726
3916
  //# sourceMappingURL=index.node.mjs.map