@firebase/ai 2.3.0 → 2.4.0-canary.22e0a1adb
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-public.d.ts +240 -35
- package/dist/ai.d.ts +241 -35
- package/dist/esm/index.esm.js +295 -92
- package/dist/esm/index.esm.js.map +1 -1
- package/dist/esm/src/api.d.ts +1 -1
- package/dist/esm/src/methods/chrome-adapter.d.ts +1 -1
- package/dist/esm/src/methods/live-session-helpers.d.ts +2 -2
- package/dist/esm/src/methods/live-session.d.ts +71 -7
- package/dist/esm/src/models/imagen-model.d.ts +2 -2
- package/dist/esm/src/requests/hybrid-helpers.d.ts +7 -2
- package/dist/esm/src/requests/imagen-image-format.d.ts +3 -3
- package/dist/esm/src/requests/response-helpers.d.ts +2 -2
- package/dist/esm/src/requests/stream-reader.d.ts +2 -1
- package/dist/esm/src/types/content.d.ts +4 -4
- package/dist/esm/src/types/enums.d.ts +19 -4
- package/dist/esm/src/types/googleai.d.ts +2 -1
- package/dist/esm/src/types/imagen/requests.d.ts +9 -9
- package/dist/esm/src/types/imagen/responses.d.ts +3 -3
- package/dist/esm/src/types/live-responses.d.ts +16 -2
- package/dist/esm/src/types/requests.d.ts +22 -2
- package/dist/esm/src/types/responses.d.ts +99 -1
- package/dist/index.cjs.js +296 -91
- package/dist/index.cjs.js.map +1 -1
- package/dist/index.node.cjs.js +276 -84
- package/dist/index.node.cjs.js.map +1 -1
- package/dist/index.node.mjs +275 -85
- package/dist/index.node.mjs.map +1 -1
- package/dist/src/api.d.ts +1 -1
- package/dist/src/methods/chrome-adapter.d.ts +1 -1
- package/dist/src/methods/live-session-helpers.d.ts +2 -2
- package/dist/src/methods/live-session.d.ts +71 -7
- package/dist/src/models/imagen-model.d.ts +2 -2
- package/dist/src/requests/hybrid-helpers.d.ts +7 -2
- package/dist/src/requests/imagen-image-format.d.ts +3 -3
- package/dist/src/requests/response-helpers.d.ts +2 -2
- package/dist/src/requests/stream-reader.d.ts +2 -1
- package/dist/src/types/content.d.ts +4 -4
- package/dist/src/types/enums.d.ts +19 -4
- package/dist/src/types/googleai.d.ts +2 -1
- package/dist/src/types/imagen/requests.d.ts +9 -9
- package/dist/src/types/imagen/responses.d.ts +3 -3
- package/dist/src/types/live-responses.d.ts +16 -2
- package/dist/src/types/requests.d.ts +22 -2
- package/dist/src/types/responses.d.ts +99 -1
- package/package.json +8 -8
package/dist/index.node.mjs
CHANGED
|
@@ -4,7 +4,7 @@ import { FirebaseError, Deferred, getModularInstance } from '@firebase/util';
|
|
|
4
4
|
import { Logger } from '@firebase/logger';
|
|
5
5
|
|
|
6
6
|
var name = "@firebase/ai";
|
|
7
|
-
var version = "2.
|
|
7
|
+
var version = "2.4.0-canary.22e0a1adb";
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
10
|
* @license
|
|
@@ -323,10 +323,19 @@ const InferenceMode = {
|
|
|
323
323
|
'ONLY_IN_CLOUD': 'only_in_cloud',
|
|
324
324
|
'PREFER_IN_CLOUD': 'prefer_in_cloud'
|
|
325
325
|
};
|
|
326
|
+
/**
|
|
327
|
+
* Indicates whether inference happened on-device or in-cloud.
|
|
328
|
+
*
|
|
329
|
+
* @beta
|
|
330
|
+
*/
|
|
331
|
+
const InferenceSource = {
|
|
332
|
+
'ON_DEVICE': 'on_device',
|
|
333
|
+
'IN_CLOUD': 'in_cloud'
|
|
334
|
+
};
|
|
326
335
|
/**
|
|
327
336
|
* Represents the result of the code execution.
|
|
328
337
|
*
|
|
329
|
-
* @
|
|
338
|
+
* @beta
|
|
330
339
|
*/
|
|
331
340
|
const Outcome = {
|
|
332
341
|
UNSPECIFIED: 'OUTCOME_UNSPECIFIED',
|
|
@@ -337,7 +346,7 @@ const Outcome = {
|
|
|
337
346
|
/**
|
|
338
347
|
* The programming language of the code.
|
|
339
348
|
*
|
|
340
|
-
* @
|
|
349
|
+
* @beta
|
|
341
350
|
*/
|
|
342
351
|
const Language = {
|
|
343
352
|
UNSPECIFIED: 'LANGUAGE_UNSPECIFIED',
|
|
@@ -360,6 +369,45 @@ const Language = {
|
|
|
360
369
|
* See the License for the specific language governing permissions and
|
|
361
370
|
* limitations under the License.
|
|
362
371
|
*/
|
|
372
|
+
/**
|
|
373
|
+
* The status of a URL retrieval.
|
|
374
|
+
*
|
|
375
|
+
* @remarks
|
|
376
|
+
* <b>URL_RETRIEVAL_STATUS_UNSPECIFIED:</b> Unspecified retrieval status.
|
|
377
|
+
* <br/>
|
|
378
|
+
* <b>URL_RETRIEVAL_STATUS_SUCCESS:</b> The URL retrieval was successful.
|
|
379
|
+
* <br/>
|
|
380
|
+
* <b>URL_RETRIEVAL_STATUS_ERROR:</b> The URL retrieval failed.
|
|
381
|
+
* <br/>
|
|
382
|
+
* <b>URL_RETRIEVAL_STATUS_PAYWALL:</b> The URL retrieval failed because the content is behind a paywall.
|
|
383
|
+
* <br/>
|
|
384
|
+
* <b>URL_RETRIEVAL_STATUS_UNSAFE:</b> The URL retrieval failed because the content is unsafe.
|
|
385
|
+
* <br/>
|
|
386
|
+
*
|
|
387
|
+
* @beta
|
|
388
|
+
*/
|
|
389
|
+
const URLRetrievalStatus = {
|
|
390
|
+
/**
|
|
391
|
+
* Unspecified retrieval status.
|
|
392
|
+
*/
|
|
393
|
+
URL_RETRIEVAL_STATUS_UNSPECIFIED: 'URL_RETRIEVAL_STATUS_UNSPECIFIED',
|
|
394
|
+
/**
|
|
395
|
+
* The URL retrieval was successful.
|
|
396
|
+
*/
|
|
397
|
+
URL_RETRIEVAL_STATUS_SUCCESS: 'URL_RETRIEVAL_STATUS_SUCCESS',
|
|
398
|
+
/**
|
|
399
|
+
* The URL retrieval failed.
|
|
400
|
+
*/
|
|
401
|
+
URL_RETRIEVAL_STATUS_ERROR: 'URL_RETRIEVAL_STATUS_ERROR',
|
|
402
|
+
/**
|
|
403
|
+
* The URL retrieval failed because the content is behind a paywall.
|
|
404
|
+
*/
|
|
405
|
+
URL_RETRIEVAL_STATUS_PAYWALL: 'URL_RETRIEVAL_STATUS_PAYWALL',
|
|
406
|
+
/**
|
|
407
|
+
* The URL retrieval failed because the content is unsafe.
|
|
408
|
+
*/
|
|
409
|
+
URL_RETRIEVAL_STATUS_UNSAFE: 'URL_RETRIEVAL_STATUS_UNSAFE'
|
|
410
|
+
};
|
|
363
411
|
/**
|
|
364
412
|
* The types of responses that can be returned by {@link LiveSession.receive}.
|
|
365
413
|
*
|
|
@@ -486,7 +534,7 @@ const SchemaType = {
|
|
|
486
534
|
* and the {@link https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters | Responsible AI and usage guidelines}
|
|
487
535
|
* for more details.
|
|
488
536
|
*
|
|
489
|
-
* @
|
|
537
|
+
* @public
|
|
490
538
|
*/
|
|
491
539
|
const ImagenSafetyFilterLevel = {
|
|
492
540
|
/**
|
|
@@ -515,7 +563,7 @@ const ImagenSafetyFilterLevel = {
|
|
|
515
563
|
* See the <a href="http://firebase.google.com/docs/vertex-ai/generate-images">personGeneration</a>
|
|
516
564
|
* documentation for more details.
|
|
517
565
|
*
|
|
518
|
-
* @
|
|
566
|
+
* @public
|
|
519
567
|
*/
|
|
520
568
|
const ImagenPersonFilterLevel = {
|
|
521
569
|
/**
|
|
@@ -548,7 +596,7 @@ const ImagenPersonFilterLevel = {
|
|
|
548
596
|
* See the {@link http://firebase.google.com/docs/vertex-ai/generate-images | documentation }
|
|
549
597
|
* for more details and examples of the supported aspect ratios.
|
|
550
598
|
*
|
|
551
|
-
* @
|
|
599
|
+
* @public
|
|
552
600
|
*/
|
|
553
601
|
const ImagenAspectRatio = {
|
|
554
602
|
/**
|
|
@@ -1230,7 +1278,7 @@ function hasValidCandidates(response) {
|
|
|
1230
1278
|
* Creates an EnhancedGenerateContentResponse object that has helper functions and
|
|
1231
1279
|
* other modifications that improve usability.
|
|
1232
1280
|
*/
|
|
1233
|
-
function createEnhancedContentResponse(response) {
|
|
1281
|
+
function createEnhancedContentResponse(response, inferenceSource = InferenceSource.IN_CLOUD) {
|
|
1234
1282
|
/**
|
|
1235
1283
|
* The Vertex AI backend omits default values.
|
|
1236
1284
|
* This causes the `index` property to be omitted from the first candidate in the
|
|
@@ -1241,6 +1289,7 @@ function createEnhancedContentResponse(response) {
|
|
|
1241
1289
|
response.candidates[0].index = 0;
|
|
1242
1290
|
}
|
|
1243
1291
|
const responseWithHelpers = addHelpers(response);
|
|
1292
|
+
responseWithHelpers.inferenceSource = inferenceSource;
|
|
1244
1293
|
return responseWithHelpers;
|
|
1245
1294
|
}
|
|
1246
1295
|
/**
|
|
@@ -1553,7 +1602,7 @@ function mapGenerateContentCandidates(candidates) {
|
|
|
1553
1602
|
// videoMetadata is not supported.
|
|
1554
1603
|
// Throw early since developers may send a long video as input and only expect to pay
|
|
1555
1604
|
// for inference on a small portion of the video.
|
|
1556
|
-
if (candidate.content?.parts
|
|
1605
|
+
if (candidate.content?.parts?.some(part => part?.videoMetadata)) {
|
|
1557
1606
|
throw new AIError(AIErrorCode.UNSUPPORTED, 'Part.videoMetadata is not supported in the Gemini Developer API. Please remove this property.');
|
|
1558
1607
|
}
|
|
1559
1608
|
const mappedCandidate = {
|
|
@@ -1563,7 +1612,8 @@ function mapGenerateContentCandidates(candidates) {
|
|
|
1563
1612
|
finishMessage: candidate.finishMessage,
|
|
1564
1613
|
safetyRatings: mappedSafetyRatings,
|
|
1565
1614
|
citationMetadata,
|
|
1566
|
-
groundingMetadata: candidate.groundingMetadata
|
|
1615
|
+
groundingMetadata: candidate.groundingMetadata,
|
|
1616
|
+
urlContextMetadata: candidate.urlContextMetadata
|
|
1567
1617
|
};
|
|
1568
1618
|
mappedCandidates.push(mappedCandidate);
|
|
1569
1619
|
});
|
|
@@ -1616,16 +1666,16 @@ const responseLineRE = /^data\: (.*)(?:\n\n|\r\r|\r\n\r\n)/;
|
|
|
1616
1666
|
*
|
|
1617
1667
|
* @param response - Response from a fetch call
|
|
1618
1668
|
*/
|
|
1619
|
-
function processStream(response, apiSettings) {
|
|
1669
|
+
function processStream(response, apiSettings, inferenceSource) {
|
|
1620
1670
|
const inputStream = response.body.pipeThrough(new TextDecoderStream('utf8', { fatal: true }));
|
|
1621
1671
|
const responseStream = getResponseStream(inputStream);
|
|
1622
1672
|
const [stream1, stream2] = responseStream.tee();
|
|
1623
1673
|
return {
|
|
1624
|
-
stream: generateResponseSequence(stream1, apiSettings),
|
|
1625
|
-
response: getResponsePromise(stream2, apiSettings)
|
|
1674
|
+
stream: generateResponseSequence(stream1, apiSettings, inferenceSource),
|
|
1675
|
+
response: getResponsePromise(stream2, apiSettings, inferenceSource)
|
|
1626
1676
|
};
|
|
1627
1677
|
}
|
|
1628
|
-
async function getResponsePromise(stream, apiSettings) {
|
|
1678
|
+
async function getResponsePromise(stream, apiSettings, inferenceSource) {
|
|
1629
1679
|
const allResponses = [];
|
|
1630
1680
|
const reader = stream.getReader();
|
|
1631
1681
|
while (true) {
|
|
@@ -1635,12 +1685,12 @@ async function getResponsePromise(stream, apiSettings) {
|
|
|
1635
1685
|
if (apiSettings.backend.backendType === BackendType.GOOGLE_AI) {
|
|
1636
1686
|
generateContentResponse = mapGenerateContentResponse(generateContentResponse);
|
|
1637
1687
|
}
|
|
1638
|
-
return createEnhancedContentResponse(generateContentResponse);
|
|
1688
|
+
return createEnhancedContentResponse(generateContentResponse, inferenceSource);
|
|
1639
1689
|
}
|
|
1640
1690
|
allResponses.push(value);
|
|
1641
1691
|
}
|
|
1642
1692
|
}
|
|
1643
|
-
async function* generateResponseSequence(stream, apiSettings) {
|
|
1693
|
+
async function* generateResponseSequence(stream, apiSettings, inferenceSource) {
|
|
1644
1694
|
const reader = stream.getReader();
|
|
1645
1695
|
while (true) {
|
|
1646
1696
|
const { value, done } = await reader.read();
|
|
@@ -1649,10 +1699,18 @@ async function* generateResponseSequence(stream, apiSettings) {
|
|
|
1649
1699
|
}
|
|
1650
1700
|
let enhancedResponse;
|
|
1651
1701
|
if (apiSettings.backend.backendType === BackendType.GOOGLE_AI) {
|
|
1652
|
-
enhancedResponse = createEnhancedContentResponse(mapGenerateContentResponse(value));
|
|
1702
|
+
enhancedResponse = createEnhancedContentResponse(mapGenerateContentResponse(value), inferenceSource);
|
|
1653
1703
|
}
|
|
1654
1704
|
else {
|
|
1655
|
-
enhancedResponse = createEnhancedContentResponse(value);
|
|
1705
|
+
enhancedResponse = createEnhancedContentResponse(value, inferenceSource);
|
|
1706
|
+
}
|
|
1707
|
+
const firstCandidate = enhancedResponse.candidates?.[0];
|
|
1708
|
+
// Don't yield a response with no useful data for the developer.
|
|
1709
|
+
if (!firstCandidate?.content?.parts &&
|
|
1710
|
+
!firstCandidate?.finishReason &&
|
|
1711
|
+
!firstCandidate?.citationMetadata &&
|
|
1712
|
+
!firstCandidate?.urlContextMetadata) {
|
|
1713
|
+
continue;
|
|
1656
1714
|
}
|
|
1657
1715
|
yield enhancedResponse;
|
|
1658
1716
|
}
|
|
@@ -1733,36 +1791,43 @@ function aggregateResponses(responses) {
|
|
|
1733
1791
|
candidate.safetyRatings;
|
|
1734
1792
|
aggregatedResponse.candidates[i].groundingMetadata =
|
|
1735
1793
|
candidate.groundingMetadata;
|
|
1794
|
+
// The urlContextMetadata object is defined in the first chunk of the response stream.
|
|
1795
|
+
// In all subsequent chunks, the urlContextMetadata object will be undefined. We need to
|
|
1796
|
+
// make sure that we don't overwrite the first value urlContextMetadata object with undefined.
|
|
1797
|
+
// FIXME: What happens if we receive a second, valid urlContextMetadata object?
|
|
1798
|
+
const urlContextMetadata = candidate.urlContextMetadata;
|
|
1799
|
+
if (typeof urlContextMetadata === 'object' &&
|
|
1800
|
+
urlContextMetadata !== null &&
|
|
1801
|
+
Object.keys(urlContextMetadata).length > 0) {
|
|
1802
|
+
aggregatedResponse.candidates[i].urlContextMetadata =
|
|
1803
|
+
urlContextMetadata;
|
|
1804
|
+
}
|
|
1736
1805
|
/**
|
|
1737
1806
|
* Candidates should always have content and parts, but this handles
|
|
1738
1807
|
* possible malformed responses.
|
|
1739
1808
|
*/
|
|
1740
|
-
if (candidate.content
|
|
1809
|
+
if (candidate.content) {
|
|
1810
|
+
// Skip a candidate without parts.
|
|
1811
|
+
if (!candidate.content.parts) {
|
|
1812
|
+
continue;
|
|
1813
|
+
}
|
|
1741
1814
|
if (!aggregatedResponse.candidates[i].content) {
|
|
1742
1815
|
aggregatedResponse.candidates[i].content = {
|
|
1743
1816
|
role: candidate.content.role || 'user',
|
|
1744
1817
|
parts: []
|
|
1745
1818
|
};
|
|
1746
1819
|
}
|
|
1747
|
-
const newPart = {};
|
|
1748
1820
|
for (const part of candidate.content.parts) {
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
}
|
|
1756
|
-
newPart.text = part.text;
|
|
1821
|
+
const newPart = { ...part };
|
|
1822
|
+
// The backend can send empty text parts. If these are sent back
|
|
1823
|
+
// (e.g. in chat history), the backend will respond with an error.
|
|
1824
|
+
// To prevent this, ignore empty text parts.
|
|
1825
|
+
if (part.text === '') {
|
|
1826
|
+
continue;
|
|
1757
1827
|
}
|
|
1758
|
-
if (
|
|
1759
|
-
|
|
1828
|
+
if (Object.keys(newPart).length > 0) {
|
|
1829
|
+
aggregatedResponse.candidates[i].content.parts.push(newPart);
|
|
1760
1830
|
}
|
|
1761
|
-
if (Object.keys(newPart).length === 0) {
|
|
1762
|
-
throw new AIError(AIErrorCode.INVALID_CONTENT, 'Part should have at least one property, but there are none. This is likely caused ' +
|
|
1763
|
-
'by a malformed response from the backend.');
|
|
1764
|
-
}
|
|
1765
|
-
aggregatedResponse.candidates[i].content.parts.push(newPart);
|
|
1766
1831
|
}
|
|
1767
1832
|
}
|
|
1768
1833
|
}
|
|
@@ -1807,31 +1872,52 @@ const errorsCausingFallback = [
|
|
|
1807
1872
|
*/
|
|
1808
1873
|
async function callCloudOrDevice(request, chromeAdapter, onDeviceCall, inCloudCall) {
|
|
1809
1874
|
if (!chromeAdapter) {
|
|
1810
|
-
return
|
|
1875
|
+
return {
|
|
1876
|
+
response: await inCloudCall(),
|
|
1877
|
+
inferenceSource: InferenceSource.IN_CLOUD
|
|
1878
|
+
};
|
|
1811
1879
|
}
|
|
1812
1880
|
switch (chromeAdapter.mode) {
|
|
1813
1881
|
case InferenceMode.ONLY_ON_DEVICE:
|
|
1814
1882
|
if (await chromeAdapter.isAvailable(request)) {
|
|
1815
|
-
return
|
|
1883
|
+
return {
|
|
1884
|
+
response: await onDeviceCall(),
|
|
1885
|
+
inferenceSource: InferenceSource.ON_DEVICE
|
|
1886
|
+
};
|
|
1816
1887
|
}
|
|
1817
1888
|
throw new AIError(AIErrorCode.UNSUPPORTED, 'Inference mode is ONLY_ON_DEVICE, but an on-device model is not available.');
|
|
1818
1889
|
case InferenceMode.ONLY_IN_CLOUD:
|
|
1819
|
-
return
|
|
1890
|
+
return {
|
|
1891
|
+
response: await inCloudCall(),
|
|
1892
|
+
inferenceSource: InferenceSource.IN_CLOUD
|
|
1893
|
+
};
|
|
1820
1894
|
case InferenceMode.PREFER_IN_CLOUD:
|
|
1821
1895
|
try {
|
|
1822
|
-
return
|
|
1896
|
+
return {
|
|
1897
|
+
response: await inCloudCall(),
|
|
1898
|
+
inferenceSource: InferenceSource.IN_CLOUD
|
|
1899
|
+
};
|
|
1823
1900
|
}
|
|
1824
1901
|
catch (e) {
|
|
1825
1902
|
if (e instanceof AIError && errorsCausingFallback.includes(e.code)) {
|
|
1826
|
-
return
|
|
1903
|
+
return {
|
|
1904
|
+
response: await onDeviceCall(),
|
|
1905
|
+
inferenceSource: InferenceSource.ON_DEVICE
|
|
1906
|
+
};
|
|
1827
1907
|
}
|
|
1828
1908
|
throw e;
|
|
1829
1909
|
}
|
|
1830
1910
|
case InferenceMode.PREFER_ON_DEVICE:
|
|
1831
1911
|
if (await chromeAdapter.isAvailable(request)) {
|
|
1832
|
-
return
|
|
1912
|
+
return {
|
|
1913
|
+
response: await onDeviceCall(),
|
|
1914
|
+
inferenceSource: InferenceSource.ON_DEVICE
|
|
1915
|
+
};
|
|
1833
1916
|
}
|
|
1834
|
-
return
|
|
1917
|
+
return {
|
|
1918
|
+
response: await inCloudCall(),
|
|
1919
|
+
inferenceSource: InferenceSource.IN_CLOUD
|
|
1920
|
+
};
|
|
1835
1921
|
default:
|
|
1836
1922
|
throw new AIError(AIErrorCode.ERROR, `Unexpected infererence mode: ${chromeAdapter.mode}`);
|
|
1837
1923
|
}
|
|
@@ -1861,8 +1947,8 @@ async function generateContentStreamOnCloud(apiSettings, model, params, requestO
|
|
|
1861
1947
|
/* stream */ true, JSON.stringify(params), requestOptions);
|
|
1862
1948
|
}
|
|
1863
1949
|
async function generateContentStream(apiSettings, model, params, chromeAdapter, requestOptions) {
|
|
1864
|
-
const
|
|
1865
|
-
return processStream(response, apiSettings); // TODO: Map streaming responses
|
|
1950
|
+
const callResult = await callCloudOrDevice(params, chromeAdapter, () => chromeAdapter.generateContentStream(params), () => generateContentStreamOnCloud(apiSettings, model, params, requestOptions));
|
|
1951
|
+
return processStream(callResult.response, apiSettings); // TODO: Map streaming responses
|
|
1866
1952
|
}
|
|
1867
1953
|
async function generateContentOnCloud(apiSettings, model, params, requestOptions) {
|
|
1868
1954
|
if (apiSettings.backend.backendType === BackendType.GOOGLE_AI) {
|
|
@@ -1872,9 +1958,9 @@ async function generateContentOnCloud(apiSettings, model, params, requestOptions
|
|
|
1872
1958
|
/* stream */ false, JSON.stringify(params), requestOptions);
|
|
1873
1959
|
}
|
|
1874
1960
|
async function generateContent(apiSettings, model, params, chromeAdapter, requestOptions) {
|
|
1875
|
-
const
|
|
1876
|
-
const generateContentResponse = await processGenerateContentResponse(response, apiSettings);
|
|
1877
|
-
const enhancedResponse = createEnhancedContentResponse(generateContentResponse);
|
|
1961
|
+
const callResult = await callCloudOrDevice(params, chromeAdapter, () => chromeAdapter.generateContent(params), () => generateContentOnCloud(apiSettings, model, params, requestOptions));
|
|
1962
|
+
const generateContentResponse = await processGenerateContentResponse(callResult.response, apiSettings);
|
|
1963
|
+
const enhancedResponse = createEnhancedContentResponse(generateContentResponse, callResult.inferenceSource);
|
|
1878
1964
|
return {
|
|
1879
1965
|
response: enhancedResponse
|
|
1880
1966
|
};
|
|
@@ -2448,56 +2534,104 @@ class LiveSession {
|
|
|
2448
2534
|
this.webSocketHandler.send(JSON.stringify(message));
|
|
2449
2535
|
}
|
|
2450
2536
|
/**
|
|
2451
|
-
* Sends
|
|
2537
|
+
* Sends text to the server in realtime.
|
|
2452
2538
|
*
|
|
2453
|
-
* @
|
|
2539
|
+
* @example
|
|
2540
|
+
* ```javascript
|
|
2541
|
+
* liveSession.sendTextRealtime("Hello, how are you?");
|
|
2542
|
+
* ```
|
|
2543
|
+
*
|
|
2544
|
+
* @param text - The text data to send.
|
|
2454
2545
|
* @throws If this session has been closed.
|
|
2455
2546
|
*
|
|
2456
2547
|
* @beta
|
|
2457
2548
|
*/
|
|
2458
|
-
async
|
|
2549
|
+
async sendTextRealtime(text) {
|
|
2459
2550
|
if (this.isClosed) {
|
|
2460
2551
|
throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
|
|
2461
2552
|
}
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
this.webSocketHandler.send(JSON.stringify(message));
|
|
2469
|
-
});
|
|
2553
|
+
const message = {
|
|
2554
|
+
realtimeInput: {
|
|
2555
|
+
text
|
|
2556
|
+
}
|
|
2557
|
+
};
|
|
2558
|
+
this.webSocketHandler.send(JSON.stringify(message));
|
|
2470
2559
|
}
|
|
2471
2560
|
/**
|
|
2472
|
-
* Sends
|
|
2561
|
+
* Sends audio data to the server in realtime.
|
|
2473
2562
|
*
|
|
2474
|
-
* @
|
|
2563
|
+
* @remarks The server requires that the audio data is base64-encoded 16-bit PCM at 16kHz
|
|
2564
|
+
* little-endian.
|
|
2565
|
+
*
|
|
2566
|
+
* @example
|
|
2567
|
+
* ```javascript
|
|
2568
|
+
* // const pcmData = ... base64-encoded 16-bit PCM at 16kHz little-endian.
|
|
2569
|
+
* const blob = { mimeType: "audio/pcm", data: pcmData };
|
|
2570
|
+
* liveSession.sendAudioRealtime(blob);
|
|
2571
|
+
* ```
|
|
2572
|
+
*
|
|
2573
|
+
* @param blob - The base64-encoded PCM data to send to the server in realtime.
|
|
2475
2574
|
* @throws If this session has been closed.
|
|
2476
2575
|
*
|
|
2477
2576
|
* @beta
|
|
2478
2577
|
*/
|
|
2479
|
-
async
|
|
2578
|
+
async sendAudioRealtime(blob) {
|
|
2480
2579
|
if (this.isClosed) {
|
|
2481
2580
|
throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
|
|
2482
2581
|
}
|
|
2483
|
-
const
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
const { done, value } = await reader.read();
|
|
2487
|
-
if (done) {
|
|
2488
|
-
break;
|
|
2489
|
-
}
|
|
2490
|
-
else if (!value) {
|
|
2491
|
-
throw new Error('Missing chunk in reader, but reader is not done.');
|
|
2492
|
-
}
|
|
2493
|
-
await this.sendMediaChunks([value]);
|
|
2582
|
+
const message = {
|
|
2583
|
+
realtimeInput: {
|
|
2584
|
+
audio: blob
|
|
2494
2585
|
}
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2586
|
+
};
|
|
2587
|
+
this.webSocketHandler.send(JSON.stringify(message));
|
|
2588
|
+
}
|
|
2589
|
+
/**
|
|
2590
|
+
* Sends video data to the server in realtime.
|
|
2591
|
+
*
|
|
2592
|
+
* @remarks The server requires that the video is sent as individual video frames at 1 FPS. It
|
|
2593
|
+
* is recommended to set `mimeType` to `image/jpeg`.
|
|
2594
|
+
*
|
|
2595
|
+
* @example
|
|
2596
|
+
* ```javascript
|
|
2597
|
+
* // const videoFrame = ... base64-encoded JPEG data
|
|
2598
|
+
* const blob = { mimeType: "image/jpeg", data: videoFrame };
|
|
2599
|
+
* liveSession.sendVideoRealtime(blob);
|
|
2600
|
+
* ```
|
|
2601
|
+
* @param blob - The base64-encoded video data to send to the server in realtime.
|
|
2602
|
+
* @throws If this session has been closed.
|
|
2603
|
+
*
|
|
2604
|
+
* @beta
|
|
2605
|
+
*/
|
|
2606
|
+
async sendVideoRealtime(blob) {
|
|
2607
|
+
if (this.isClosed) {
|
|
2608
|
+
throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
|
|
2609
|
+
}
|
|
2610
|
+
const message = {
|
|
2611
|
+
realtimeInput: {
|
|
2612
|
+
video: blob
|
|
2499
2613
|
}
|
|
2614
|
+
};
|
|
2615
|
+
this.webSocketHandler.send(JSON.stringify(message));
|
|
2616
|
+
}
|
|
2617
|
+
/**
|
|
2618
|
+
* Sends function responses to the server.
|
|
2619
|
+
*
|
|
2620
|
+
* @param functionResponses - The function responses to send.
|
|
2621
|
+
* @throws If this session has been closed.
|
|
2622
|
+
*
|
|
2623
|
+
* @beta
|
|
2624
|
+
*/
|
|
2625
|
+
async sendFunctionResponses(functionResponses) {
|
|
2626
|
+
if (this.isClosed) {
|
|
2627
|
+
throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
|
|
2500
2628
|
}
|
|
2629
|
+
const message = {
|
|
2630
|
+
toolResponse: {
|
|
2631
|
+
functionResponses
|
|
2632
|
+
}
|
|
2633
|
+
};
|
|
2634
|
+
this.webSocketHandler.send(JSON.stringify(message));
|
|
2501
2635
|
}
|
|
2502
2636
|
/**
|
|
2503
2637
|
* Yields messages received from the server.
|
|
@@ -2555,6 +2689,62 @@ class LiveSession {
|
|
|
2555
2689
|
await this.webSocketHandler.close(1000, 'Client closed session.');
|
|
2556
2690
|
}
|
|
2557
2691
|
}
|
|
2692
|
+
/**
|
|
2693
|
+
* Sends realtime input to the server.
|
|
2694
|
+
*
|
|
2695
|
+
* @deprecated Use `sendTextRealtime()`, `sendAudioRealtime()`, and `sendVideoRealtime()` instead.
|
|
2696
|
+
*
|
|
2697
|
+
* @param mediaChunks - The media chunks to send.
|
|
2698
|
+
* @throws If this session has been closed.
|
|
2699
|
+
*
|
|
2700
|
+
* @beta
|
|
2701
|
+
*/
|
|
2702
|
+
async sendMediaChunks(mediaChunks) {
|
|
2703
|
+
if (this.isClosed) {
|
|
2704
|
+
throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
|
|
2705
|
+
}
|
|
2706
|
+
// The backend does not support sending more than one mediaChunk in one message.
|
|
2707
|
+
// Work around this limitation by sending mediaChunks in separate messages.
|
|
2708
|
+
mediaChunks.forEach(mediaChunk => {
|
|
2709
|
+
const message = {
|
|
2710
|
+
realtimeInput: { mediaChunks: [mediaChunk] }
|
|
2711
|
+
};
|
|
2712
|
+
this.webSocketHandler.send(JSON.stringify(message));
|
|
2713
|
+
});
|
|
2714
|
+
}
|
|
2715
|
+
/**
|
|
2716
|
+
* @deprecated Use `sendTextRealtime()`, `sendAudioRealtime()`, and `sendVideoRealtime()` instead.
|
|
2717
|
+
*
|
|
2718
|
+
* Sends a stream of {@link GenerativeContentBlob}.
|
|
2719
|
+
*
|
|
2720
|
+
* @param mediaChunkStream - The stream of {@link GenerativeContentBlob} to send.
|
|
2721
|
+
* @throws If this session has been closed.
|
|
2722
|
+
*
|
|
2723
|
+
* @beta
|
|
2724
|
+
*/
|
|
2725
|
+
async sendMediaStream(mediaChunkStream) {
|
|
2726
|
+
if (this.isClosed) {
|
|
2727
|
+
throw new AIError(AIErrorCode.REQUEST_ERROR, 'This LiveSession has been closed and cannot be used.');
|
|
2728
|
+
}
|
|
2729
|
+
const reader = mediaChunkStream.getReader();
|
|
2730
|
+
while (true) {
|
|
2731
|
+
try {
|
|
2732
|
+
const { done, value } = await reader.read();
|
|
2733
|
+
if (done) {
|
|
2734
|
+
break;
|
|
2735
|
+
}
|
|
2736
|
+
else if (!value) {
|
|
2737
|
+
throw new Error('Missing chunk in reader, but reader is not done.');
|
|
2738
|
+
}
|
|
2739
|
+
await this.sendMediaChunks([value]);
|
|
2740
|
+
}
|
|
2741
|
+
catch (e) {
|
|
2742
|
+
// Re-throw any errors that occur during stream consumption or sending.
|
|
2743
|
+
const message = e instanceof Error ? e.message : 'Error processing media stream.';
|
|
2744
|
+
throw new AIError(AIErrorCode.REQUEST_ERROR, message);
|
|
2745
|
+
}
|
|
2746
|
+
}
|
|
2747
|
+
}
|
|
2558
2748
|
}
|
|
2559
2749
|
|
|
2560
2750
|
/**
|
|
@@ -2682,7 +2872,7 @@ class LiveGenerativeModel extends AIModel {
|
|
|
2682
2872
|
* }
|
|
2683
2873
|
* ```
|
|
2684
2874
|
*
|
|
2685
|
-
* @
|
|
2875
|
+
* @public
|
|
2686
2876
|
*/
|
|
2687
2877
|
class ImagenModel extends AIModel {
|
|
2688
2878
|
/**
|
|
@@ -2718,7 +2908,7 @@ class ImagenModel extends AIModel {
|
|
|
2718
2908
|
* returned object will have a `filteredReason` property.
|
|
2719
2909
|
* If all images are filtered, the `images` array will be empty.
|
|
2720
2910
|
*
|
|
2721
|
-
* @
|
|
2911
|
+
* @public
|
|
2722
2912
|
*/
|
|
2723
2913
|
async generateImages(prompt) {
|
|
2724
2914
|
const body = createPredictRequestBody(prompt, {
|
|
@@ -3181,7 +3371,7 @@ class AnyOfSchema extends Schema {
|
|
|
3181
3371
|
* }
|
|
3182
3372
|
* ```
|
|
3183
3373
|
*
|
|
3184
|
-
* @
|
|
3374
|
+
* @public
|
|
3185
3375
|
*/
|
|
3186
3376
|
class ImagenImageFormat {
|
|
3187
3377
|
constructor() {
|
|
@@ -3193,7 +3383,7 @@ class ImagenImageFormat {
|
|
|
3193
3383
|
* @param compressionQuality - The level of compression (a number between 0 and 100).
|
|
3194
3384
|
* @returns An {@link ImagenImageFormat} object for a JPEG image.
|
|
3195
3385
|
*
|
|
3196
|
-
* @
|
|
3386
|
+
* @public
|
|
3197
3387
|
*/
|
|
3198
3388
|
static jpeg(compressionQuality) {
|
|
3199
3389
|
if (compressionQuality &&
|
|
@@ -3207,7 +3397,7 @@ class ImagenImageFormat {
|
|
|
3207
3397
|
*
|
|
3208
3398
|
* @returns An {@link ImagenImageFormat} object for a PNG image.
|
|
3209
3399
|
*
|
|
3210
|
-
* @
|
|
3400
|
+
* @public
|
|
3211
3401
|
*/
|
|
3212
3402
|
static png() {
|
|
3213
3403
|
return { mimeType: 'image/png' };
|
|
@@ -3327,7 +3517,7 @@ class AudioConversationRunner {
|
|
|
3327
3517
|
mimeType: 'audio/pcm',
|
|
3328
3518
|
data: base64
|
|
3329
3519
|
};
|
|
3330
|
-
void this.liveSession.
|
|
3520
|
+
void this.liveSession.sendAudioRealtime(chunk);
|
|
3331
3521
|
};
|
|
3332
3522
|
}
|
|
3333
3523
|
/**
|
|
@@ -3449,9 +3639,9 @@ class AudioConversationRunner {
|
|
|
3449
3639
|
}
|
|
3450
3640
|
else {
|
|
3451
3641
|
try {
|
|
3452
|
-
const
|
|
3642
|
+
const functionResponse = await this.options.functionCallingHandler(message.functionCalls);
|
|
3453
3643
|
if (!this.isStopped) {
|
|
3454
|
-
void this.liveSession.
|
|
3644
|
+
void this.liveSession.sendFunctionResponses([functionResponse]);
|
|
3455
3645
|
}
|
|
3456
3646
|
}
|
|
3457
3647
|
catch (e) {
|
|
@@ -3671,7 +3861,7 @@ function getGenerativeModel(ai, modelParams, requestOptions) {
|
|
|
3671
3861
|
* @throws If the `apiKey` or `projectId` fields are missing in your
|
|
3672
3862
|
* Firebase config.
|
|
3673
3863
|
*
|
|
3674
|
-
* @
|
|
3864
|
+
* @public
|
|
3675
3865
|
*/
|
|
3676
3866
|
function getImagenModel(ai, modelParams, requestOptions) {
|
|
3677
3867
|
if (!modelParams.model) {
|
|
@@ -3722,5 +3912,5 @@ function registerAI() {
|
|
|
3722
3912
|
}
|
|
3723
3913
|
registerAI();
|
|
3724
3914
|
|
|
3725
|
-
export { AIError, AIErrorCode, AIModel, AnyOfSchema, ArraySchema, Backend, BackendType, BlockReason, BooleanSchema, ChatSession, FinishReason, FunctionCallingMode, GenerativeModel, GoogleAIBackend, HarmBlockMethod, HarmBlockThreshold, HarmCategory, HarmProbability, HarmSeverity, ImagenAspectRatio, ImagenImageFormat, ImagenModel, ImagenPersonFilterLevel, ImagenSafetyFilterLevel, InferenceMode, IntegerSchema, Language, LiveGenerativeModel, LiveResponseType, LiveSession, Modality, NumberSchema, ObjectSchema, Outcome, POSSIBLE_ROLES, ResponseModality, Schema, SchemaType, StringSchema, VertexAIBackend, getAI, getGenerativeModel, getImagenModel, getLiveGenerativeModel, startAudioConversation };
|
|
3915
|
+
export { AIError, AIErrorCode, AIModel, AnyOfSchema, ArraySchema, Backend, BackendType, BlockReason, BooleanSchema, ChatSession, FinishReason, FunctionCallingMode, GenerativeModel, GoogleAIBackend, HarmBlockMethod, HarmBlockThreshold, HarmCategory, HarmProbability, HarmSeverity, ImagenAspectRatio, ImagenImageFormat, ImagenModel, ImagenPersonFilterLevel, ImagenSafetyFilterLevel, InferenceMode, InferenceSource, IntegerSchema, Language, LiveGenerativeModel, LiveResponseType, LiveSession, Modality, NumberSchema, ObjectSchema, Outcome, POSSIBLE_ROLES, ResponseModality, Schema, SchemaType, StringSchema, URLRetrievalStatus, VertexAIBackend, getAI, getGenerativeModel, getImagenModel, getLiveGenerativeModel, startAudioConversation };
|
|
3726
3916
|
//# sourceMappingURL=index.node.mjs.map
|