@mastra/voice-google-gemini-live 0.11.0-beta.1 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -58
- package/README.md +3 -3
- package/dist/index.cjs +211 -195
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +7 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +211 -195
- package/dist/index.js.map +1 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +13 -11
package/CHANGELOG.md
CHANGED
|
@@ -1,80 +1,53 @@
|
|
|
1
1
|
# @mastra/voice-google-gemini-live
|
|
2
2
|
|
|
3
|
-
## 0.11.0
|
|
3
|
+
## 0.11.0
|
|
4
4
|
|
|
5
5
|
### Minor Changes
|
|
6
6
|
|
|
7
|
-
-
|
|
7
|
+
- Backport gemini fix to 0.x ([#10234](https://github.com/mastra-ai/mastra/pull/10234))
|
|
8
8
|
|
|
9
9
|
### Patch Changes
|
|
10
10
|
|
|
11
|
-
- Updated dependencies [[`
|
|
12
|
-
- @mastra/core@
|
|
11
|
+
- Updated dependencies [[`eebb7bb`](https://github.com/mastra-ai/mastra/commit/eebb7bb407c57342a3be3a2efbe68c696589feeb), [`c6e6d07`](https://github.com/mastra-ai/mastra/commit/c6e6d071ecf346a80dceab410af2c567c7e66a57), [`0e6df8f`](https://github.com/mastra-ai/mastra/commit/0e6df8f66340992cb1b319834657deb17368de52), [`6295fd7`](https://github.com/mastra-ai/mastra/commit/6295fd783d49075d5bf2c18ff9486e94d36aaa56), [`d813cf7`](https://github.com/mastra-ai/mastra/commit/d813cf7251695d85cc60469058384ffa74974484)]:
|
|
12
|
+
- @mastra/core@0.24.2
|
|
13
13
|
|
|
14
|
-
## 0.11.0-
|
|
14
|
+
## 0.11.0-alpha.0
|
|
15
15
|
|
|
16
16
|
### Minor Changes
|
|
17
17
|
|
|
18
|
-
-
|
|
18
|
+
- Backport gemini fix to 0.x ([#10234](https://github.com/mastra-ai/mastra/pull/10234))
|
|
19
19
|
|
|
20
|
-
|
|
20
|
+
### Patch Changes
|
|
21
|
+
|
|
22
|
+
- Updated dependencies [[`eebb7bb`](https://github.com/mastra-ai/mastra/commit/eebb7bb407c57342a3be3a2efbe68c696589feeb), [`c6e6d07`](https://github.com/mastra-ai/mastra/commit/c6e6d071ecf346a80dceab410af2c567c7e66a57), [`0e6df8f`](https://github.com/mastra-ai/mastra/commit/0e6df8f66340992cb1b319834657deb17368de52), [`6295fd7`](https://github.com/mastra-ai/mastra/commit/6295fd783d49075d5bf2c18ff9486e94d36aaa56), [`d813cf7`](https://github.com/mastra-ai/mastra/commit/d813cf7251695d85cc60469058384ffa74974484)]:
|
|
23
|
+
- @mastra/core@0.24.2-alpha.0
|
|
24
|
+
|
|
25
|
+
## 0.10.18
|
|
26
|
+
|
|
27
|
+
### Patch Changes
|
|
28
|
+
|
|
29
|
+
- update peerdeps ([`5ca1cca`](https://github.com/mastra-ai/mastra/commit/5ca1ccac61ffa7141e6d9fa8f22d3ad4d03bf5dc))
|
|
21
30
|
|
|
22
|
-
-
|
|
31
|
+
- Updated dependencies [[`5ca1cca`](https://github.com/mastra-ai/mastra/commit/5ca1ccac61ffa7141e6d9fa8f22d3ad4d03bf5dc), [`6d7e90d`](https://github.com/mastra-ai/mastra/commit/6d7e90db09713e6250f4d6c3d3cff1b4740e50f9), [`f78b908`](https://github.com/mastra-ai/mastra/commit/f78b9080e11af765969b36b4a619761056030840), [`23c2614`](https://github.com/mastra-ai/mastra/commit/23c26140fdbf04b8c59e8d7d52106d67dad962ec), [`e365eda`](https://github.com/mastra-ai/mastra/commit/e365eda45795b43707310531cac1e2ce4e5a0712)]:
|
|
32
|
+
- @mastra/core@0.24.0
|
|
23
33
|
|
|
24
|
-
|
|
34
|
+
## 0.10.18-alpha.0
|
|
25
35
|
|
|
26
|
-
|
|
36
|
+
### Patch Changes
|
|
37
|
+
|
|
38
|
+
- update peerdeps ([`5ca1cca`](https://github.com/mastra-ai/mastra/commit/5ca1ccac61ffa7141e6d9fa8f22d3ad4d03bf5dc))
|
|
39
|
+
|
|
40
|
+
- Updated dependencies [[`5ca1cca`](https://github.com/mastra-ai/mastra/commit/5ca1ccac61ffa7141e6d9fa8f22d3ad4d03bf5dc), [`6d7e90d`](https://github.com/mastra-ai/mastra/commit/6d7e90db09713e6250f4d6c3d3cff1b4740e50f9), [`f78b908`](https://github.com/mastra-ai/mastra/commit/f78b9080e11af765969b36b4a619761056030840), [`23c2614`](https://github.com/mastra-ai/mastra/commit/23c26140fdbf04b8c59e8d7d52106d67dad962ec), [`e365eda`](https://github.com/mastra-ai/mastra/commit/e365eda45795b43707310531cac1e2ce4e5a0712)]:
|
|
41
|
+
- @mastra/core@0.24.0-alpha.0
|
|
42
|
+
|
|
43
|
+
## 0.10.17
|
|
27
44
|
|
|
28
45
|
### Patch Changes
|
|
29
46
|
|
|
30
|
-
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
```typescript
|
|
35
|
-
// before depending on the context the tool was executed in
|
|
36
|
-
tool.execute({ context: data });
|
|
37
|
-
tool.execute({ context: { inputData: data } });
|
|
38
|
-
tool.execute(data);
|
|
39
|
-
|
|
40
|
-
// now, for all contexts
|
|
41
|
-
tool.execute(data, context);
|
|
42
|
-
```
|
|
43
|
-
|
|
44
|
-
**Before:**
|
|
45
|
-
|
|
46
|
-
```typescript
|
|
47
|
-
inputSchema: z.object({ something: z.string() }),
|
|
48
|
-
execute: async ({ context, tracingContext, runId, ... }) => {
|
|
49
|
-
return doSomething(context.string);
|
|
50
|
-
}
|
|
51
|
-
```
|
|
52
|
-
|
|
53
|
-
**After:**
|
|
54
|
-
|
|
55
|
-
```typescript
|
|
56
|
-
inputSchema: z.object({ something: z.string() }),
|
|
57
|
-
execute: async (inputData, context) => {
|
|
58
|
-
const { agent, mcp, workflow, ...sharedContext } = context
|
|
59
|
-
|
|
60
|
-
// context that only an agent would get like toolCallId, messages, suspend, resume, etc
|
|
61
|
-
if (agent) {
|
|
62
|
-
doSomething(inputData.something, agent)
|
|
63
|
-
// context that only a workflow would get like runId, state, suspend, resume, etc
|
|
64
|
-
} else if (workflow) {
|
|
65
|
-
doSomething(inputData.something, workflow)
|
|
66
|
-
// context that only a workflow would get like "extra", "elicitation"
|
|
67
|
-
} else if (mcp) {
|
|
68
|
-
doSomething(inputData.something, mcp)
|
|
69
|
-
} else {
|
|
70
|
-
// Running a tool in no execution context
|
|
71
|
-
return doSomething(inputData.something);
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
```
|
|
75
|
-
|
|
76
|
-
- Updated dependencies [[`39c9743`](https://github.com/mastra-ai/mastra/commit/39c97432d084294f8ba85fbf3ef28098ff21459e), [`f743dbb`](https://github.com/mastra-ai/mastra/commit/f743dbb8b40d1627b5c10c0e6fc154f4ebb6e394), [`fec5129`](https://github.com/mastra-ai/mastra/commit/fec5129de7fc64423ea03661a56cef31dc747a0d), [`0491e7c`](https://github.com/mastra-ai/mastra/commit/0491e7c9b714cb0ba22187ee062147ec2dd7c712), [`f6f4903`](https://github.com/mastra-ai/mastra/commit/f6f4903397314f73362061dc5a3e8e7c61ea34aa), [`0e8ed46`](https://github.com/mastra-ai/mastra/commit/0e8ed467c54d6901a6a365f270ec15d6faadb36c), [`6c049d9`](https://github.com/mastra-ai/mastra/commit/6c049d94063fdcbd5b81c4912a2bf82a92c9cc0b), [`2f897df`](https://github.com/mastra-ai/mastra/commit/2f897df208508f46f51b7625e5dd20c37f93e0e3), [`3443770`](https://github.com/mastra-ai/mastra/commit/3443770662df8eb24c9df3589b2792d78cfcb811), [`f0a07e0`](https://github.com/mastra-ai/mastra/commit/f0a07e0111b3307c5fabfa4094c5c2cfb734fbe6), [`aaa40e7`](https://github.com/mastra-ai/mastra/commit/aaa40e788628b319baa8e889407d11ad626547fa), [`1521d71`](https://github.com/mastra-ai/mastra/commit/1521d716e5daedc74690c983fbd961123c56756b), [`9e1911d`](https://github.com/mastra-ai/mastra/commit/9e1911db2b4db85e0e768c3f15e0d61e319869f6), [`ebac155`](https://github.com/mastra-ai/mastra/commit/ebac15564a590117db7078233f927a7e28a85106), [`dd1c38d`](https://github.com/mastra-ai/mastra/commit/dd1c38d1b75f1b695c27b40d8d9d6ed00d5e0f6f), [`5948e6a`](https://github.com/mastra-ai/mastra/commit/5948e6a5146c83666ba3f294b2be576c82a513fb), [`8940859`](https://github.com/mastra-ai/mastra/commit/89408593658199b4ad67f7b65e888f344e64a442), [`e629310`](https://github.com/mastra-ai/mastra/commit/e629310f1a73fa236d49ec7a1d1cceb6229dc7cc), [`4c6b492`](https://github.com/mastra-ai/mastra/commit/4c6b492c4dd591c6a592520c1f6855d6e936d71f), [`dff01d8`](https://github.com/mastra-ai/mastra/commit/dff01d81ce1f4e4087cfac20fa868e6db138dd14), [`9d819d5`](https://github.com/mastra-ai/mastra/commit/9d819d54b61481639f4008e4694791bddf187edd), [`71c8d6c`](https://github.com/mastra-ai/mastra/commit/71c8d6c161253207b2b9588bdadb7eed604f7253), [`6179a9b`](https://github.com/mastra-ai/mastra/commit/6179a9ba36ffac326de3cc3c43cdc8028d37c251), [`00f4921`](https://github.com/mastra-ai/mastra/commit/00f4921dd2c91a1e5446799599ef7116a8214a1a), [`ca8041c`](https://github.com/mastra-ai/mastra/commit/ca8041cce0379fda22ed293a565bcb5b6ddca68a), [`7051bf3`](https://github.com/mastra-ai/mastra/commit/7051bf38b3b122a069008f861f7bfc004a6d9f6e), [`a8f1494`](https://github.com/mastra-ai/mastra/commit/a8f1494f4bbdc2770bcf327d4c7d869e332183f1), [`0793497`](https://github.com/mastra-ai/mastra/commit/079349753620c40246ffd673e3f9d7d9820beff3), [`5df9cce`](https://github.com/mastra-ai/mastra/commit/5df9cce1a753438413f64c11eeef8f845745c2a8), [`a854ede`](https://github.com/mastra-ai/mastra/commit/a854ede62bf5ac0945a624ac48913dd69c73aabf), [`c576fc0`](https://github.com/mastra-ai/mastra/commit/c576fc0b100b2085afded91a37c97a0ea0ec09c7), [`3defc80`](https://github.com/mastra-ai/mastra/commit/3defc80cf2b88a1b7fc1cc4ddcb91e982a614609), [`16153fe`](https://github.com/mastra-ai/mastra/commit/16153fe7eb13c99401f48e6ca32707c965ee28b9), [`9f4a683`](https://github.com/mastra-ai/mastra/commit/9f4a6833e88b52574665c028fd5508ad5c2f6004), [`bc94344`](https://github.com/mastra-ai/mastra/commit/bc943444a1342d8a662151b7bce1df7dae32f59c), [`57d157f`](https://github.com/mastra-ai/mastra/commit/57d157f0b163a95c3e6c9eae31bdb11d1bfc64f9), [`903f67d`](https://github.com/mastra-ai/mastra/commit/903f67d184504a273893818c02b961f5423a79ad), [`2a90c55`](https://github.com/mastra-ai/mastra/commit/2a90c55a86a9210697d5adaab5ee94584b079adc), [`eb09742`](https://github.com/mastra-ai/mastra/commit/eb09742197f66c4c38154c3beec78313e69760b2), [`96d35f6`](https://github.com/mastra-ai/mastra/commit/96d35f61376bc2b1bf148648a2c1985bd51bef55), [`5cbe88a`](https://github.com/mastra-ai/mastra/commit/5cbe88aefbd9f933bca669fd371ea36bf939ac6d), [`a1bd7b8`](https://github.com/mastra-ai/mastra/commit/a1bd7b8571db16b94eb01588f451a74758c96d65), [`d78b38d`](https://github.com/mastra-ai/mastra/commit/d78b38d898fce285260d3bbb4befade54331617f), [`0633100`](https://github.com/mastra-ai/mastra/commit/0633100a911ad22f5256471bdf753da21c104742), [`c710c16`](https://github.com/mastra-ai/mastra/commit/c710c1652dccfdc4111c8412bca7a6bb1d48b441), [`354ad0b`](https://github.com/mastra-ai/mastra/commit/354ad0b7b1b8183ac567f236a884fc7ede6d7138), [`cfae733`](https://github.com/mastra-ai/mastra/commit/cfae73394f4920635e6c919c8e95ff9a0788e2e5), [`e3dfda7`](https://github.com/mastra-ai/mastra/commit/e3dfda7b11bf3b8c4bb55637028befb5f387fc74), [`844ea5d`](https://github.com/mastra-ai/mastra/commit/844ea5dc0c248961e7bf73629ae7dcff503e853c), [`398fde3`](https://github.com/mastra-ai/mastra/commit/398fde3f39e707cda79372cdae8f9870e3b57c8d), [`f0f8f12`](https://github.com/mastra-ai/mastra/commit/f0f8f125c308f2d0fd36942ef652fd852df7522f), [`0d7618b`](https://github.com/mastra-ai/mastra/commit/0d7618bc650bf2800934b243eca5648f4aeed9c2), [`7b763e5`](https://github.com/mastra-ai/mastra/commit/7b763e52fc3eaf699c2a99f2adf418dd46e4e9a5), [`d36cfbb`](https://github.com/mastra-ai/mastra/commit/d36cfbbb6565ba5f827883cc9bb648eb14befdc1), [`3697853`](https://github.com/mastra-ai/mastra/commit/3697853deeb72017d90e0f38a93c1e29221aeca0), [`b2e45ec`](https://github.com/mastra-ai/mastra/commit/b2e45eca727a8db01a81ba93f1a5219c7183c839), [`d6d49f7`](https://github.com/mastra-ai/mastra/commit/d6d49f7b8714fa19a52ff9c7cf7fb7e73751901e), [`a534e95`](https://github.com/mastra-ai/mastra/commit/a534e9591f83b3cc1ebff99c67edf4cda7bf81d3), [`9d0e7fe`](https://github.com/mastra-ai/mastra/commit/9d0e7feca8ed98de959f53476ee1456073673348), [`53d927c`](https://github.com/mastra-ai/mastra/commit/53d927cc6f03bff33655b7e2b788da445a08731d), [`3f2faf2`](https://github.com/mastra-ai/mastra/commit/3f2faf2e2d685d6c053cc5af1bf9fedf267b2ce5), [`22f64bc`](https://github.com/mastra-ai/mastra/commit/22f64bc1d37149480b58bf2fefe35b79a1e3e7d5), [`83d5942`](https://github.com/mastra-ai/mastra/commit/83d5942669ce7bba4a6ca4fd4da697a10eb5ebdc), [`b7959e6`](https://github.com/mastra-ai/mastra/commit/b7959e6e25a46b480f9ea2217c4c6c588c423791), [`bda6370`](https://github.com/mastra-ai/mastra/commit/bda637009360649aaf579919e7873e33553c273e), [`d7acd8e`](https://github.com/mastra-ai/mastra/commit/d7acd8e987b5d7eff4fd98b0906c17c06a2e83d5), [`c7f1f7d`](https://github.com/mastra-ai/mastra/commit/c7f1f7d24f61f247f018cc2d1f33bf63212959a7), [`0bddc6d`](https://github.com/mastra-ai/mastra/commit/0bddc6d8dbd6f6008c0cba2e4960a2da75a55af1), [`735d8c1`](https://github.com/mastra-ai/mastra/commit/735d8c1c0d19fbc09e6f8b66cf41bc7655993838), [`acf322e`](https://github.com/mastra-ai/mastra/commit/acf322e0f1fd0189684cf529d91c694bea918a45), [`c942802`](https://github.com/mastra-ai/mastra/commit/c942802a477a925b01859a7b8688d4355715caaa), [`a0c8c1b`](https://github.com/mastra-ai/mastra/commit/a0c8c1b87d4fee252aebda73e8637fbe01d761c9), [`cc34739`](https://github.com/mastra-ai/mastra/commit/cc34739c34b6266a91bea561119240a7acf47887), [`c218bd3`](https://github.com/mastra-ai/mastra/commit/c218bd3759e32423735b04843a09404572631014), [`2c4438b`](https://github.com/mastra-ai/mastra/commit/2c4438b87817ab7eed818c7990fef010475af1a3), [`2b8893c`](https://github.com/mastra-ai/mastra/commit/2b8893cb108ef9acb72ee7835cd625610d2c1a4a), [`8e5c75b`](https://github.com/mastra-ai/mastra/commit/8e5c75bdb1d08a42d45309a4c72def4b6890230f), [`e59e0d3`](https://github.com/mastra-ai/mastra/commit/e59e0d32afb5fcf2c9f3c00c8f81f6c21d3a63fa), [`fa8409b`](https://github.com/mastra-ai/mastra/commit/fa8409bc39cfd8ba6643b9db5269b90b22e2a2f7), [`173c535`](https://github.com/mastra-ai/mastra/commit/173c535c0645b0da404fe09f003778f0b0d4e019)]:
|
|
77
|
-
- @mastra/core@1.0.0-beta.0
|
|
47
|
+
- Fix peerdependencies ([`eb7c1c8`](https://github.com/mastra-ai/mastra/commit/eb7c1c8c592d8fb16dfd250e337d9cdc73c8d5de))
|
|
48
|
+
|
|
49
|
+
- Updated dependencies []:
|
|
50
|
+
- @mastra/core@0.23.1
|
|
78
51
|
|
|
79
52
|
## 0.10.16
|
|
80
53
|
|
package/README.md
CHANGED
|
@@ -337,8 +337,8 @@ const searchTool = createTool({
|
|
|
337
337
|
id: 'search',
|
|
338
338
|
description: 'Search the web',
|
|
339
339
|
inputSchema: z.object({ query: z.string() }),
|
|
340
|
-
execute: async
|
|
341
|
-
const { query } =
|
|
340
|
+
execute: async ({ context }) => {
|
|
341
|
+
const { query } = context;
|
|
342
342
|
// ... perform search
|
|
343
343
|
return { results: [] };
|
|
344
344
|
},
|
|
@@ -355,7 +355,7 @@ voice.addTools({
|
|
|
355
355
|
id: 'search',
|
|
356
356
|
description: 'Search the web',
|
|
357
357
|
inputSchema: { type: 'object', properties: { query: { type: 'string' } } },
|
|
358
|
-
execute: async (
|
|
358
|
+
execute: async ({ context }) => ({ results: [] }),
|
|
359
359
|
},
|
|
360
360
|
});
|
|
361
361
|
```
|
package/dist/index.cjs
CHANGED
|
@@ -1262,7 +1262,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
|
|
|
1262
1262
|
sessionDurationTimeout;
|
|
1263
1263
|
// Tool integration properties
|
|
1264
1264
|
tools;
|
|
1265
|
-
|
|
1265
|
+
runtimeContext;
|
|
1266
1266
|
// Store the configuration options
|
|
1267
1267
|
options;
|
|
1268
1268
|
/**
|
|
@@ -1499,67 +1499,69 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
|
|
|
1499
1499
|
/**
|
|
1500
1500
|
* Establish connection to the Gemini Live API
|
|
1501
1501
|
*/
|
|
1502
|
-
async connect({
|
|
1503
|
-
|
|
1504
|
-
this.
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
this.requestContext = requestContext;
|
|
1508
|
-
this.emit("session", { state: "connecting" });
|
|
1509
|
-
try {
|
|
1510
|
-
let wsUrl;
|
|
1511
|
-
let headers = {};
|
|
1512
|
-
if (this.options.vertexAI) {
|
|
1513
|
-
wsUrl = `wss://${this.options.location}-aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1beta1.PredictionService.ServerStreamingPredict`;
|
|
1514
|
-
await this.authManager.initialize();
|
|
1515
|
-
const accessToken = await this.authManager.getAccessToken();
|
|
1516
|
-
headers = { headers: { Authorization: `Bearer ${accessToken}` } };
|
|
1517
|
-
this.log("Using Vertex AI authentication with OAuth token");
|
|
1518
|
-
} else {
|
|
1519
|
-
wsUrl = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent`;
|
|
1520
|
-
headers = {
|
|
1521
|
-
headers: {
|
|
1522
|
-
"x-goog-api-key": this.options.apiKey || "",
|
|
1523
|
-
"Content-Type": "application/json"
|
|
1524
|
-
}
|
|
1525
|
-
};
|
|
1526
|
-
this.log("Using Live API authentication with API key");
|
|
1527
|
-
}
|
|
1528
|
-
this.log("Connecting to:", wsUrl);
|
|
1529
|
-
this.ws = new ws.WebSocket(wsUrl, void 0, headers);
|
|
1530
|
-
this.connectionManager.setWebSocket(this.ws);
|
|
1531
|
-
this.setupEventListeners();
|
|
1532
|
-
await this.connectionManager.waitForOpen();
|
|
1533
|
-
if (this.isResuming && this.sessionHandle) {
|
|
1534
|
-
await this.sendSessionResumption();
|
|
1535
|
-
} else {
|
|
1536
|
-
this.sendInitialConfig();
|
|
1537
|
-
this.sessionStartTime = Date.now();
|
|
1538
|
-
this.sessionId = crypto.randomUUID();
|
|
1502
|
+
async connect({ runtimeContext } = {}) {
|
|
1503
|
+
return this.traced(async () => {
|
|
1504
|
+
if (this.state === "connected") {
|
|
1505
|
+
this.log("Already connected to Gemini Live API");
|
|
1506
|
+
return;
|
|
1539
1507
|
}
|
|
1540
|
-
|
|
1541
|
-
this.state
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1508
|
+
this.runtimeContext = runtimeContext;
|
|
1509
|
+
this.emit("session", { state: "connecting" });
|
|
1510
|
+
try {
|
|
1511
|
+
let wsUrl;
|
|
1512
|
+
let headers = {};
|
|
1513
|
+
if (this.options.vertexAI) {
|
|
1514
|
+
wsUrl = `wss://${this.options.location}-aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1beta1.PredictionService.ServerStreamingPredict`;
|
|
1515
|
+
await this.authManager.initialize();
|
|
1516
|
+
const accessToken = await this.authManager.getAccessToken();
|
|
1517
|
+
headers = { headers: { Authorization: `Bearer ${accessToken}` } };
|
|
1518
|
+
this.log("Using Vertex AI authentication with OAuth token");
|
|
1519
|
+
} else {
|
|
1520
|
+
wsUrl = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent`;
|
|
1521
|
+
headers = {
|
|
1522
|
+
headers: {
|
|
1523
|
+
"x-goog-api-key": this.options.apiKey || "",
|
|
1524
|
+
"Content-Type": "application/json"
|
|
1525
|
+
}
|
|
1526
|
+
};
|
|
1527
|
+
this.log("Using Live API authentication with API key");
|
|
1528
|
+
}
|
|
1529
|
+
this.log("Connecting to:", wsUrl);
|
|
1530
|
+
this.ws = new ws.WebSocket(wsUrl, void 0, headers);
|
|
1531
|
+
this.connectionManager.setWebSocket(this.ws);
|
|
1532
|
+
this.setupEventListeners();
|
|
1533
|
+
await this.connectionManager.waitForOpen();
|
|
1534
|
+
if (this.isResuming && this.sessionHandle) {
|
|
1535
|
+
await this.sendSessionResumption();
|
|
1536
|
+
} else {
|
|
1537
|
+
this.sendInitialConfig();
|
|
1538
|
+
this.sessionStartTime = Date.now();
|
|
1539
|
+
this.sessionId = crypto.randomUUID();
|
|
1540
|
+
}
|
|
1541
|
+
await this.waitForSessionCreated();
|
|
1542
|
+
this.state = "connected";
|
|
1543
|
+
this.emit("session", {
|
|
1544
|
+
state: "connected",
|
|
1545
|
+
config: {
|
|
1546
|
+
sessionId: this.sessionId,
|
|
1547
|
+
isResuming: this.isResuming,
|
|
1548
|
+
toolCount: Object.keys(this.tools || {}).length
|
|
1549
|
+
}
|
|
1550
|
+
});
|
|
1551
|
+
this.log("Successfully connected to Gemini Live API", {
|
|
1545
1552
|
sessionId: this.sessionId,
|
|
1546
1553
|
isResuming: this.isResuming,
|
|
1547
1554
|
toolCount: Object.keys(this.tools || {}).length
|
|
1555
|
+
});
|
|
1556
|
+
if (this.options.sessionConfig?.maxDuration) {
|
|
1557
|
+
this.startSessionDurationMonitor();
|
|
1548
1558
|
}
|
|
1549
|
-
})
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
toolCount: Object.keys(this.tools || {}).length
|
|
1554
|
-
});
|
|
1555
|
-
if (this.options.sessionConfig?.maxDuration) {
|
|
1556
|
-
this.startSessionDurationMonitor();
|
|
1559
|
+
} catch (error) {
|
|
1560
|
+
this.state = "disconnected";
|
|
1561
|
+
this.log("Connection failed", error);
|
|
1562
|
+
throw error;
|
|
1557
1563
|
}
|
|
1558
|
-
}
|
|
1559
|
-
this.state = "disconnected";
|
|
1560
|
-
this.log("Connection failed", error);
|
|
1561
|
-
throw error;
|
|
1562
|
-
}
|
|
1564
|
+
}, "gemini-live.connect")();
|
|
1563
1565
|
}
|
|
1564
1566
|
/**
|
|
1565
1567
|
* Disconnect from the Gemini Live API
|
|
@@ -1597,164 +1599,172 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
|
|
|
1597
1599
|
* Send text to be converted to speech
|
|
1598
1600
|
*/
|
|
1599
1601
|
async speak(input, options) {
|
|
1600
|
-
this.
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1602
|
+
return this.traced(async () => {
|
|
1603
|
+
this.validateConnectionState();
|
|
1604
|
+
if (typeof input !== "string") {
|
|
1605
|
+
const chunks = [];
|
|
1606
|
+
for await (const chunk of input) {
|
|
1607
|
+
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)));
|
|
1608
|
+
}
|
|
1609
|
+
input = Buffer.concat(chunks).toString("utf-8");
|
|
1605
1610
|
}
|
|
1606
|
-
input
|
|
1607
|
-
|
|
1608
|
-
if (input.trim().length === 0) {
|
|
1609
|
-
throw this.createAndEmitError("invalid_audio_format" /* INVALID_AUDIO_FORMAT */, "Input text is empty");
|
|
1610
|
-
}
|
|
1611
|
-
this.addToContext("user", input);
|
|
1612
|
-
const textMessage = {
|
|
1613
|
-
client_content: {
|
|
1614
|
-
turns: [
|
|
1615
|
-
{
|
|
1616
|
-
role: "user",
|
|
1617
|
-
parts: [
|
|
1618
|
-
{
|
|
1619
|
-
text: input
|
|
1620
|
-
}
|
|
1621
|
-
]
|
|
1622
|
-
}
|
|
1623
|
-
],
|
|
1624
|
-
turnComplete: true
|
|
1611
|
+
if (input.trim().length === 0) {
|
|
1612
|
+
throw this.createAndEmitError("invalid_audio_format" /* INVALID_AUDIO_FORMAT */, "Input text is empty");
|
|
1625
1613
|
}
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1614
|
+
this.addToContext("user", input);
|
|
1615
|
+
const textMessage = {
|
|
1616
|
+
client_content: {
|
|
1617
|
+
turns: [
|
|
1618
|
+
{
|
|
1619
|
+
role: "user",
|
|
1620
|
+
parts: [
|
|
1621
|
+
{
|
|
1622
|
+
text: input
|
|
1623
|
+
}
|
|
1624
|
+
]
|
|
1636
1625
|
}
|
|
1637
|
-
|
|
1626
|
+
],
|
|
1627
|
+
turnComplete: true
|
|
1638
1628
|
}
|
|
1639
1629
|
};
|
|
1630
|
+
if (options && (options.speaker || options.languageCode || options.responseModalities)) {
|
|
1631
|
+
const updateMessage = {
|
|
1632
|
+
type: "session.update",
|
|
1633
|
+
session: {
|
|
1634
|
+
generation_config: {
|
|
1635
|
+
...options.responseModalities ? { response_modalities: options.responseModalities } : {},
|
|
1636
|
+
speech_config: {
|
|
1637
|
+
...options.languageCode ? { language_code: options.languageCode } : {},
|
|
1638
|
+
...options.speaker ? { voice_config: { prebuilt_voice_config: { voice_name: options.speaker } } } : {}
|
|
1639
|
+
}
|
|
1640
|
+
}
|
|
1641
|
+
}
|
|
1642
|
+
};
|
|
1643
|
+
try {
|
|
1644
|
+
this.sendEvent("session.update", updateMessage);
|
|
1645
|
+
this.log("Applied per-turn runtime options", options);
|
|
1646
|
+
} catch (error) {
|
|
1647
|
+
this.log("Failed to apply per-turn runtime options", error);
|
|
1648
|
+
}
|
|
1649
|
+
}
|
|
1640
1650
|
try {
|
|
1641
|
-
this.sendEvent("
|
|
1642
|
-
this.log("
|
|
1651
|
+
this.sendEvent("client_content", textMessage);
|
|
1652
|
+
this.log("Text message sent", { text: input });
|
|
1643
1653
|
} catch (error) {
|
|
1644
|
-
this.log("Failed to
|
|
1654
|
+
this.log("Failed to send text message", error);
|
|
1655
|
+
throw this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to send text message", error);
|
|
1645
1656
|
}
|
|
1646
|
-
}
|
|
1647
|
-
try {
|
|
1648
|
-
this.sendEvent("client_content", textMessage);
|
|
1649
|
-
this.log("Text message sent", { text: input });
|
|
1650
|
-
} catch (error) {
|
|
1651
|
-
this.log("Failed to send text message", error);
|
|
1652
|
-
throw this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to send text message", error);
|
|
1653
|
-
}
|
|
1657
|
+
}, "gemini-live.speak")();
|
|
1654
1658
|
}
|
|
1655
1659
|
/**
|
|
1656
1660
|
* Send audio stream for processing
|
|
1657
1661
|
*/
|
|
1658
1662
|
async send(audioData) {
|
|
1659
|
-
this.
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1663
|
+
return this.traced(async () => {
|
|
1664
|
+
this.validateConnectionState();
|
|
1665
|
+
if ("readable" in audioData && typeof audioData.on === "function") {
|
|
1666
|
+
const stream = audioData;
|
|
1667
|
+
stream.on("data", (chunk) => {
|
|
1668
|
+
try {
|
|
1669
|
+
const base64Audio = this.audioStreamManager.processAudioChunk(chunk);
|
|
1670
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "realtime");
|
|
1671
|
+
this.sendEvent("realtime_input", message);
|
|
1672
|
+
} catch (error) {
|
|
1673
|
+
this.log("Failed to process audio chunk", error);
|
|
1674
|
+
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to process audio chunk", error);
|
|
1675
|
+
}
|
|
1676
|
+
});
|
|
1677
|
+
stream.on("error", (error) => {
|
|
1678
|
+
this.log("Audio stream error", error);
|
|
1679
|
+
this.createAndEmitError("audio_stream_error" /* AUDIO_STREAM_ERROR */, "Audio stream error", error);
|
|
1680
|
+
});
|
|
1681
|
+
stream.on("end", () => {
|
|
1682
|
+
this.log("Audio stream ended");
|
|
1683
|
+
});
|
|
1684
|
+
} else {
|
|
1685
|
+
const validateAudio = this.audioStreamManager.validateAndConvertAudioInput(audioData);
|
|
1686
|
+
const base64Audio = this.audioStreamManager.int16ArrayToBase64(validateAudio);
|
|
1687
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "realtime");
|
|
1688
|
+
this.sendEvent("realtime_input", message);
|
|
1689
|
+
}
|
|
1690
|
+
}, "gemini-live.send")();
|
|
1685
1691
|
}
|
|
1686
1692
|
/**
|
|
1687
1693
|
* Process speech from audio stream (traditional STT interface)
|
|
1688
1694
|
*/
|
|
1689
1695
|
async listen(audioStream, _options) {
|
|
1690
|
-
this.
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
};
|
|
1698
|
-
const onError = (error) => {
|
|
1699
|
-
throw new Error(`Transcription failed: ${error.message}`);
|
|
1700
|
-
};
|
|
1701
|
-
const onSession = (data) => {
|
|
1702
|
-
if (data.state === "disconnected") {
|
|
1703
|
-
throw new Error("Session disconnected during transcription");
|
|
1704
|
-
}
|
|
1705
|
-
};
|
|
1706
|
-
this.on("writing", onWriting);
|
|
1707
|
-
this.on("error", onError);
|
|
1708
|
-
this.on("session", onSession);
|
|
1709
|
-
try {
|
|
1710
|
-
const result = await this.audioStreamManager.handleAudioTranscription(
|
|
1711
|
-
audioStream,
|
|
1712
|
-
(base64Audio) => {
|
|
1713
|
-
return new Promise((resolve, reject) => {
|
|
1714
|
-
try {
|
|
1715
|
-
const message = this.audioStreamManager.createAudioMessage(base64Audio, "input");
|
|
1716
|
-
const cleanup = () => {
|
|
1717
|
-
this.off("turnComplete", onTurnComplete);
|
|
1718
|
-
this.off("error", onErr);
|
|
1719
|
-
};
|
|
1720
|
-
const onTurnComplete = () => {
|
|
1721
|
-
cleanup();
|
|
1722
|
-
resolve(transcriptionText.trim());
|
|
1723
|
-
};
|
|
1724
|
-
const onErr = (e) => {
|
|
1725
|
-
cleanup();
|
|
1726
|
-
reject(new Error(e.message));
|
|
1727
|
-
};
|
|
1728
|
-
this.on("turnComplete", onTurnComplete);
|
|
1729
|
-
this.on("error", onErr);
|
|
1730
|
-
this.sendEvent("client_content", message);
|
|
1731
|
-
this.log("Sent audio for transcription");
|
|
1732
|
-
} catch (err) {
|
|
1733
|
-
reject(err);
|
|
1734
|
-
}
|
|
1735
|
-
});
|
|
1736
|
-
},
|
|
1737
|
-
(error) => {
|
|
1738
|
-
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Audio transcription failed", error);
|
|
1696
|
+
return this.traced(async () => {
|
|
1697
|
+
this.validateConnectionState();
|
|
1698
|
+
let transcriptionText = "";
|
|
1699
|
+
const onWriting = (data) => {
|
|
1700
|
+
if (data.role === "user") {
|
|
1701
|
+
transcriptionText += data.text;
|
|
1702
|
+
this.log("Received transcription text:", { text: data.text, total: transcriptionText });
|
|
1739
1703
|
}
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1704
|
+
};
|
|
1705
|
+
const onError = (error) => {
|
|
1706
|
+
throw new Error(`Transcription failed: ${error.message}`);
|
|
1707
|
+
};
|
|
1708
|
+
const onSession = (data) => {
|
|
1709
|
+
if (data.state === "disconnected") {
|
|
1710
|
+
throw new Error("Session disconnected during transcription");
|
|
1711
|
+
}
|
|
1712
|
+
};
|
|
1713
|
+
this.on("writing", onWriting);
|
|
1714
|
+
this.on("error", onError);
|
|
1715
|
+
this.on("session", onSession);
|
|
1716
|
+
try {
|
|
1717
|
+
const result = await this.audioStreamManager.handleAudioTranscription(
|
|
1718
|
+
audioStream,
|
|
1719
|
+
(base64Audio) => {
|
|
1720
|
+
return new Promise((resolve, reject) => {
|
|
1721
|
+
try {
|
|
1722
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "input");
|
|
1723
|
+
const cleanup = () => {
|
|
1724
|
+
this.off("turnComplete", onTurnComplete);
|
|
1725
|
+
this.off("error", onErr);
|
|
1726
|
+
};
|
|
1727
|
+
const onTurnComplete = () => {
|
|
1728
|
+
cleanup();
|
|
1729
|
+
resolve(transcriptionText.trim());
|
|
1730
|
+
};
|
|
1731
|
+
const onErr = (e) => {
|
|
1732
|
+
cleanup();
|
|
1733
|
+
reject(new Error(e.message));
|
|
1734
|
+
};
|
|
1735
|
+
this.on("turnComplete", onTurnComplete);
|
|
1736
|
+
this.on("error", onErr);
|
|
1737
|
+
this.sendEvent("client_content", message);
|
|
1738
|
+
this.log("Sent audio for transcription");
|
|
1739
|
+
} catch (err) {
|
|
1740
|
+
reject(err);
|
|
1741
|
+
}
|
|
1742
|
+
});
|
|
1743
|
+
},
|
|
1744
|
+
(error) => {
|
|
1745
|
+
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Audio transcription failed", error);
|
|
1746
|
+
}
|
|
1747
|
+
);
|
|
1748
|
+
return result;
|
|
1749
|
+
} finally {
|
|
1750
|
+
this.off("writing", onWriting);
|
|
1751
|
+
this.off("error", onError);
|
|
1752
|
+
this.off("session", onSession);
|
|
1753
|
+
}
|
|
1754
|
+
}, "gemini-live.listen")();
|
|
1747
1755
|
}
|
|
1748
1756
|
/**
|
|
1749
1757
|
* Get available speakers/voices
|
|
1750
1758
|
*/
|
|
1751
1759
|
async getSpeakers() {
|
|
1752
|
-
return
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1760
|
+
return this.traced(async () => {
|
|
1761
|
+
return [
|
|
1762
|
+
{ voiceId: "Puck", description: "Conversational, friendly" },
|
|
1763
|
+
{ voiceId: "Charon", description: "Deep, authoritative" },
|
|
1764
|
+
{ voiceId: "Kore", description: "Neutral, professional" },
|
|
1765
|
+
{ voiceId: "Fenrir", description: "Warm, approachable" }
|
|
1766
|
+
];
|
|
1767
|
+
}, "gemini-live.getSpeakers")();
|
|
1758
1768
|
}
|
|
1759
1769
|
/**
|
|
1760
1770
|
* Resume a previous session using a session handle
|
|
@@ -2382,7 +2392,13 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
|
|
|
2382
2392
|
let result;
|
|
2383
2393
|
if (tool.execute) {
|
|
2384
2394
|
this.log("Executing tool", { toolName, toolArgs });
|
|
2385
|
-
result = await tool.execute(
|
|
2395
|
+
result = await tool.execute(
|
|
2396
|
+
{ context: toolArgs, runtimeContext: this.runtimeContext },
|
|
2397
|
+
{
|
|
2398
|
+
toolCallId: toolId,
|
|
2399
|
+
messages: []
|
|
2400
|
+
}
|
|
2401
|
+
);
|
|
2386
2402
|
this.log("Tool executed successfully", { toolName, result });
|
|
2387
2403
|
} else {
|
|
2388
2404
|
this.log("Tool has no execute function", { toolName });
|
|
@@ -2659,14 +2675,14 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
|
|
|
2659
2675
|
* inputSchema: z.object({
|
|
2660
2676
|
* location: z.string().describe("The city and state, e.g. San Francisco, CA"),
|
|
2661
2677
|
* }),
|
|
2662
|
-
* execute: async (
|
|
2678
|
+
* execute: async ({ context }) => {
|
|
2663
2679
|
* // Fetch weather data from an API
|
|
2664
2680
|
* const response = await fetch(
|
|
2665
|
-
* `https://api.weather.com?location=${encodeURIComponent(
|
|
2681
|
+
* `https://api.weather.com?location=${encodeURIComponent(context.location)}`,
|
|
2666
2682
|
* );
|
|
2667
2683
|
* const data = await response.json();
|
|
2668
2684
|
* return {
|
|
2669
|
-
* message: `The current temperature in ${
|
|
2685
|
+
* message: `The current temperature in ${context.location} is ${data.temperature}°F with ${data.conditions}.`,
|
|
2670
2686
|
* };
|
|
2671
2687
|
* },
|
|
2672
2688
|
* });
|
|
@@ -2684,7 +2700,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
|
|
|
2684
2700
|
* Get the current tools configured for this voice instance
|
|
2685
2701
|
* @returns Object containing the current tools
|
|
2686
2702
|
*/
|
|
2687
|
-
|
|
2703
|
+
getTools() {
|
|
2688
2704
|
return this.tools;
|
|
2689
2705
|
}
|
|
2690
2706
|
log(message, ...args) {
|