@mastra/voice-google-gemini-live 0.11.0-alpha.0 → 0.11.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +59 -21
- package/README.md +3 -3
- package/dist/index.cjs +195 -211
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +7 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +195 -211
- package/dist/index.js.map +1 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +11 -13
package/CHANGELOG.md
CHANGED
|
@@ -1,42 +1,80 @@
|
|
|
1
1
|
# @mastra/voice-google-gemini-live
|
|
2
2
|
|
|
3
|
-
## 0.11.0-
|
|
3
|
+
## 0.11.0-beta.1
|
|
4
4
|
|
|
5
5
|
### Minor Changes
|
|
6
6
|
|
|
7
|
-
-
|
|
7
|
+
- Fix tool calling args and execution for Gemini Live ([#10226](https://github.com/mastra-ai/mastra/pull/10226))
|
|
8
8
|
|
|
9
9
|
### Patch Changes
|
|
10
10
|
|
|
11
|
-
- Updated dependencies [[`
|
|
12
|
-
- @mastra/core@0.
|
|
11
|
+
- Updated dependencies [[`2319326`](https://github.com/mastra-ai/mastra/commit/2319326f8c64e503a09bbcf14be2dd65405445e0), [`d629361`](https://github.com/mastra-ai/mastra/commit/d629361a60f6565b5bfb11976fdaf7308af858e2), [`08c31c1`](https://github.com/mastra-ai/mastra/commit/08c31c188ebccd598acaf55e888b6397d01f7eae), [`fd3d338`](https://github.com/mastra-ai/mastra/commit/fd3d338a2c362174ed5b383f1f011ad9fb0302aa), [`c30400a`](https://github.com/mastra-ai/mastra/commit/c30400a49b994b1b97256fe785eb6c906fc2b232), [`69e0a87`](https://github.com/mastra-ai/mastra/commit/69e0a878896a2da9494945d86e056a5f8f05b851), [`01f8878`](https://github.com/mastra-ai/mastra/commit/01f88783de25e4de048c1c8aace43e26373c6ea5), [`4c77209`](https://github.com/mastra-ai/mastra/commit/4c77209e6c11678808b365d545845918c40045c8), [`d827d08`](https://github.com/mastra-ai/mastra/commit/d827d0808ffe1f3553a84e975806cc989b9735dd), [`23c10a1`](https://github.com/mastra-ai/mastra/commit/23c10a1efdd9a693c405511ab2dc8a1236603162), [`676ccc7`](https://github.com/mastra-ai/mastra/commit/676ccc7fe92468d2d45d39c31a87825c89fd1ea0), [`c10398d`](https://github.com/mastra-ai/mastra/commit/c10398d5b88f1d4af556f4267ff06f1d11e89179), [`00c2387`](https://github.com/mastra-ai/mastra/commit/00c2387f5f04a365316f851e58666ac43f8c4edf), [`ad6250d`](https://github.com/mastra-ai/mastra/commit/ad6250dbdaad927e29f74a27b83f6c468b50a705), [`3a73998`](https://github.com/mastra-ai/mastra/commit/3a73998fa4ebeb7f3dc9301afe78095fc63e7999), [`e16d553`](https://github.com/mastra-ai/mastra/commit/e16d55338403c7553531cc568125c63d53653dff), [`4d59f58`](https://github.com/mastra-ai/mastra/commit/4d59f58de2d90d6e2810a19d4518e38ddddb9038), [`e1bb9c9`](https://github.com/mastra-ai/mastra/commit/e1bb9c94b4eb68b019ae275981be3feb769b5365), [`351a11f`](https://github.com/mastra-ai/mastra/commit/351a11fcaf2ed1008977fa9b9a489fc422e51cd4)]:
|
|
12
|
+
- @mastra/core@1.0.0-beta.3
|
|
13
13
|
|
|
14
|
-
## 0.
|
|
14
|
+
## 0.11.0-beta.0
|
|
15
15
|
|
|
16
|
-
###
|
|
17
|
-
|
|
18
|
-
- update peerdeps ([`5ca1cca`](https://github.com/mastra-ai/mastra/commit/5ca1ccac61ffa7141e6d9fa8f22d3ad4d03bf5dc))
|
|
16
|
+
### Minor Changes
|
|
19
17
|
|
|
20
|
-
-
|
|
21
|
-
- @mastra/core@0.24.0
|
|
18
|
+
- Update peer dependencies to match core package version bump (1.0.0) ([#9237](https://github.com/mastra-ai/mastra/pull/9237))
|
|
22
19
|
|
|
23
|
-
|
|
20
|
+
- Bump minimum required Node.js version to 22.13.0 ([#9706](https://github.com/mastra-ai/mastra/pull/9706))
|
|
24
21
|
|
|
25
|
-
|
|
22
|
+
- Rename RuntimeContext to RequestContext ([#9511](https://github.com/mastra-ai/mastra/pull/9511))
|
|
26
23
|
|
|
27
|
-
-
|
|
24
|
+
- Changing getAgents -> listAgents, getTools -> listTools, getWorkflows -> listWorkflows ([#9495](https://github.com/mastra-ai/mastra/pull/9495))
|
|
28
25
|
|
|
29
|
-
-
|
|
30
|
-
- @mastra/core@0.24.0-alpha.0
|
|
31
|
-
|
|
32
|
-
## 0.10.17
|
|
26
|
+
- Removed old tracing code based on OpenTelemetry ([#9237](https://github.com/mastra-ai/mastra/pull/9237))
|
|
33
27
|
|
|
34
28
|
### Patch Changes
|
|
35
29
|
|
|
36
|
-
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
30
|
+
- Update tool execution signature ([#9587](https://github.com/mastra-ai/mastra/pull/9587))
|
|
31
|
+
|
|
32
|
+
Consolidated the 3 different execution contexts to one
|
|
33
|
+
|
|
34
|
+
```typescript
|
|
35
|
+
// before depending on the context the tool was executed in
|
|
36
|
+
tool.execute({ context: data });
|
|
37
|
+
tool.execute({ context: { inputData: data } });
|
|
38
|
+
tool.execute(data);
|
|
39
|
+
|
|
40
|
+
// now, for all contexts
|
|
41
|
+
tool.execute(data, context);
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
**Before:**
|
|
45
|
+
|
|
46
|
+
```typescript
|
|
47
|
+
inputSchema: z.object({ something: z.string() }),
|
|
48
|
+
execute: async ({ context, tracingContext, runId, ... }) => {
|
|
49
|
+
return doSomething(context.string);
|
|
50
|
+
}
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
**After:**
|
|
54
|
+
|
|
55
|
+
```typescript
|
|
56
|
+
inputSchema: z.object({ something: z.string() }),
|
|
57
|
+
execute: async (inputData, context) => {
|
|
58
|
+
const { agent, mcp, workflow, ...sharedContext } = context
|
|
59
|
+
|
|
60
|
+
// context that only an agent would get like toolCallId, messages, suspend, resume, etc
|
|
61
|
+
if (agent) {
|
|
62
|
+
doSomething(inputData.something, agent)
|
|
63
|
+
// context that only a workflow would get like runId, state, suspend, resume, etc
|
|
64
|
+
} else if (workflow) {
|
|
65
|
+
doSomething(inputData.something, workflow)
|
|
66
|
+
// context that only a workflow would get like "extra", "elicitation"
|
|
67
|
+
} else if (mcp) {
|
|
68
|
+
doSomething(inputData.something, mcp)
|
|
69
|
+
} else {
|
|
70
|
+
// Running a tool in no execution context
|
|
71
|
+
return doSomething(inputData.something);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
- Updated dependencies [[`39c9743`](https://github.com/mastra-ai/mastra/commit/39c97432d084294f8ba85fbf3ef28098ff21459e), [`f743dbb`](https://github.com/mastra-ai/mastra/commit/f743dbb8b40d1627b5c10c0e6fc154f4ebb6e394), [`fec5129`](https://github.com/mastra-ai/mastra/commit/fec5129de7fc64423ea03661a56cef31dc747a0d), [`0491e7c`](https://github.com/mastra-ai/mastra/commit/0491e7c9b714cb0ba22187ee062147ec2dd7c712), [`f6f4903`](https://github.com/mastra-ai/mastra/commit/f6f4903397314f73362061dc5a3e8e7c61ea34aa), [`0e8ed46`](https://github.com/mastra-ai/mastra/commit/0e8ed467c54d6901a6a365f270ec15d6faadb36c), [`6c049d9`](https://github.com/mastra-ai/mastra/commit/6c049d94063fdcbd5b81c4912a2bf82a92c9cc0b), [`2f897df`](https://github.com/mastra-ai/mastra/commit/2f897df208508f46f51b7625e5dd20c37f93e0e3), [`3443770`](https://github.com/mastra-ai/mastra/commit/3443770662df8eb24c9df3589b2792d78cfcb811), [`f0a07e0`](https://github.com/mastra-ai/mastra/commit/f0a07e0111b3307c5fabfa4094c5c2cfb734fbe6), [`aaa40e7`](https://github.com/mastra-ai/mastra/commit/aaa40e788628b319baa8e889407d11ad626547fa), [`1521d71`](https://github.com/mastra-ai/mastra/commit/1521d716e5daedc74690c983fbd961123c56756b), [`9e1911d`](https://github.com/mastra-ai/mastra/commit/9e1911db2b4db85e0e768c3f15e0d61e319869f6), [`ebac155`](https://github.com/mastra-ai/mastra/commit/ebac15564a590117db7078233f927a7e28a85106), [`dd1c38d`](https://github.com/mastra-ai/mastra/commit/dd1c38d1b75f1b695c27b40d8d9d6ed00d5e0f6f), [`5948e6a`](https://github.com/mastra-ai/mastra/commit/5948e6a5146c83666ba3f294b2be576c82a513fb), [`8940859`](https://github.com/mastra-ai/mastra/commit/89408593658199b4ad67f7b65e888f344e64a442), [`e629310`](https://github.com/mastra-ai/mastra/commit/e629310f1a73fa236d49ec7a1d1cceb6229dc7cc), [`4c6b492`](https://github.com/mastra-ai/mastra/commit/4c6b492c4dd591c6a592520c1f6855d6e936d71f), [`dff01d8`](https://github.com/mastra-ai/mastra/commit/dff01d81ce1f4e4087cfac20fa868e6db138dd14), [`9d819d5`](https://github.com/mastra-ai/mastra/commit/9d819d54b61481639f4008e4694791bddf187edd), [`71c8d6c`](https://github.com/mastra-ai/mastra/commit/71c8d6c161253207b2b9588bdadb7eed604f7253), [`6179a9b`](https://github.com/mastra-ai/mastra/commit/6179a9ba36ffac326de3cc3c43cdc8028d37c251), [`00f4921`](https://github.com/mastra-ai/mastra/commit/00f4921dd2c91a1e5446799599ef7116a8214a1a), [`ca8041c`](https://github.com/mastra-ai/mastra/commit/ca8041cce0379fda22ed293a565bcb5b6ddca68a), [`7051bf3`](https://github.com/mastra-ai/mastra/commit/7051bf38b3b122a069008f861f7bfc004a6d9f6e), [`a8f1494`](https://github.com/mastra-ai/mastra/commit/a8f1494f4bbdc2770bcf327d4c7d869e332183f1), [`0793497`](https://github.com/mastra-ai/mastra/commit/079349753620c40246ffd673e3f9d7d9820beff3), [`5df9cce`](https://github.com/mastra-ai/mastra/commit/5df9cce1a753438413f64c11eeef8f845745c2a8), [`a854ede`](https://github.com/mastra-ai/mastra/commit/a854ede62bf5ac0945a624ac48913dd69c73aabf), [`c576fc0`](https://github.com/mastra-ai/mastra/commit/c576fc0b100b2085afded91a37c97a0ea0ec09c7), [`3defc80`](https://github.com/mastra-ai/mastra/commit/3defc80cf2b88a1b7fc1cc4ddcb91e982a614609), [`16153fe`](https://github.com/mastra-ai/mastra/commit/16153fe7eb13c99401f48e6ca32707c965ee28b9), [`9f4a683`](https://github.com/mastra-ai/mastra/commit/9f4a6833e88b52574665c028fd5508ad5c2f6004), [`bc94344`](https://github.com/mastra-ai/mastra/commit/bc943444a1342d8a662151b7bce1df7dae32f59c), [`57d157f`](https://github.com/mastra-ai/mastra/commit/57d157f0b163a95c3e6c9eae31bdb11d1bfc64f9), [`903f67d`](https://github.com/mastra-ai/mastra/commit/903f67d184504a273893818c02b961f5423a79ad), [`2a90c55`](https://github.com/mastra-ai/mastra/commit/2a90c55a86a9210697d5adaab5ee94584b079adc), [`eb09742`](https://github.com/mastra-ai/mastra/commit/eb09742197f66c4c38154c3beec78313e69760b2), [`96d35f6`](https://github.com/mastra-ai/mastra/commit/96d35f61376bc2b1bf148648a2c1985bd51bef55), [`5cbe88a`](https://github.com/mastra-ai/mastra/commit/5cbe88aefbd9f933bca669fd371ea36bf939ac6d), [`a1bd7b8`](https://github.com/mastra-ai/mastra/commit/a1bd7b8571db16b94eb01588f451a74758c96d65), [`d78b38d`](https://github.com/mastra-ai/mastra/commit/d78b38d898fce285260d3bbb4befade54331617f), [`0633100`](https://github.com/mastra-ai/mastra/commit/0633100a911ad22f5256471bdf753da21c104742), [`c710c16`](https://github.com/mastra-ai/mastra/commit/c710c1652dccfdc4111c8412bca7a6bb1d48b441), [`354ad0b`](https://github.com/mastra-ai/mastra/commit/354ad0b7b1b8183ac567f236a884fc7ede6d7138), [`cfae733`](https://github.com/mastra-ai/mastra/commit/cfae73394f4920635e6c919c8e95ff9a0788e2e5), [`e3dfda7`](https://github.com/mastra-ai/mastra/commit/e3dfda7b11bf3b8c4bb55637028befb5f387fc74), [`844ea5d`](https://github.com/mastra-ai/mastra/commit/844ea5dc0c248961e7bf73629ae7dcff503e853c), [`398fde3`](https://github.com/mastra-ai/mastra/commit/398fde3f39e707cda79372cdae8f9870e3b57c8d), [`f0f8f12`](https://github.com/mastra-ai/mastra/commit/f0f8f125c308f2d0fd36942ef652fd852df7522f), [`0d7618b`](https://github.com/mastra-ai/mastra/commit/0d7618bc650bf2800934b243eca5648f4aeed9c2), [`7b763e5`](https://github.com/mastra-ai/mastra/commit/7b763e52fc3eaf699c2a99f2adf418dd46e4e9a5), [`d36cfbb`](https://github.com/mastra-ai/mastra/commit/d36cfbbb6565ba5f827883cc9bb648eb14befdc1), [`3697853`](https://github.com/mastra-ai/mastra/commit/3697853deeb72017d90e0f38a93c1e29221aeca0), [`b2e45ec`](https://github.com/mastra-ai/mastra/commit/b2e45eca727a8db01a81ba93f1a5219c7183c839), [`d6d49f7`](https://github.com/mastra-ai/mastra/commit/d6d49f7b8714fa19a52ff9c7cf7fb7e73751901e), [`a534e95`](https://github.com/mastra-ai/mastra/commit/a534e9591f83b3cc1ebff99c67edf4cda7bf81d3), [`9d0e7fe`](https://github.com/mastra-ai/mastra/commit/9d0e7feca8ed98de959f53476ee1456073673348), [`53d927c`](https://github.com/mastra-ai/mastra/commit/53d927cc6f03bff33655b7e2b788da445a08731d), [`3f2faf2`](https://github.com/mastra-ai/mastra/commit/3f2faf2e2d685d6c053cc5af1bf9fedf267b2ce5), [`22f64bc`](https://github.com/mastra-ai/mastra/commit/22f64bc1d37149480b58bf2fefe35b79a1e3e7d5), [`83d5942`](https://github.com/mastra-ai/mastra/commit/83d5942669ce7bba4a6ca4fd4da697a10eb5ebdc), [`b7959e6`](https://github.com/mastra-ai/mastra/commit/b7959e6e25a46b480f9ea2217c4c6c588c423791), [`bda6370`](https://github.com/mastra-ai/mastra/commit/bda637009360649aaf579919e7873e33553c273e), [`d7acd8e`](https://github.com/mastra-ai/mastra/commit/d7acd8e987b5d7eff4fd98b0906c17c06a2e83d5), [`c7f1f7d`](https://github.com/mastra-ai/mastra/commit/c7f1f7d24f61f247f018cc2d1f33bf63212959a7), [`0bddc6d`](https://github.com/mastra-ai/mastra/commit/0bddc6d8dbd6f6008c0cba2e4960a2da75a55af1), [`735d8c1`](https://github.com/mastra-ai/mastra/commit/735d8c1c0d19fbc09e6f8b66cf41bc7655993838), [`acf322e`](https://github.com/mastra-ai/mastra/commit/acf322e0f1fd0189684cf529d91c694bea918a45), [`c942802`](https://github.com/mastra-ai/mastra/commit/c942802a477a925b01859a7b8688d4355715caaa), [`a0c8c1b`](https://github.com/mastra-ai/mastra/commit/a0c8c1b87d4fee252aebda73e8637fbe01d761c9), [`cc34739`](https://github.com/mastra-ai/mastra/commit/cc34739c34b6266a91bea561119240a7acf47887), [`c218bd3`](https://github.com/mastra-ai/mastra/commit/c218bd3759e32423735b04843a09404572631014), [`2c4438b`](https://github.com/mastra-ai/mastra/commit/2c4438b87817ab7eed818c7990fef010475af1a3), [`2b8893c`](https://github.com/mastra-ai/mastra/commit/2b8893cb108ef9acb72ee7835cd625610d2c1a4a), [`8e5c75b`](https://github.com/mastra-ai/mastra/commit/8e5c75bdb1d08a42d45309a4c72def4b6890230f), [`e59e0d3`](https://github.com/mastra-ai/mastra/commit/e59e0d32afb5fcf2c9f3c00c8f81f6c21d3a63fa), [`fa8409b`](https://github.com/mastra-ai/mastra/commit/fa8409bc39cfd8ba6643b9db5269b90b22e2a2f7), [`173c535`](https://github.com/mastra-ai/mastra/commit/173c535c0645b0da404fe09f003778f0b0d4e019)]:
|
|
77
|
+
- @mastra/core@1.0.0-beta.0
|
|
40
78
|
|
|
41
79
|
## 0.10.16
|
|
42
80
|
|
package/README.md
CHANGED
|
@@ -337,8 +337,8 @@ const searchTool = createTool({
|
|
|
337
337
|
id: 'search',
|
|
338
338
|
description: 'Search the web',
|
|
339
339
|
inputSchema: z.object({ query: z.string() }),
|
|
340
|
-
execute: async
|
|
341
|
-
const { query } =
|
|
340
|
+
execute: async inputData => {
|
|
341
|
+
const { query } = inputData;
|
|
342
342
|
// ... perform search
|
|
343
343
|
return { results: [] };
|
|
344
344
|
},
|
|
@@ -355,7 +355,7 @@ voice.addTools({
|
|
|
355
355
|
id: 'search',
|
|
356
356
|
description: 'Search the web',
|
|
357
357
|
inputSchema: { type: 'object', properties: { query: { type: 'string' } } },
|
|
358
|
-
execute: async (
|
|
358
|
+
execute: async (inputData, context) => ({ results: [] }),
|
|
359
359
|
},
|
|
360
360
|
});
|
|
361
361
|
```
|
package/dist/index.cjs
CHANGED
|
@@ -1262,7 +1262,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
|
|
|
1262
1262
|
sessionDurationTimeout;
|
|
1263
1263
|
// Tool integration properties
|
|
1264
1264
|
tools;
|
|
1265
|
-
|
|
1265
|
+
requestContext;
|
|
1266
1266
|
// Store the configuration options
|
|
1267
1267
|
options;
|
|
1268
1268
|
/**
|
|
@@ -1499,69 +1499,67 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
|
|
|
1499
1499
|
/**
|
|
1500
1500
|
* Establish connection to the Gemini Live API
|
|
1501
1501
|
*/
|
|
1502
|
-
async connect({
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
headers
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
"Content-Type": "application/json"
|
|
1525
|
-
}
|
|
1526
|
-
};
|
|
1527
|
-
this.log("Using Live API authentication with API key");
|
|
1528
|
-
}
|
|
1529
|
-
this.log("Connecting to:", wsUrl);
|
|
1530
|
-
this.ws = new ws.WebSocket(wsUrl, void 0, headers);
|
|
1531
|
-
this.connectionManager.setWebSocket(this.ws);
|
|
1532
|
-
this.setupEventListeners();
|
|
1533
|
-
await this.connectionManager.waitForOpen();
|
|
1534
|
-
if (this.isResuming && this.sessionHandle) {
|
|
1535
|
-
await this.sendSessionResumption();
|
|
1536
|
-
} else {
|
|
1537
|
-
this.sendInitialConfig();
|
|
1538
|
-
this.sessionStartTime = Date.now();
|
|
1539
|
-
this.sessionId = crypto.randomUUID();
|
|
1540
|
-
}
|
|
1541
|
-
await this.waitForSessionCreated();
|
|
1542
|
-
this.state = "connected";
|
|
1543
|
-
this.emit("session", {
|
|
1544
|
-
state: "connected",
|
|
1545
|
-
config: {
|
|
1546
|
-
sessionId: this.sessionId,
|
|
1547
|
-
isResuming: this.isResuming,
|
|
1548
|
-
toolCount: Object.keys(this.tools || {}).length
|
|
1502
|
+
async connect({ requestContext } = {}) {
|
|
1503
|
+
if (this.state === "connected") {
|
|
1504
|
+
this.log("Already connected to Gemini Live API");
|
|
1505
|
+
return;
|
|
1506
|
+
}
|
|
1507
|
+
this.requestContext = requestContext;
|
|
1508
|
+
this.emit("session", { state: "connecting" });
|
|
1509
|
+
try {
|
|
1510
|
+
let wsUrl;
|
|
1511
|
+
let headers = {};
|
|
1512
|
+
if (this.options.vertexAI) {
|
|
1513
|
+
wsUrl = `wss://${this.options.location}-aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1beta1.PredictionService.ServerStreamingPredict`;
|
|
1514
|
+
await this.authManager.initialize();
|
|
1515
|
+
const accessToken = await this.authManager.getAccessToken();
|
|
1516
|
+
headers = { headers: { Authorization: `Bearer ${accessToken}` } };
|
|
1517
|
+
this.log("Using Vertex AI authentication with OAuth token");
|
|
1518
|
+
} else {
|
|
1519
|
+
wsUrl = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent`;
|
|
1520
|
+
headers = {
|
|
1521
|
+
headers: {
|
|
1522
|
+
"x-goog-api-key": this.options.apiKey || "",
|
|
1523
|
+
"Content-Type": "application/json"
|
|
1549
1524
|
}
|
|
1550
|
-
}
|
|
1551
|
-
this.log("
|
|
1525
|
+
};
|
|
1526
|
+
this.log("Using Live API authentication with API key");
|
|
1527
|
+
}
|
|
1528
|
+
this.log("Connecting to:", wsUrl);
|
|
1529
|
+
this.ws = new ws.WebSocket(wsUrl, void 0, headers);
|
|
1530
|
+
this.connectionManager.setWebSocket(this.ws);
|
|
1531
|
+
this.setupEventListeners();
|
|
1532
|
+
await this.connectionManager.waitForOpen();
|
|
1533
|
+
if (this.isResuming && this.sessionHandle) {
|
|
1534
|
+
await this.sendSessionResumption();
|
|
1535
|
+
} else {
|
|
1536
|
+
this.sendInitialConfig();
|
|
1537
|
+
this.sessionStartTime = Date.now();
|
|
1538
|
+
this.sessionId = crypto.randomUUID();
|
|
1539
|
+
}
|
|
1540
|
+
await this.waitForSessionCreated();
|
|
1541
|
+
this.state = "connected";
|
|
1542
|
+
this.emit("session", {
|
|
1543
|
+
state: "connected",
|
|
1544
|
+
config: {
|
|
1552
1545
|
sessionId: this.sessionId,
|
|
1553
1546
|
isResuming: this.isResuming,
|
|
1554
1547
|
toolCount: Object.keys(this.tools || {}).length
|
|
1555
|
-
});
|
|
1556
|
-
if (this.options.sessionConfig?.maxDuration) {
|
|
1557
|
-
this.startSessionDurationMonitor();
|
|
1558
1548
|
}
|
|
1559
|
-
}
|
|
1560
|
-
|
|
1561
|
-
this.
|
|
1562
|
-
|
|
1549
|
+
});
|
|
1550
|
+
this.log("Successfully connected to Gemini Live API", {
|
|
1551
|
+
sessionId: this.sessionId,
|
|
1552
|
+
isResuming: this.isResuming,
|
|
1553
|
+
toolCount: Object.keys(this.tools || {}).length
|
|
1554
|
+
});
|
|
1555
|
+
if (this.options.sessionConfig?.maxDuration) {
|
|
1556
|
+
this.startSessionDurationMonitor();
|
|
1563
1557
|
}
|
|
1564
|
-
}
|
|
1558
|
+
} catch (error) {
|
|
1559
|
+
this.state = "disconnected";
|
|
1560
|
+
this.log("Connection failed", error);
|
|
1561
|
+
throw error;
|
|
1562
|
+
}
|
|
1565
1563
|
}
|
|
1566
1564
|
/**
|
|
1567
1565
|
* Disconnect from the Gemini Live API
|
|
@@ -1599,172 +1597,164 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
|
|
|
1599
1597
|
* Send text to be converted to speech
|
|
1600
1598
|
*/
|
|
1601
1599
|
async speak(input, options) {
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)));
|
|
1608
|
-
}
|
|
1609
|
-
input = Buffer.concat(chunks).toString("utf-8");
|
|
1610
|
-
}
|
|
1611
|
-
if (input.trim().length === 0) {
|
|
1612
|
-
throw this.createAndEmitError("invalid_audio_format" /* INVALID_AUDIO_FORMAT */, "Input text is empty");
|
|
1600
|
+
this.validateConnectionState();
|
|
1601
|
+
if (typeof input !== "string") {
|
|
1602
|
+
const chunks = [];
|
|
1603
|
+
for await (const chunk of input) {
|
|
1604
|
+
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)));
|
|
1613
1605
|
}
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
}
|
|
1629
|
-
};
|
|
1630
|
-
if (options && (options.speaker || options.languageCode || options.responseModalities)) {
|
|
1631
|
-
const updateMessage = {
|
|
1632
|
-
type: "session.update",
|
|
1633
|
-
session: {
|
|
1634
|
-
generation_config: {
|
|
1635
|
-
...options.responseModalities ? { response_modalities: options.responseModalities } : {},
|
|
1636
|
-
speech_config: {
|
|
1637
|
-
...options.languageCode ? { language_code: options.languageCode } : {},
|
|
1638
|
-
...options.speaker ? { voice_config: { prebuilt_voice_config: { voice_name: options.speaker } } } : {}
|
|
1606
|
+
input = Buffer.concat(chunks).toString("utf-8");
|
|
1607
|
+
}
|
|
1608
|
+
if (input.trim().length === 0) {
|
|
1609
|
+
throw this.createAndEmitError("invalid_audio_format" /* INVALID_AUDIO_FORMAT */, "Input text is empty");
|
|
1610
|
+
}
|
|
1611
|
+
this.addToContext("user", input);
|
|
1612
|
+
const textMessage = {
|
|
1613
|
+
client_content: {
|
|
1614
|
+
turns: [
|
|
1615
|
+
{
|
|
1616
|
+
role: "user",
|
|
1617
|
+
parts: [
|
|
1618
|
+
{
|
|
1619
|
+
text: input
|
|
1639
1620
|
}
|
|
1621
|
+
]
|
|
1622
|
+
}
|
|
1623
|
+
],
|
|
1624
|
+
turnComplete: true
|
|
1625
|
+
}
|
|
1626
|
+
};
|
|
1627
|
+
if (options && (options.speaker || options.languageCode || options.responseModalities)) {
|
|
1628
|
+
const updateMessage = {
|
|
1629
|
+
type: "session.update",
|
|
1630
|
+
session: {
|
|
1631
|
+
generation_config: {
|
|
1632
|
+
...options.responseModalities ? { response_modalities: options.responseModalities } : {},
|
|
1633
|
+
speech_config: {
|
|
1634
|
+
...options.languageCode ? { language_code: options.languageCode } : {},
|
|
1635
|
+
...options.speaker ? { voice_config: { prebuilt_voice_config: { voice_name: options.speaker } } } : {}
|
|
1640
1636
|
}
|
|
1641
1637
|
}
|
|
1642
|
-
};
|
|
1643
|
-
try {
|
|
1644
|
-
this.sendEvent("session.update", updateMessage);
|
|
1645
|
-
this.log("Applied per-turn runtime options", options);
|
|
1646
|
-
} catch (error) {
|
|
1647
|
-
this.log("Failed to apply per-turn runtime options", error);
|
|
1648
1638
|
}
|
|
1649
|
-
}
|
|
1639
|
+
};
|
|
1650
1640
|
try {
|
|
1651
|
-
this.sendEvent("
|
|
1652
|
-
this.log("
|
|
1641
|
+
this.sendEvent("session.update", updateMessage);
|
|
1642
|
+
this.log("Applied per-turn runtime options", options);
|
|
1653
1643
|
} catch (error) {
|
|
1654
|
-
this.log("Failed to
|
|
1655
|
-
throw this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to send text message", error);
|
|
1644
|
+
this.log("Failed to apply per-turn runtime options", error);
|
|
1656
1645
|
}
|
|
1657
|
-
}
|
|
1646
|
+
}
|
|
1647
|
+
try {
|
|
1648
|
+
this.sendEvent("client_content", textMessage);
|
|
1649
|
+
this.log("Text message sent", { text: input });
|
|
1650
|
+
} catch (error) {
|
|
1651
|
+
this.log("Failed to send text message", error);
|
|
1652
|
+
throw this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to send text message", error);
|
|
1653
|
+
}
|
|
1658
1654
|
}
|
|
1659
1655
|
/**
|
|
1660
1656
|
* Send audio stream for processing
|
|
1661
1657
|
*/
|
|
1662
1658
|
async send(audioData) {
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
}
|
|
1690
|
-
}, "gemini-live.send")();
|
|
1659
|
+
this.validateConnectionState();
|
|
1660
|
+
if ("readable" in audioData && typeof audioData.on === "function") {
|
|
1661
|
+
const stream = audioData;
|
|
1662
|
+
stream.on("data", (chunk) => {
|
|
1663
|
+
try {
|
|
1664
|
+
const base64Audio = this.audioStreamManager.processAudioChunk(chunk);
|
|
1665
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "realtime");
|
|
1666
|
+
this.sendEvent("realtime_input", message);
|
|
1667
|
+
} catch (error) {
|
|
1668
|
+
this.log("Failed to process audio chunk", error);
|
|
1669
|
+
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to process audio chunk", error);
|
|
1670
|
+
}
|
|
1671
|
+
});
|
|
1672
|
+
stream.on("error", (error) => {
|
|
1673
|
+
this.log("Audio stream error", error);
|
|
1674
|
+
this.createAndEmitError("audio_stream_error" /* AUDIO_STREAM_ERROR */, "Audio stream error", error);
|
|
1675
|
+
});
|
|
1676
|
+
stream.on("end", () => {
|
|
1677
|
+
this.log("Audio stream ended");
|
|
1678
|
+
});
|
|
1679
|
+
} else {
|
|
1680
|
+
const validateAudio = this.audioStreamManager.validateAndConvertAudioInput(audioData);
|
|
1681
|
+
const base64Audio = this.audioStreamManager.int16ArrayToBase64(validateAudio);
|
|
1682
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "realtime");
|
|
1683
|
+
this.sendEvent("realtime_input", message);
|
|
1684
|
+
}
|
|
1691
1685
|
}
|
|
1692
1686
|
/**
|
|
1693
1687
|
* Process speech from audio stream (traditional STT interface)
|
|
1694
1688
|
*/
|
|
1695
1689
|
async listen(audioStream, _options) {
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
this.log("Received transcription text:", { text: data.text, total: transcriptionText });
|
|
1703
|
-
}
|
|
1704
|
-
};
|
|
1705
|
-
const onError = (error) => {
|
|
1706
|
-
throw new Error(`Transcription failed: ${error.message}`);
|
|
1707
|
-
};
|
|
1708
|
-
const onSession = (data) => {
|
|
1709
|
-
if (data.state === "disconnected") {
|
|
1710
|
-
throw new Error("Session disconnected during transcription");
|
|
1711
|
-
}
|
|
1712
|
-
};
|
|
1713
|
-
this.on("writing", onWriting);
|
|
1714
|
-
this.on("error", onError);
|
|
1715
|
-
this.on("session", onSession);
|
|
1716
|
-
try {
|
|
1717
|
-
const result = await this.audioStreamManager.handleAudioTranscription(
|
|
1718
|
-
audioStream,
|
|
1719
|
-
(base64Audio) => {
|
|
1720
|
-
return new Promise((resolve, reject) => {
|
|
1721
|
-
try {
|
|
1722
|
-
const message = this.audioStreamManager.createAudioMessage(base64Audio, "input");
|
|
1723
|
-
const cleanup = () => {
|
|
1724
|
-
this.off("turnComplete", onTurnComplete);
|
|
1725
|
-
this.off("error", onErr);
|
|
1726
|
-
};
|
|
1727
|
-
const onTurnComplete = () => {
|
|
1728
|
-
cleanup();
|
|
1729
|
-
resolve(transcriptionText.trim());
|
|
1730
|
-
};
|
|
1731
|
-
const onErr = (e) => {
|
|
1732
|
-
cleanup();
|
|
1733
|
-
reject(new Error(e.message));
|
|
1734
|
-
};
|
|
1735
|
-
this.on("turnComplete", onTurnComplete);
|
|
1736
|
-
this.on("error", onErr);
|
|
1737
|
-
this.sendEvent("client_content", message);
|
|
1738
|
-
this.log("Sent audio for transcription");
|
|
1739
|
-
} catch (err) {
|
|
1740
|
-
reject(err);
|
|
1741
|
-
}
|
|
1742
|
-
});
|
|
1743
|
-
},
|
|
1744
|
-
(error) => {
|
|
1745
|
-
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Audio transcription failed", error);
|
|
1746
|
-
}
|
|
1747
|
-
);
|
|
1748
|
-
return result;
|
|
1749
|
-
} finally {
|
|
1750
|
-
this.off("writing", onWriting);
|
|
1751
|
-
this.off("error", onError);
|
|
1752
|
-
this.off("session", onSession);
|
|
1690
|
+
this.validateConnectionState();
|
|
1691
|
+
let transcriptionText = "";
|
|
1692
|
+
const onWriting = (data) => {
|
|
1693
|
+
if (data.role === "user") {
|
|
1694
|
+
transcriptionText += data.text;
|
|
1695
|
+
this.log("Received transcription text:", { text: data.text, total: transcriptionText });
|
|
1753
1696
|
}
|
|
1754
|
-
}
|
|
1697
|
+
};
|
|
1698
|
+
const onError = (error) => {
|
|
1699
|
+
throw new Error(`Transcription failed: ${error.message}`);
|
|
1700
|
+
};
|
|
1701
|
+
const onSession = (data) => {
|
|
1702
|
+
if (data.state === "disconnected") {
|
|
1703
|
+
throw new Error("Session disconnected during transcription");
|
|
1704
|
+
}
|
|
1705
|
+
};
|
|
1706
|
+
this.on("writing", onWriting);
|
|
1707
|
+
this.on("error", onError);
|
|
1708
|
+
this.on("session", onSession);
|
|
1709
|
+
try {
|
|
1710
|
+
const result = await this.audioStreamManager.handleAudioTranscription(
|
|
1711
|
+
audioStream,
|
|
1712
|
+
(base64Audio) => {
|
|
1713
|
+
return new Promise((resolve, reject) => {
|
|
1714
|
+
try {
|
|
1715
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "input");
|
|
1716
|
+
const cleanup = () => {
|
|
1717
|
+
this.off("turnComplete", onTurnComplete);
|
|
1718
|
+
this.off("error", onErr);
|
|
1719
|
+
};
|
|
1720
|
+
const onTurnComplete = () => {
|
|
1721
|
+
cleanup();
|
|
1722
|
+
resolve(transcriptionText.trim());
|
|
1723
|
+
};
|
|
1724
|
+
const onErr = (e) => {
|
|
1725
|
+
cleanup();
|
|
1726
|
+
reject(new Error(e.message));
|
|
1727
|
+
};
|
|
1728
|
+
this.on("turnComplete", onTurnComplete);
|
|
1729
|
+
this.on("error", onErr);
|
|
1730
|
+
this.sendEvent("client_content", message);
|
|
1731
|
+
this.log("Sent audio for transcription");
|
|
1732
|
+
} catch (err) {
|
|
1733
|
+
reject(err);
|
|
1734
|
+
}
|
|
1735
|
+
});
|
|
1736
|
+
},
|
|
1737
|
+
(error) => {
|
|
1738
|
+
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Audio transcription failed", error);
|
|
1739
|
+
}
|
|
1740
|
+
);
|
|
1741
|
+
return result;
|
|
1742
|
+
} finally {
|
|
1743
|
+
this.off("writing", onWriting);
|
|
1744
|
+
this.off("error", onError);
|
|
1745
|
+
this.off("session", onSession);
|
|
1746
|
+
}
|
|
1755
1747
|
}
|
|
1756
1748
|
/**
|
|
1757
1749
|
* Get available speakers/voices
|
|
1758
1750
|
*/
|
|
1759
1751
|
async getSpeakers() {
|
|
1760
|
-
return
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
];
|
|
1767
|
-
}, "gemini-live.getSpeakers")();
|
|
1752
|
+
return [
|
|
1753
|
+
{ voiceId: "Puck", description: "Conversational, friendly" },
|
|
1754
|
+
{ voiceId: "Charon", description: "Deep, authoritative" },
|
|
1755
|
+
{ voiceId: "Kore", description: "Neutral, professional" },
|
|
1756
|
+
{ voiceId: "Fenrir", description: "Warm, approachable" }
|
|
1757
|
+
];
|
|
1768
1758
|
}
|
|
1769
1759
|
/**
|
|
1770
1760
|
* Resume a previous session using a session handle
|
|
@@ -2392,13 +2382,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
|
|
|
2392
2382
|
let result;
|
|
2393
2383
|
if (tool.execute) {
|
|
2394
2384
|
this.log("Executing tool", { toolName, toolArgs });
|
|
2395
|
-
result = await tool.execute(
|
|
2396
|
-
{ context: toolArgs, runtimeContext: this.runtimeContext },
|
|
2397
|
-
{
|
|
2398
|
-
toolCallId: toolId,
|
|
2399
|
-
messages: []
|
|
2400
|
-
}
|
|
2401
|
-
);
|
|
2385
|
+
result = await tool.execute(toolArgs, { requestContext: this.requestContext });
|
|
2402
2386
|
this.log("Tool executed successfully", { toolName, result });
|
|
2403
2387
|
} else {
|
|
2404
2388
|
this.log("Tool has no execute function", { toolName });
|
|
@@ -2675,14 +2659,14 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
|
|
|
2675
2659
|
* inputSchema: z.object({
|
|
2676
2660
|
* location: z.string().describe("The city and state, e.g. San Francisco, CA"),
|
|
2677
2661
|
* }),
|
|
2678
|
-
* execute: async (
|
|
2662
|
+
* execute: async (inputData) => {
|
|
2679
2663
|
* // Fetch weather data from an API
|
|
2680
2664
|
* const response = await fetch(
|
|
2681
|
-
* `https://api.weather.com?location=${encodeURIComponent(
|
|
2665
|
+
* `https://api.weather.com?location=${encodeURIComponent(inputData.location)}`,
|
|
2682
2666
|
* );
|
|
2683
2667
|
* const data = await response.json();
|
|
2684
2668
|
* return {
|
|
2685
|
-
* message: `The current temperature in ${
|
|
2669
|
+
* message: `The current temperature in ${inputData.location} is ${data.temperature}°F with ${data.conditions}.`,
|
|
2686
2670
|
* };
|
|
2687
2671
|
* },
|
|
2688
2672
|
* });
|
|
@@ -2700,7 +2684,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
|
|
|
2700
2684
|
* Get the current tools configured for this voice instance
|
|
2701
2685
|
* @returns Object containing the current tools
|
|
2702
2686
|
*/
|
|
2703
|
-
|
|
2687
|
+
listTools() {
|
|
2704
2688
|
return this.tools;
|
|
2705
2689
|
}
|
|
2706
2690
|
log(message, ...args) {
|