@mastra/voice-google-gemini-live 0.10.18-alpha.0 → 0.11.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +57 -10
- package/README.md +3 -3
- package/dist/index.cjs +195 -211
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +7 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +195 -211
- package/dist/index.js.map +1 -1
- package/package.json +7 -11
package/CHANGELOG.md
CHANGED
|
@@ -1,22 +1,69 @@
|
|
|
1
1
|
# @mastra/voice-google-gemini-live
|
|
2
2
|
|
|
3
|
-
## 0.
|
|
3
|
+
## 0.11.0-beta.0
|
|
4
4
|
|
|
5
|
-
###
|
|
5
|
+
### Minor Changes
|
|
6
6
|
|
|
7
|
-
-
|
|
7
|
+
- Update peer dependencies to match core package version bump (1.0.0) ([#9237](https://github.com/mastra-ai/mastra/pull/9237))
|
|
8
8
|
|
|
9
|
-
-
|
|
10
|
-
- @mastra/core@0.24.0-alpha.0
|
|
9
|
+
- Bump minimum required Node.js version to 22.13.0 ([#9706](https://github.com/mastra-ai/mastra/pull/9706))
|
|
11
10
|
|
|
12
|
-
|
|
11
|
+
- Rename RuntimeContext to RequestContext ([#9511](https://github.com/mastra-ai/mastra/pull/9511))
|
|
13
12
|
|
|
14
|
-
|
|
13
|
+
- Changing getAgents -> listAgents, getTools -> listTools, getWorkflows -> listWorkflows ([#9495](https://github.com/mastra-ai/mastra/pull/9495))
|
|
15
14
|
|
|
16
|
-
-
|
|
15
|
+
- Removed old tracing code based on OpenTelemetry ([#9237](https://github.com/mastra-ai/mastra/pull/9237))
|
|
17
16
|
|
|
18
|
-
|
|
19
|
-
|
|
17
|
+
### Patch Changes
|
|
18
|
+
|
|
19
|
+
- Update tool execution signature ([#9587](https://github.com/mastra-ai/mastra/pull/9587))
|
|
20
|
+
|
|
21
|
+
Consolidated the 3 different execution contexts to one
|
|
22
|
+
|
|
23
|
+
```typescript
|
|
24
|
+
// before depending on the context the tool was executed in
|
|
25
|
+
tool.execute({ context: data });
|
|
26
|
+
tool.execute({ context: { inputData: data } });
|
|
27
|
+
tool.execute(data);
|
|
28
|
+
|
|
29
|
+
// now, for all contexts
|
|
30
|
+
tool.execute(data, context);
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
**Before:**
|
|
34
|
+
|
|
35
|
+
```typescript
|
|
36
|
+
inputSchema: z.object({ something: z.string() }),
|
|
37
|
+
execute: async ({ context, tracingContext, runId, ... }) => {
|
|
38
|
+
return doSomething(context.string);
|
|
39
|
+
}
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
**After:**
|
|
43
|
+
|
|
44
|
+
```typescript
|
|
45
|
+
inputSchema: z.object({ something: z.string() }),
|
|
46
|
+
execute: async (inputData, context) => {
|
|
47
|
+
const { agent, mcp, workflow, ...sharedContext } = context
|
|
48
|
+
|
|
49
|
+
// context that only an agent would get like toolCallId, messages, suspend, resume, etc
|
|
50
|
+
if (agent) {
|
|
51
|
+
doSomething(inputData.something, agent)
|
|
52
|
+
// context that only a workflow would get like runId, state, suspend, resume, etc
|
|
53
|
+
} else if (workflow) {
|
|
54
|
+
doSomething(inputData.something, workflow)
|
|
55
|
+
// context that only a workflow would get like "extra", "elicitation"
|
|
56
|
+
} else if (mcp) {
|
|
57
|
+
doSomething(inputData.something, mcp)
|
|
58
|
+
} else {
|
|
59
|
+
// Running a tool in no execution context
|
|
60
|
+
return doSomething(inputData.something);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
- Updated dependencies [[`39c9743`](https://github.com/mastra-ai/mastra/commit/39c97432d084294f8ba85fbf3ef28098ff21459e), [`f743dbb`](https://github.com/mastra-ai/mastra/commit/f743dbb8b40d1627b5c10c0e6fc154f4ebb6e394), [`fec5129`](https://github.com/mastra-ai/mastra/commit/fec5129de7fc64423ea03661a56cef31dc747a0d), [`0491e7c`](https://github.com/mastra-ai/mastra/commit/0491e7c9b714cb0ba22187ee062147ec2dd7c712), [`f6f4903`](https://github.com/mastra-ai/mastra/commit/f6f4903397314f73362061dc5a3e8e7c61ea34aa), [`0e8ed46`](https://github.com/mastra-ai/mastra/commit/0e8ed467c54d6901a6a365f270ec15d6faadb36c), [`6c049d9`](https://github.com/mastra-ai/mastra/commit/6c049d94063fdcbd5b81c4912a2bf82a92c9cc0b), [`2f897df`](https://github.com/mastra-ai/mastra/commit/2f897df208508f46f51b7625e5dd20c37f93e0e3), [`3443770`](https://github.com/mastra-ai/mastra/commit/3443770662df8eb24c9df3589b2792d78cfcb811), [`f0a07e0`](https://github.com/mastra-ai/mastra/commit/f0a07e0111b3307c5fabfa4094c5c2cfb734fbe6), [`aaa40e7`](https://github.com/mastra-ai/mastra/commit/aaa40e788628b319baa8e889407d11ad626547fa), [`1521d71`](https://github.com/mastra-ai/mastra/commit/1521d716e5daedc74690c983fbd961123c56756b), [`9e1911d`](https://github.com/mastra-ai/mastra/commit/9e1911db2b4db85e0e768c3f15e0d61e319869f6), [`ebac155`](https://github.com/mastra-ai/mastra/commit/ebac15564a590117db7078233f927a7e28a85106), [`dd1c38d`](https://github.com/mastra-ai/mastra/commit/dd1c38d1b75f1b695c27b40d8d9d6ed00d5e0f6f), [`5948e6a`](https://github.com/mastra-ai/mastra/commit/5948e6a5146c83666ba3f294b2be576c82a513fb), [`8940859`](https://github.com/mastra-ai/mastra/commit/89408593658199b4ad67f7b65e888f344e64a442), [`e629310`](https://github.com/mastra-ai/mastra/commit/e629310f1a73fa236d49ec7a1d1cceb6229dc7cc), [`4c6b492`](https://github.com/mastra-ai/mastra/commit/4c6b492c4dd591c6a592520c1f6855d6e936d71f), [`dff01d8`](https://github.com/mastra-ai/mastra/commit/dff01d81ce1f4e4087cfac20fa868e6db138dd14), [`9d819d5`](https://github.com/mastra-ai/mastra/commit/9d819d54b61481639f4008e4694791bddf187edd), [`71c8d6c`](https://github.com/mastra-ai/mastra/commit/71c8d6c161253207b2b9588bdadb7eed604f7253), [`6179a9b`](https://github.com/mastra-ai/mastra/commit/6179a9ba36ffac326de3cc3c43cdc8028d37c251), [`00f4921`](https://github.com/mastra-ai/mastra/commit/00f4921dd2c91a1e5446799599ef7116a8214a1a), [`ca8041c`](https://github.com/mastra-ai/mastra/commit/ca8041cce0379fda22ed293a565bcb5b6ddca68a), [`7051bf3`](https://github.com/mastra-ai/mastra/commit/7051bf38b3b122a069008f861f7bfc004a6d9f6e), [`a8f1494`](https://github.com/mastra-ai/mastra/commit/a8f1494f4bbdc2770bcf327d4c7d869e332183f1), [`0793497`](https://github.com/mastra-ai/mastra/commit/079349753620c40246ffd673e3f9d7d9820beff3), [`5df9cce`](https://github.com/mastra-ai/mastra/commit/5df9cce1a753438413f64c11eeef8f845745c2a8), [`a854ede`](https://github.com/mastra-ai/mastra/commit/a854ede62bf5ac0945a624ac48913dd69c73aabf), [`c576fc0`](https://github.com/mastra-ai/mastra/commit/c576fc0b100b2085afded91a37c97a0ea0ec09c7), [`3defc80`](https://github.com/mastra-ai/mastra/commit/3defc80cf2b88a1b7fc1cc4ddcb91e982a614609), [`16153fe`](https://github.com/mastra-ai/mastra/commit/16153fe7eb13c99401f48e6ca32707c965ee28b9), [`9f4a683`](https://github.com/mastra-ai/mastra/commit/9f4a6833e88b52574665c028fd5508ad5c2f6004), [`bc94344`](https://github.com/mastra-ai/mastra/commit/bc943444a1342d8a662151b7bce1df7dae32f59c), [`57d157f`](https://github.com/mastra-ai/mastra/commit/57d157f0b163a95c3e6c9eae31bdb11d1bfc64f9), [`903f67d`](https://github.com/mastra-ai/mastra/commit/903f67d184504a273893818c02b961f5423a79ad), [`2a90c55`](https://github.com/mastra-ai/mastra/commit/2a90c55a86a9210697d5adaab5ee94584b079adc), [`eb09742`](https://github.com/mastra-ai/mastra/commit/eb09742197f66c4c38154c3beec78313e69760b2), [`96d35f6`](https://github.com/mastra-ai/mastra/commit/96d35f61376bc2b1bf148648a2c1985bd51bef55), [`5cbe88a`](https://github.com/mastra-ai/mastra/commit/5cbe88aefbd9f933bca669fd371ea36bf939ac6d), [`a1bd7b8`](https://github.com/mastra-ai/mastra/commit/a1bd7b8571db16b94eb01588f451a74758c96d65), [`d78b38d`](https://github.com/mastra-ai/mastra/commit/d78b38d898fce285260d3bbb4befade54331617f), [`0633100`](https://github.com/mastra-ai/mastra/commit/0633100a911ad22f5256471bdf753da21c104742), [`c710c16`](https://github.com/mastra-ai/mastra/commit/c710c1652dccfdc4111c8412bca7a6bb1d48b441), [`354ad0b`](https://github.com/mastra-ai/mastra/commit/354ad0b7b1b8183ac567f236a884fc7ede6d7138), [`cfae733`](https://github.com/mastra-ai/mastra/commit/cfae73394f4920635e6c919c8e95ff9a0788e2e5), [`e3dfda7`](https://github.com/mastra-ai/mastra/commit/e3dfda7b11bf3b8c4bb55637028befb5f387fc74), [`844ea5d`](https://github.com/mastra-ai/mastra/commit/844ea5dc0c248961e7bf73629ae7dcff503e853c), [`398fde3`](https://github.com/mastra-ai/mastra/commit/398fde3f39e707cda79372cdae8f9870e3b57c8d), [`f0f8f12`](https://github.com/mastra-ai/mastra/commit/f0f8f125c308f2d0fd36942ef652fd852df7522f), [`0d7618b`](https://github.com/mastra-ai/mastra/commit/0d7618bc650bf2800934b243eca5648f4aeed9c2), [`7b763e5`](https://github.com/mastra-ai/mastra/commit/7b763e52fc3eaf699c2a99f2adf418dd46e4e9a5), [`d36cfbb`](https://github.com/mastra-ai/mastra/commit/d36cfbbb6565ba5f827883cc9bb648eb14befdc1), [`3697853`](https://github.com/mastra-ai/mastra/commit/3697853deeb72017d90e0f38a93c1e29221aeca0), [`b2e45ec`](https://github.com/mastra-ai/mastra/commit/b2e45eca727a8db01a81ba93f1a5219c7183c839), [`d6d49f7`](https://github.com/mastra-ai/mastra/commit/d6d49f7b8714fa19a52ff9c7cf7fb7e73751901e), [`a534e95`](https://github.com/mastra-ai/mastra/commit/a534e9591f83b3cc1ebff99c67edf4cda7bf81d3), [`9d0e7fe`](https://github.com/mastra-ai/mastra/commit/9d0e7feca8ed98de959f53476ee1456073673348), [`53d927c`](https://github.com/mastra-ai/mastra/commit/53d927cc6f03bff33655b7e2b788da445a08731d), [`3f2faf2`](https://github.com/mastra-ai/mastra/commit/3f2faf2e2d685d6c053cc5af1bf9fedf267b2ce5), [`22f64bc`](https://github.com/mastra-ai/mastra/commit/22f64bc1d37149480b58bf2fefe35b79a1e3e7d5), [`83d5942`](https://github.com/mastra-ai/mastra/commit/83d5942669ce7bba4a6ca4fd4da697a10eb5ebdc), [`b7959e6`](https://github.com/mastra-ai/mastra/commit/b7959e6e25a46b480f9ea2217c4c6c588c423791), [`bda6370`](https://github.com/mastra-ai/mastra/commit/bda637009360649aaf579919e7873e33553c273e), [`d7acd8e`](https://github.com/mastra-ai/mastra/commit/d7acd8e987b5d7eff4fd98b0906c17c06a2e83d5), [`c7f1f7d`](https://github.com/mastra-ai/mastra/commit/c7f1f7d24f61f247f018cc2d1f33bf63212959a7), [`0bddc6d`](https://github.com/mastra-ai/mastra/commit/0bddc6d8dbd6f6008c0cba2e4960a2da75a55af1), [`735d8c1`](https://github.com/mastra-ai/mastra/commit/735d8c1c0d19fbc09e6f8b66cf41bc7655993838), [`acf322e`](https://github.com/mastra-ai/mastra/commit/acf322e0f1fd0189684cf529d91c694bea918a45), [`c942802`](https://github.com/mastra-ai/mastra/commit/c942802a477a925b01859a7b8688d4355715caaa), [`a0c8c1b`](https://github.com/mastra-ai/mastra/commit/a0c8c1b87d4fee252aebda73e8637fbe01d761c9), [`cc34739`](https://github.com/mastra-ai/mastra/commit/cc34739c34b6266a91bea561119240a7acf47887), [`c218bd3`](https://github.com/mastra-ai/mastra/commit/c218bd3759e32423735b04843a09404572631014), [`2c4438b`](https://github.com/mastra-ai/mastra/commit/2c4438b87817ab7eed818c7990fef010475af1a3), [`2b8893c`](https://github.com/mastra-ai/mastra/commit/2b8893cb108ef9acb72ee7835cd625610d2c1a4a), [`8e5c75b`](https://github.com/mastra-ai/mastra/commit/8e5c75bdb1d08a42d45309a4c72def4b6890230f), [`e59e0d3`](https://github.com/mastra-ai/mastra/commit/e59e0d32afb5fcf2c9f3c00c8f81f6c21d3a63fa), [`fa8409b`](https://github.com/mastra-ai/mastra/commit/fa8409bc39cfd8ba6643b9db5269b90b22e2a2f7), [`173c535`](https://github.com/mastra-ai/mastra/commit/173c535c0645b0da404fe09f003778f0b0d4e019)]:
|
|
66
|
+
- @mastra/core@1.0.0-beta.0
|
|
20
67
|
|
|
21
68
|
## 0.10.16
|
|
22
69
|
|
package/README.md
CHANGED
|
@@ -337,8 +337,8 @@ const searchTool = createTool({
|
|
|
337
337
|
id: 'search',
|
|
338
338
|
description: 'Search the web',
|
|
339
339
|
inputSchema: z.object({ query: z.string() }),
|
|
340
|
-
execute: async
|
|
341
|
-
const { query } =
|
|
340
|
+
execute: async inputData => {
|
|
341
|
+
const { query } = inputData;
|
|
342
342
|
// ... perform search
|
|
343
343
|
return { results: [] };
|
|
344
344
|
},
|
|
@@ -355,7 +355,7 @@ voice.addTools({
|
|
|
355
355
|
id: 'search',
|
|
356
356
|
description: 'Search the web',
|
|
357
357
|
inputSchema: { type: 'object', properties: { query: { type: 'string' } } },
|
|
358
|
-
execute: async (
|
|
358
|
+
execute: async (inputData, context) => ({ results: [] }),
|
|
359
359
|
},
|
|
360
360
|
});
|
|
361
361
|
```
|
package/dist/index.cjs
CHANGED
|
@@ -1262,7 +1262,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
|
|
|
1262
1262
|
sessionDurationTimeout;
|
|
1263
1263
|
// Tool integration properties
|
|
1264
1264
|
tools;
|
|
1265
|
-
|
|
1265
|
+
requestContext;
|
|
1266
1266
|
// Store the configuration options
|
|
1267
1267
|
options;
|
|
1268
1268
|
/**
|
|
@@ -1499,69 +1499,67 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
|
|
|
1499
1499
|
/**
|
|
1500
1500
|
* Establish connection to the Gemini Live API
|
|
1501
1501
|
*/
|
|
1502
|
-
async connect({
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
headers
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
"Content-Type": "application/json"
|
|
1525
|
-
}
|
|
1526
|
-
};
|
|
1527
|
-
this.log("Using Live API authentication with API key");
|
|
1528
|
-
}
|
|
1529
|
-
this.log("Connecting to:", wsUrl);
|
|
1530
|
-
this.ws = new ws.WebSocket(wsUrl, void 0, headers);
|
|
1531
|
-
this.connectionManager.setWebSocket(this.ws);
|
|
1532
|
-
this.setupEventListeners();
|
|
1533
|
-
await this.connectionManager.waitForOpen();
|
|
1534
|
-
if (this.isResuming && this.sessionHandle) {
|
|
1535
|
-
await this.sendSessionResumption();
|
|
1536
|
-
} else {
|
|
1537
|
-
this.sendInitialConfig();
|
|
1538
|
-
this.sessionStartTime = Date.now();
|
|
1539
|
-
this.sessionId = crypto.randomUUID();
|
|
1540
|
-
}
|
|
1541
|
-
await this.waitForSessionCreated();
|
|
1542
|
-
this.state = "connected";
|
|
1543
|
-
this.emit("session", {
|
|
1544
|
-
state: "connected",
|
|
1545
|
-
config: {
|
|
1546
|
-
sessionId: this.sessionId,
|
|
1547
|
-
isResuming: this.isResuming,
|
|
1548
|
-
toolCount: Object.keys(this.tools || {}).length
|
|
1502
|
+
async connect({ requestContext } = {}) {
|
|
1503
|
+
if (this.state === "connected") {
|
|
1504
|
+
this.log("Already connected to Gemini Live API");
|
|
1505
|
+
return;
|
|
1506
|
+
}
|
|
1507
|
+
this.requestContext = requestContext;
|
|
1508
|
+
this.emit("session", { state: "connecting" });
|
|
1509
|
+
try {
|
|
1510
|
+
let wsUrl;
|
|
1511
|
+
let headers = {};
|
|
1512
|
+
if (this.options.vertexAI) {
|
|
1513
|
+
wsUrl = `wss://${this.options.location}-aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1beta1.PredictionService.ServerStreamingPredict`;
|
|
1514
|
+
await this.authManager.initialize();
|
|
1515
|
+
const accessToken = await this.authManager.getAccessToken();
|
|
1516
|
+
headers = { headers: { Authorization: `Bearer ${accessToken}` } };
|
|
1517
|
+
this.log("Using Vertex AI authentication with OAuth token");
|
|
1518
|
+
} else {
|
|
1519
|
+
wsUrl = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent`;
|
|
1520
|
+
headers = {
|
|
1521
|
+
headers: {
|
|
1522
|
+
"x-goog-api-key": this.options.apiKey || "",
|
|
1523
|
+
"Content-Type": "application/json"
|
|
1549
1524
|
}
|
|
1550
|
-
}
|
|
1551
|
-
this.log("
|
|
1525
|
+
};
|
|
1526
|
+
this.log("Using Live API authentication with API key");
|
|
1527
|
+
}
|
|
1528
|
+
this.log("Connecting to:", wsUrl);
|
|
1529
|
+
this.ws = new ws.WebSocket(wsUrl, void 0, headers);
|
|
1530
|
+
this.connectionManager.setWebSocket(this.ws);
|
|
1531
|
+
this.setupEventListeners();
|
|
1532
|
+
await this.connectionManager.waitForOpen();
|
|
1533
|
+
if (this.isResuming && this.sessionHandle) {
|
|
1534
|
+
await this.sendSessionResumption();
|
|
1535
|
+
} else {
|
|
1536
|
+
this.sendInitialConfig();
|
|
1537
|
+
this.sessionStartTime = Date.now();
|
|
1538
|
+
this.sessionId = crypto.randomUUID();
|
|
1539
|
+
}
|
|
1540
|
+
await this.waitForSessionCreated();
|
|
1541
|
+
this.state = "connected";
|
|
1542
|
+
this.emit("session", {
|
|
1543
|
+
state: "connected",
|
|
1544
|
+
config: {
|
|
1552
1545
|
sessionId: this.sessionId,
|
|
1553
1546
|
isResuming: this.isResuming,
|
|
1554
1547
|
toolCount: Object.keys(this.tools || {}).length
|
|
1555
|
-
});
|
|
1556
|
-
if (this.options.sessionConfig?.maxDuration) {
|
|
1557
|
-
this.startSessionDurationMonitor();
|
|
1558
1548
|
}
|
|
1559
|
-
}
|
|
1560
|
-
|
|
1561
|
-
this.
|
|
1562
|
-
|
|
1549
|
+
});
|
|
1550
|
+
this.log("Successfully connected to Gemini Live API", {
|
|
1551
|
+
sessionId: this.sessionId,
|
|
1552
|
+
isResuming: this.isResuming,
|
|
1553
|
+
toolCount: Object.keys(this.tools || {}).length
|
|
1554
|
+
});
|
|
1555
|
+
if (this.options.sessionConfig?.maxDuration) {
|
|
1556
|
+
this.startSessionDurationMonitor();
|
|
1563
1557
|
}
|
|
1564
|
-
}
|
|
1558
|
+
} catch (error) {
|
|
1559
|
+
this.state = "disconnected";
|
|
1560
|
+
this.log("Connection failed", error);
|
|
1561
|
+
throw error;
|
|
1562
|
+
}
|
|
1565
1563
|
}
|
|
1566
1564
|
/**
|
|
1567
1565
|
* Disconnect from the Gemini Live API
|
|
@@ -1599,172 +1597,164 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
|
|
|
1599
1597
|
* Send text to be converted to speech
|
|
1600
1598
|
*/
|
|
1601
1599
|
async speak(input, options) {
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)));
|
|
1608
|
-
}
|
|
1609
|
-
input = Buffer.concat(chunks).toString("utf-8");
|
|
1610
|
-
}
|
|
1611
|
-
if (input.trim().length === 0) {
|
|
1612
|
-
throw this.createAndEmitError("invalid_audio_format" /* INVALID_AUDIO_FORMAT */, "Input text is empty");
|
|
1600
|
+
this.validateConnectionState();
|
|
1601
|
+
if (typeof input !== "string") {
|
|
1602
|
+
const chunks = [];
|
|
1603
|
+
for await (const chunk of input) {
|
|
1604
|
+
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)));
|
|
1613
1605
|
}
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
}
|
|
1629
|
-
};
|
|
1630
|
-
if (options && (options.speaker || options.languageCode || options.responseModalities)) {
|
|
1631
|
-
const updateMessage = {
|
|
1632
|
-
type: "session.update",
|
|
1633
|
-
session: {
|
|
1634
|
-
generation_config: {
|
|
1635
|
-
...options.responseModalities ? { response_modalities: options.responseModalities } : {},
|
|
1636
|
-
speech_config: {
|
|
1637
|
-
...options.languageCode ? { language_code: options.languageCode } : {},
|
|
1638
|
-
...options.speaker ? { voice_config: { prebuilt_voice_config: { voice_name: options.speaker } } } : {}
|
|
1606
|
+
input = Buffer.concat(chunks).toString("utf-8");
|
|
1607
|
+
}
|
|
1608
|
+
if (input.trim().length === 0) {
|
|
1609
|
+
throw this.createAndEmitError("invalid_audio_format" /* INVALID_AUDIO_FORMAT */, "Input text is empty");
|
|
1610
|
+
}
|
|
1611
|
+
this.addToContext("user", input);
|
|
1612
|
+
const textMessage = {
|
|
1613
|
+
client_content: {
|
|
1614
|
+
turns: [
|
|
1615
|
+
{
|
|
1616
|
+
role: "user",
|
|
1617
|
+
parts: [
|
|
1618
|
+
{
|
|
1619
|
+
text: input
|
|
1639
1620
|
}
|
|
1621
|
+
]
|
|
1622
|
+
}
|
|
1623
|
+
],
|
|
1624
|
+
turnComplete: true
|
|
1625
|
+
}
|
|
1626
|
+
};
|
|
1627
|
+
if (options && (options.speaker || options.languageCode || options.responseModalities)) {
|
|
1628
|
+
const updateMessage = {
|
|
1629
|
+
type: "session.update",
|
|
1630
|
+
session: {
|
|
1631
|
+
generation_config: {
|
|
1632
|
+
...options.responseModalities ? { response_modalities: options.responseModalities } : {},
|
|
1633
|
+
speech_config: {
|
|
1634
|
+
...options.languageCode ? { language_code: options.languageCode } : {},
|
|
1635
|
+
...options.speaker ? { voice_config: { prebuilt_voice_config: { voice_name: options.speaker } } } : {}
|
|
1640
1636
|
}
|
|
1641
1637
|
}
|
|
1642
|
-
};
|
|
1643
|
-
try {
|
|
1644
|
-
this.sendEvent("session.update", updateMessage);
|
|
1645
|
-
this.log("Applied per-turn runtime options", options);
|
|
1646
|
-
} catch (error) {
|
|
1647
|
-
this.log("Failed to apply per-turn runtime options", error);
|
|
1648
1638
|
}
|
|
1649
|
-
}
|
|
1639
|
+
};
|
|
1650
1640
|
try {
|
|
1651
|
-
this.sendEvent("
|
|
1652
|
-
this.log("
|
|
1641
|
+
this.sendEvent("session.update", updateMessage);
|
|
1642
|
+
this.log("Applied per-turn runtime options", options);
|
|
1653
1643
|
} catch (error) {
|
|
1654
|
-
this.log("Failed to
|
|
1655
|
-
throw this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to send text message", error);
|
|
1644
|
+
this.log("Failed to apply per-turn runtime options", error);
|
|
1656
1645
|
}
|
|
1657
|
-
}
|
|
1646
|
+
}
|
|
1647
|
+
try {
|
|
1648
|
+
this.sendEvent("client_content", textMessage);
|
|
1649
|
+
this.log("Text message sent", { text: input });
|
|
1650
|
+
} catch (error) {
|
|
1651
|
+
this.log("Failed to send text message", error);
|
|
1652
|
+
throw this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to send text message", error);
|
|
1653
|
+
}
|
|
1658
1654
|
}
|
|
1659
1655
|
/**
|
|
1660
1656
|
* Send audio stream for processing
|
|
1661
1657
|
*/
|
|
1662
1658
|
async send(audioData) {
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
}
|
|
1690
|
-
}, "gemini-live.send")();
|
|
1659
|
+
this.validateConnectionState();
|
|
1660
|
+
if ("readable" in audioData && typeof audioData.on === "function") {
|
|
1661
|
+
const stream = audioData;
|
|
1662
|
+
stream.on("data", (chunk) => {
|
|
1663
|
+
try {
|
|
1664
|
+
const base64Audio = this.audioStreamManager.processAudioChunk(chunk);
|
|
1665
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "realtime");
|
|
1666
|
+
this.sendEvent("realtime_input", message);
|
|
1667
|
+
} catch (error) {
|
|
1668
|
+
this.log("Failed to process audio chunk", error);
|
|
1669
|
+
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to process audio chunk", error);
|
|
1670
|
+
}
|
|
1671
|
+
});
|
|
1672
|
+
stream.on("error", (error) => {
|
|
1673
|
+
this.log("Audio stream error", error);
|
|
1674
|
+
this.createAndEmitError("audio_stream_error" /* AUDIO_STREAM_ERROR */, "Audio stream error", error);
|
|
1675
|
+
});
|
|
1676
|
+
stream.on("end", () => {
|
|
1677
|
+
this.log("Audio stream ended");
|
|
1678
|
+
});
|
|
1679
|
+
} else {
|
|
1680
|
+
const validateAudio = this.audioStreamManager.validateAndConvertAudioInput(audioData);
|
|
1681
|
+
const base64Audio = this.audioStreamManager.int16ArrayToBase64(validateAudio);
|
|
1682
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "realtime");
|
|
1683
|
+
this.sendEvent("realtime_input", message);
|
|
1684
|
+
}
|
|
1691
1685
|
}
|
|
1692
1686
|
/**
|
|
1693
1687
|
* Process speech from audio stream (traditional STT interface)
|
|
1694
1688
|
*/
|
|
1695
1689
|
async listen(audioStream, _options) {
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
this.log("Received transcription text:", { text: data.text, total: transcriptionText });
|
|
1703
|
-
}
|
|
1704
|
-
};
|
|
1705
|
-
const onError = (error) => {
|
|
1706
|
-
throw new Error(`Transcription failed: ${error.message}`);
|
|
1707
|
-
};
|
|
1708
|
-
const onSession = (data) => {
|
|
1709
|
-
if (data.state === "disconnected") {
|
|
1710
|
-
throw new Error("Session disconnected during transcription");
|
|
1711
|
-
}
|
|
1712
|
-
};
|
|
1713
|
-
this.on("writing", onWriting);
|
|
1714
|
-
this.on("error", onError);
|
|
1715
|
-
this.on("session", onSession);
|
|
1716
|
-
try {
|
|
1717
|
-
const result = await this.audioStreamManager.handleAudioTranscription(
|
|
1718
|
-
audioStream,
|
|
1719
|
-
(base64Audio) => {
|
|
1720
|
-
return new Promise((resolve, reject) => {
|
|
1721
|
-
try {
|
|
1722
|
-
const message = this.audioStreamManager.createAudioMessage(base64Audio, "input");
|
|
1723
|
-
const cleanup = () => {
|
|
1724
|
-
this.off("turnComplete", onTurnComplete);
|
|
1725
|
-
this.off("error", onErr);
|
|
1726
|
-
};
|
|
1727
|
-
const onTurnComplete = () => {
|
|
1728
|
-
cleanup();
|
|
1729
|
-
resolve(transcriptionText.trim());
|
|
1730
|
-
};
|
|
1731
|
-
const onErr = (e) => {
|
|
1732
|
-
cleanup();
|
|
1733
|
-
reject(new Error(e.message));
|
|
1734
|
-
};
|
|
1735
|
-
this.on("turnComplete", onTurnComplete);
|
|
1736
|
-
this.on("error", onErr);
|
|
1737
|
-
this.sendEvent("client_content", message);
|
|
1738
|
-
this.log("Sent audio for transcription");
|
|
1739
|
-
} catch (err) {
|
|
1740
|
-
reject(err);
|
|
1741
|
-
}
|
|
1742
|
-
});
|
|
1743
|
-
},
|
|
1744
|
-
(error) => {
|
|
1745
|
-
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Audio transcription failed", error);
|
|
1746
|
-
}
|
|
1747
|
-
);
|
|
1748
|
-
return result;
|
|
1749
|
-
} finally {
|
|
1750
|
-
this.off("writing", onWriting);
|
|
1751
|
-
this.off("error", onError);
|
|
1752
|
-
this.off("session", onSession);
|
|
1690
|
+
this.validateConnectionState();
|
|
1691
|
+
let transcriptionText = "";
|
|
1692
|
+
const onWriting = (data) => {
|
|
1693
|
+
if (data.role === "user") {
|
|
1694
|
+
transcriptionText += data.text;
|
|
1695
|
+
this.log("Received transcription text:", { text: data.text, total: transcriptionText });
|
|
1753
1696
|
}
|
|
1754
|
-
}
|
|
1697
|
+
};
|
|
1698
|
+
const onError = (error) => {
|
|
1699
|
+
throw new Error(`Transcription failed: ${error.message}`);
|
|
1700
|
+
};
|
|
1701
|
+
const onSession = (data) => {
|
|
1702
|
+
if (data.state === "disconnected") {
|
|
1703
|
+
throw new Error("Session disconnected during transcription");
|
|
1704
|
+
}
|
|
1705
|
+
};
|
|
1706
|
+
this.on("writing", onWriting);
|
|
1707
|
+
this.on("error", onError);
|
|
1708
|
+
this.on("session", onSession);
|
|
1709
|
+
try {
|
|
1710
|
+
const result = await this.audioStreamManager.handleAudioTranscription(
|
|
1711
|
+
audioStream,
|
|
1712
|
+
(base64Audio) => {
|
|
1713
|
+
return new Promise((resolve, reject) => {
|
|
1714
|
+
try {
|
|
1715
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "input");
|
|
1716
|
+
const cleanup = () => {
|
|
1717
|
+
this.off("turnComplete", onTurnComplete);
|
|
1718
|
+
this.off("error", onErr);
|
|
1719
|
+
};
|
|
1720
|
+
const onTurnComplete = () => {
|
|
1721
|
+
cleanup();
|
|
1722
|
+
resolve(transcriptionText.trim());
|
|
1723
|
+
};
|
|
1724
|
+
const onErr = (e) => {
|
|
1725
|
+
cleanup();
|
|
1726
|
+
reject(new Error(e.message));
|
|
1727
|
+
};
|
|
1728
|
+
this.on("turnComplete", onTurnComplete);
|
|
1729
|
+
this.on("error", onErr);
|
|
1730
|
+
this.sendEvent("client_content", message);
|
|
1731
|
+
this.log("Sent audio for transcription");
|
|
1732
|
+
} catch (err) {
|
|
1733
|
+
reject(err);
|
|
1734
|
+
}
|
|
1735
|
+
});
|
|
1736
|
+
},
|
|
1737
|
+
(error) => {
|
|
1738
|
+
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Audio transcription failed", error);
|
|
1739
|
+
}
|
|
1740
|
+
);
|
|
1741
|
+
return result;
|
|
1742
|
+
} finally {
|
|
1743
|
+
this.off("writing", onWriting);
|
|
1744
|
+
this.off("error", onError);
|
|
1745
|
+
this.off("session", onSession);
|
|
1746
|
+
}
|
|
1755
1747
|
}
|
|
1756
1748
|
/**
|
|
1757
1749
|
* Get available speakers/voices
|
|
1758
1750
|
*/
|
|
1759
1751
|
async getSpeakers() {
|
|
1760
|
-
return
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
];
|
|
1767
|
-
}, "gemini-live.getSpeakers")();
|
|
1752
|
+
return [
|
|
1753
|
+
{ voiceId: "Puck", description: "Conversational, friendly" },
|
|
1754
|
+
{ voiceId: "Charon", description: "Deep, authoritative" },
|
|
1755
|
+
{ voiceId: "Kore", description: "Neutral, professional" },
|
|
1756
|
+
{ voiceId: "Fenrir", description: "Warm, approachable" }
|
|
1757
|
+
];
|
|
1768
1758
|
}
|
|
1769
1759
|
/**
|
|
1770
1760
|
* Resume a previous session using a session handle
|
|
@@ -2365,13 +2355,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
|
|
|
2365
2355
|
let result;
|
|
2366
2356
|
if (tool.execute) {
|
|
2367
2357
|
this.log("Executing tool", { toolName, toolArgs });
|
|
2368
|
-
result = await tool.execute(
|
|
2369
|
-
{ context: toolArgs, runtimeContext: this.runtimeContext },
|
|
2370
|
-
{
|
|
2371
|
-
toolCallId: toolId,
|
|
2372
|
-
messages: []
|
|
2373
|
-
}
|
|
2374
|
-
);
|
|
2358
|
+
result = await tool.execute(toolArgs, { requestContext: this.requestContext });
|
|
2375
2359
|
this.log("Tool executed successfully", { toolName, result });
|
|
2376
2360
|
} else {
|
|
2377
2361
|
this.log("Tool has no execute function", { toolName });
|
|
@@ -2638,14 +2622,14 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
|
|
|
2638
2622
|
* inputSchema: z.object({
|
|
2639
2623
|
* location: z.string().describe("The city and state, e.g. San Francisco, CA"),
|
|
2640
2624
|
* }),
|
|
2641
|
-
* execute: async (
|
|
2625
|
+
* execute: async (inputData) => {
|
|
2642
2626
|
* // Fetch weather data from an API
|
|
2643
2627
|
* const response = await fetch(
|
|
2644
|
-
* `https://api.weather.com?location=${encodeURIComponent(
|
|
2628
|
+
* `https://api.weather.com?location=${encodeURIComponent(inputData.location)}`,
|
|
2645
2629
|
* );
|
|
2646
2630
|
* const data = await response.json();
|
|
2647
2631
|
* return {
|
|
2648
|
-
* message: `The current temperature in ${
|
|
2632
|
+
* message: `The current temperature in ${inputData.location} is ${data.temperature}°F with ${data.conditions}.`,
|
|
2649
2633
|
* };
|
|
2650
2634
|
* },
|
|
2651
2635
|
* });
|
|
@@ -2663,7 +2647,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends voice.MastraVoice {
|
|
|
2663
2647
|
* Get the current tools configured for this voice instance
|
|
2664
2648
|
* @returns Object containing the current tools
|
|
2665
2649
|
*/
|
|
2666
|
-
|
|
2650
|
+
listTools() {
|
|
2667
2651
|
return this.tools;
|
|
2668
2652
|
}
|
|
2669
2653
|
log(message, ...args) {
|