@mastra/voice-google-gemini-live 0.0.0-fix-issue-10434-concurrent-write-corruption-20251124213939 → 0.0.0-fix-backport-setserver-20251201151948
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -118
- package/README.md +3 -3
- package/dist/index.cjs +212 -196
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +7 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +212 -196
- package/dist/index.js.map +1 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +13 -11
package/dist/index.d.ts
CHANGED
|
@@ -74,7 +74,7 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
74
74
|
private isResuming;
|
|
75
75
|
private sessionDurationTimeout?;
|
|
76
76
|
private tools?;
|
|
77
|
-
private
|
|
77
|
+
private runtimeContext?;
|
|
78
78
|
private options;
|
|
79
79
|
/**
|
|
80
80
|
* Normalize configuration to ensure proper VoiceConfig format
|
|
@@ -162,8 +162,8 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
162
162
|
/**
|
|
163
163
|
* Establish connection to the Gemini Live API
|
|
164
164
|
*/
|
|
165
|
-
connect({
|
|
166
|
-
|
|
165
|
+
connect({ runtimeContext }?: {
|
|
166
|
+
runtimeContext?: any;
|
|
167
167
|
}): Promise<void>;
|
|
168
168
|
/**
|
|
169
169
|
* Disconnect from the Gemini Live API
|
|
@@ -400,14 +400,14 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
400
400
|
* inputSchema: z.object({
|
|
401
401
|
* location: z.string().describe("The city and state, e.g. San Francisco, CA"),
|
|
402
402
|
* }),
|
|
403
|
-
* execute: async (
|
|
403
|
+
* execute: async ({ context }) => {
|
|
404
404
|
* // Fetch weather data from an API
|
|
405
405
|
* const response = await fetch(
|
|
406
|
-
* `https://api.weather.com?location=${encodeURIComponent(
|
|
406
|
+
* `https://api.weather.com?location=${encodeURIComponent(context.location)}`,
|
|
407
407
|
* );
|
|
408
408
|
* const data = await response.json();
|
|
409
409
|
* return {
|
|
410
|
-
* message: `The current temperature in ${
|
|
410
|
+
* message: `The current temperature in ${context.location} is ${data.temperature}°F with ${data.conditions}.`,
|
|
411
411
|
* };
|
|
412
412
|
* },
|
|
413
413
|
* });
|
|
@@ -422,7 +422,7 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
422
422
|
* Get the current tools configured for this voice instance
|
|
423
423
|
* @returns Object containing the current tools
|
|
424
424
|
*/
|
|
425
|
-
|
|
425
|
+
getTools(): ToolsInput | undefined;
|
|
426
426
|
private log;
|
|
427
427
|
/**
|
|
428
428
|
* Convert Zod schema to JSON Schema for tool parameters
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAKtE,OAAO,KAAK,EACV,qBAAqB,EACrB,sBAAsB,EACtB,kBAAkB,EAMlB,mBAAmB,EAEpB,MAAM,SAAS,CAAC;AAYjB;;GAEG;AAGH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,qBAAa,eAAgB,SAAQ,WAAW,CAC9C,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,UAAU,EACV,kBAAkB,CACnB;IACC,OAAO,CAAC,EAAE,CAAC,CAAS;IACpB,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,KAAK,CAAgD;IAC7D,OAAO,CAAC,aAAa,CAAC,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAU;IAChC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,KAAK,CAAiB;IAG9B,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,WAAW,CAAc;IAGjC,OAAO,CAAC,kBAAkB,CAAqB;IAG/C,OAAO,CAAC,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,gBAAgB,CAAC,CAAS;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,sBAAsB,CAAC,CAAiB;IAGhD,OAAO,CAAC,KAAK,CAAC,CAAa;IAC3B,OAAO,CAAC,cAAc,CAAC,CAAM;IAG7B,OAAO,CAAC,OAAO,CAAwB;IAEvC;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,eAAe;IAwB9B;;;;OAIG;gBACS,MAAM,GAAE,WAAW,CAAC,qBAAqB,CAAC,GAAG,qBAA0B;IA0DnF;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,EAAE,CAAC,CAAC,SAAS,cAAc,EACzB,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,cAAc,EAC1B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IASP;;;;OAIG;IACH,IAAI,CAAC,CAAC,SAAS,cAAc,EAC3B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;OAGG;IACH,OAAO,CAAC,IAAI;IAoCZ;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAoB7B;;;OAGG;IACH,oBAAoB,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAS9C;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAO1B;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAU/B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAU9B;;OAEG;IACG,OAAO,CAAC,EAAE,cAAc,EAAE,GAAE;QAAE,cAAc,CAAC,EAAE,GAAG,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAKtE,OAAO,KAAK,EACV,qBAAqB,EACrB,sBAAsB,EACtB,kBAAkB,EAMlB,mBAAmB,EAEpB,MAAM,SAAS,CAAC;AAYjB;;GAEG;AAGH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,qBAAa,eAAgB,SAAQ,WAAW,CAC9C,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,UAAU,EACV,kBAAkB,CACnB;IACC,OAAO,CAAC,EAAE,CAAC,CAAS;IACpB,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,KAAK,CAAgD;IAC7D,OAAO,CAAC,aAAa,CAAC,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAU;IAChC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,KAAK,CAAiB;IAG9B,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,WAAW,CAAc;IAGjC,OAAO,CAAC,kBAAkB,CAAqB;IAG/C,OAAO,CAAC,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,gBAAgB,CAAC,CAAS;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,sBAAsB,CAAC,CAAiB;IAGhD,OAAO,CAAC,KAAK,CAAC,CAAa;IAC3B,OAAO,CAAC,cAAc,CAAC,CAAM;IAG7B,OAAO,CAAC,OAAO,CAAwB;IAEvC;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,eAAe;IAwB9B;;;;OAIG;gBACS,MAAM,GAAE,WAAW,CAAC,qBAAqB,CAAC,GAAG,qBAA0B;IA0DnF;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,EAAE,CAAC,CAAC,SAAS,cAAc,EACzB,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,cAAc,EAC1B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IASP;;;;OAIG;IACH,IAAI,CAAC,CAAC,SAAS,cAAc,EAC3B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;OAGG;IACH,OAAO,CAAC,IAAI;IAoCZ;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAoB7B;;;OAGG;IACH,oBAAoB,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAS9C;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAO1B;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAU/B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAU9B;;OAEG;IACG,OAAO,CAAC,EAAE,cAAc,EAAE,GAAE;QAAE,cAAc,CAAC,EAAE,GAAG,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IA0F/E;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAiDjC;;OAEG;IACG,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,cAAc,EAAE,OAAO,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC;IA2EnG;;OAEG;IACG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,cAAc,GAAG,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAmCxE;;OAEG;IACG,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,cAAc,EAAE,QAAQ,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,MAAM,CAAC;IAuFpG;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAY9E;;OAEG;IACG,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BtG;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACG,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA6MhF;;OAEG;IACH,kBAAkB,IAAI,cAAc,GAAG,WAAW;IAIlD;;OAEG;IACH,WAAW,IAAI,OAAO;IAItB;;;OAGG;IACH,uBAAuB,IAAI,MAAM,CAAC,cAAc,GAAG,IAAI;IAIvD;;OAEG;IACH,gBAAgB,IAAI,MAAM,GAAG,SAAS;IAKtC;;OAEG;IACH,cAAc,IAAI;QAChB,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,IAAI,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,mBAAmB,CAAC;QAC7B,WAAW,EAAE,MAAM,CAAC;KACrB;IAYD;;OAEG;IACH,iBAAiB,IAAI,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IAIhF;;OAEG;IACH,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI;IAI/D;;OAEG;IACH,YAAY,IAAI,IAAI;IAKpB;;OAEG;IACH,gBAAgB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAQxC;;;OAGG;YACW,qBAAqB;IA4BnC;;;OAGG;IACH,OAAO,CAAC,2BAA2B;IAqCnC;;;OAGG;IACH,OAAO,CAAC,aAAa;IAmBrB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAKvB;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IA6C3B;;;OAGG;YACW,mBAAmB;IA0EjC;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAwB3B;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAY5B;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAsI3B;;;OAGG;YACW,cAAc;IA4B5B;;;OAGG;YACW,qBAAqB;IAoFnC;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAWzB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAMxB;;;OAGG;IACH,OAAO,CAAC,WAAW;IAcnB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAYzB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAIzB;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAkB9B;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAmIzB;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAqD7B;;;;OAIG;YACW,cAAc;IAO5B;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAI5B;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAI5B;;;OAGG;IACH,OAAO,CAAC,SAAS;IAkCjB;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,QAAQ,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAKjC;;;OAGG;IACH,QAAQ,IAAI,UAAU,GAAG,SAAS;IAIlC,OAAO,CAAC,GAAG;IAMX;;;OAGG;IACH,OAAO,CAAC,4BAA4B;IAiCpC;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IA2DjC;;OAEG;IACH,KAAK,IAAI,IAAI;IAIb;;OAEG;IACG,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAO/D;;;OAGG;IACH,eAAe,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI;CAM7C"}
|
package/dist/index.js
CHANGED
|
@@ -1260,7 +1260,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
1260
1260
|
sessionDurationTimeout;
|
|
1261
1261
|
// Tool integration properties
|
|
1262
1262
|
tools;
|
|
1263
|
-
|
|
1263
|
+
runtimeContext;
|
|
1264
1264
|
// Store the configuration options
|
|
1265
1265
|
options;
|
|
1266
1266
|
/**
|
|
@@ -1497,68 +1497,70 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
1497
1497
|
/**
|
|
1498
1498
|
* Establish connection to the Gemini Live API
|
|
1499
1499
|
*/
|
|
1500
|
-
async connect({
|
|
1501
|
-
|
|
1502
|
-
this.
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
this.requestContext = requestContext;
|
|
1506
|
-
this.emit("session", { state: "connecting" });
|
|
1507
|
-
try {
|
|
1508
|
-
let wsUrl;
|
|
1509
|
-
let headers = {};
|
|
1510
|
-
if (this.options.vertexAI) {
|
|
1511
|
-
const location = this.getVertexLocation();
|
|
1512
|
-
wsUrl = `wss://${location}-aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1beta1.LlmBidiService/BidiGenerateContent`;
|
|
1513
|
-
await this.authManager.initialize();
|
|
1514
|
-
const accessToken = await this.authManager.getAccessToken();
|
|
1515
|
-
headers = { headers: { Authorization: `Bearer ${accessToken}` } };
|
|
1516
|
-
this.log("Using Vertex AI authentication with OAuth token");
|
|
1517
|
-
} else {
|
|
1518
|
-
wsUrl = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent`;
|
|
1519
|
-
headers = {
|
|
1520
|
-
headers: {
|
|
1521
|
-
"x-goog-api-key": this.options.apiKey || "",
|
|
1522
|
-
"Content-Type": "application/json"
|
|
1523
|
-
}
|
|
1524
|
-
};
|
|
1525
|
-
this.log("Using Live API authentication with API key");
|
|
1526
|
-
}
|
|
1527
|
-
this.log("Connecting to:", wsUrl);
|
|
1528
|
-
this.ws = new WebSocket(wsUrl, void 0, headers);
|
|
1529
|
-
this.connectionManager.setWebSocket(this.ws);
|
|
1530
|
-
this.setupEventListeners();
|
|
1531
|
-
await this.connectionManager.waitForOpen();
|
|
1532
|
-
if (this.isResuming && this.sessionHandle) {
|
|
1533
|
-
await this.sendSessionResumption();
|
|
1534
|
-
} else {
|
|
1535
|
-
this.sendInitialConfig();
|
|
1536
|
-
this.sessionStartTime = Date.now();
|
|
1537
|
-
this.sessionId = randomUUID();
|
|
1500
|
+
async connect({ runtimeContext } = {}) {
|
|
1501
|
+
return this.traced(async () => {
|
|
1502
|
+
if (this.state === "connected") {
|
|
1503
|
+
this.log("Already connected to Gemini Live API");
|
|
1504
|
+
return;
|
|
1538
1505
|
}
|
|
1539
|
-
|
|
1540
|
-
this.state
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1506
|
+
this.runtimeContext = runtimeContext;
|
|
1507
|
+
this.emit("session", { state: "connecting" });
|
|
1508
|
+
try {
|
|
1509
|
+
let wsUrl;
|
|
1510
|
+
let headers = {};
|
|
1511
|
+
if (this.options.vertexAI) {
|
|
1512
|
+
const location = this.getVertexLocation();
|
|
1513
|
+
wsUrl = `wss://${location}-aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1beta1.LlmBidiService/BidiGenerateContent`;
|
|
1514
|
+
await this.authManager.initialize();
|
|
1515
|
+
const accessToken = await this.authManager.getAccessToken();
|
|
1516
|
+
headers = { headers: { Authorization: `Bearer ${accessToken}` } };
|
|
1517
|
+
this.log("Using Vertex AI authentication with OAuth token");
|
|
1518
|
+
} else {
|
|
1519
|
+
wsUrl = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent`;
|
|
1520
|
+
headers = {
|
|
1521
|
+
headers: {
|
|
1522
|
+
"x-goog-api-key": this.options.apiKey || "",
|
|
1523
|
+
"Content-Type": "application/json"
|
|
1524
|
+
}
|
|
1525
|
+
};
|
|
1526
|
+
this.log("Using Live API authentication with API key");
|
|
1527
|
+
}
|
|
1528
|
+
this.log("Connecting to:", wsUrl);
|
|
1529
|
+
this.ws = new WebSocket(wsUrl, void 0, headers);
|
|
1530
|
+
this.connectionManager.setWebSocket(this.ws);
|
|
1531
|
+
this.setupEventListeners();
|
|
1532
|
+
await this.connectionManager.waitForOpen();
|
|
1533
|
+
if (this.isResuming && this.sessionHandle) {
|
|
1534
|
+
await this.sendSessionResumption();
|
|
1535
|
+
} else {
|
|
1536
|
+
this.sendInitialConfig();
|
|
1537
|
+
this.sessionStartTime = Date.now();
|
|
1538
|
+
this.sessionId = randomUUID();
|
|
1539
|
+
}
|
|
1540
|
+
await this.waitForSessionCreated();
|
|
1541
|
+
this.state = "connected";
|
|
1542
|
+
this.emit("session", {
|
|
1543
|
+
state: "connected",
|
|
1544
|
+
config: {
|
|
1545
|
+
sessionId: this.sessionId,
|
|
1546
|
+
isResuming: this.isResuming,
|
|
1547
|
+
toolCount: Object.keys(this.tools || {}).length
|
|
1548
|
+
}
|
|
1549
|
+
});
|
|
1550
|
+
this.log("Successfully connected to Gemini Live API", {
|
|
1544
1551
|
sessionId: this.sessionId,
|
|
1545
1552
|
isResuming: this.isResuming,
|
|
1546
1553
|
toolCount: Object.keys(this.tools || {}).length
|
|
1554
|
+
});
|
|
1555
|
+
if (this.options.sessionConfig?.maxDuration) {
|
|
1556
|
+
this.startSessionDurationMonitor();
|
|
1547
1557
|
}
|
|
1548
|
-
})
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
toolCount: Object.keys(this.tools || {}).length
|
|
1553
|
-
});
|
|
1554
|
-
if (this.options.sessionConfig?.maxDuration) {
|
|
1555
|
-
this.startSessionDurationMonitor();
|
|
1558
|
+
} catch (error) {
|
|
1559
|
+
this.state = "disconnected";
|
|
1560
|
+
this.log("Connection failed", error);
|
|
1561
|
+
throw error;
|
|
1556
1562
|
}
|
|
1557
|
-
}
|
|
1558
|
-
this.state = "disconnected";
|
|
1559
|
-
this.log("Connection failed", error);
|
|
1560
|
-
throw error;
|
|
1561
|
-
}
|
|
1563
|
+
}, "gemini-live.connect")();
|
|
1562
1564
|
}
|
|
1563
1565
|
/**
|
|
1564
1566
|
* Disconnect from the Gemini Live API
|
|
@@ -1596,164 +1598,172 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
1596
1598
|
* Send text to be converted to speech
|
|
1597
1599
|
*/
|
|
1598
1600
|
async speak(input, options) {
|
|
1599
|
-
this.
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1601
|
+
return this.traced(async () => {
|
|
1602
|
+
this.validateConnectionState();
|
|
1603
|
+
if (typeof input !== "string") {
|
|
1604
|
+
const chunks = [];
|
|
1605
|
+
for await (const chunk of input) {
|
|
1606
|
+
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)));
|
|
1607
|
+
}
|
|
1608
|
+
input = Buffer.concat(chunks).toString("utf-8");
|
|
1604
1609
|
}
|
|
1605
|
-
input
|
|
1606
|
-
|
|
1607
|
-
if (input.trim().length === 0) {
|
|
1608
|
-
throw this.createAndEmitError("invalid_audio_format" /* INVALID_AUDIO_FORMAT */, "Input text is empty");
|
|
1609
|
-
}
|
|
1610
|
-
this.addToContext("user", input);
|
|
1611
|
-
const textMessage = {
|
|
1612
|
-
client_content: {
|
|
1613
|
-
turns: [
|
|
1614
|
-
{
|
|
1615
|
-
role: "user",
|
|
1616
|
-
parts: [
|
|
1617
|
-
{
|
|
1618
|
-
text: input
|
|
1619
|
-
}
|
|
1620
|
-
]
|
|
1621
|
-
}
|
|
1622
|
-
],
|
|
1623
|
-
turnComplete: true
|
|
1610
|
+
if (input.trim().length === 0) {
|
|
1611
|
+
throw this.createAndEmitError("invalid_audio_format" /* INVALID_AUDIO_FORMAT */, "Input text is empty");
|
|
1624
1612
|
}
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1613
|
+
this.addToContext("user", input);
|
|
1614
|
+
const textMessage = {
|
|
1615
|
+
client_content: {
|
|
1616
|
+
turns: [
|
|
1617
|
+
{
|
|
1618
|
+
role: "user",
|
|
1619
|
+
parts: [
|
|
1620
|
+
{
|
|
1621
|
+
text: input
|
|
1622
|
+
}
|
|
1623
|
+
]
|
|
1635
1624
|
}
|
|
1636
|
-
|
|
1625
|
+
],
|
|
1626
|
+
turnComplete: true
|
|
1637
1627
|
}
|
|
1638
1628
|
};
|
|
1629
|
+
if (options && (options.speaker || options.languageCode || options.responseModalities)) {
|
|
1630
|
+
const updateMessage = {
|
|
1631
|
+
type: "session.update",
|
|
1632
|
+
session: {
|
|
1633
|
+
generation_config: {
|
|
1634
|
+
...options.responseModalities ? { response_modalities: options.responseModalities } : {},
|
|
1635
|
+
speech_config: {
|
|
1636
|
+
...options.languageCode ? { language_code: options.languageCode } : {},
|
|
1637
|
+
...options.speaker ? { voice_config: { prebuilt_voice_config: { voice_name: options.speaker } } } : {}
|
|
1638
|
+
}
|
|
1639
|
+
}
|
|
1640
|
+
}
|
|
1641
|
+
};
|
|
1642
|
+
try {
|
|
1643
|
+
this.sendEvent("session.update", updateMessage);
|
|
1644
|
+
this.log("Applied per-turn runtime options", options);
|
|
1645
|
+
} catch (error) {
|
|
1646
|
+
this.log("Failed to apply per-turn runtime options", error);
|
|
1647
|
+
}
|
|
1648
|
+
}
|
|
1639
1649
|
try {
|
|
1640
|
-
this.sendEvent("
|
|
1641
|
-
this.log("
|
|
1650
|
+
this.sendEvent("client_content", textMessage);
|
|
1651
|
+
this.log("Text message sent", { text: input });
|
|
1642
1652
|
} catch (error) {
|
|
1643
|
-
this.log("Failed to
|
|
1653
|
+
this.log("Failed to send text message", error);
|
|
1654
|
+
throw this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to send text message", error);
|
|
1644
1655
|
}
|
|
1645
|
-
}
|
|
1646
|
-
try {
|
|
1647
|
-
this.sendEvent("client_content", textMessage);
|
|
1648
|
-
this.log("Text message sent", { text: input });
|
|
1649
|
-
} catch (error) {
|
|
1650
|
-
this.log("Failed to send text message", error);
|
|
1651
|
-
throw this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to send text message", error);
|
|
1652
|
-
}
|
|
1656
|
+
}, "gemini-live.speak")();
|
|
1653
1657
|
}
|
|
1654
1658
|
/**
|
|
1655
1659
|
* Send audio stream for processing
|
|
1656
1660
|
*/
|
|
1657
1661
|
async send(audioData) {
|
|
1658
|
-
this.
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1662
|
+
return this.traced(async () => {
|
|
1663
|
+
this.validateConnectionState();
|
|
1664
|
+
if ("readable" in audioData && typeof audioData.on === "function") {
|
|
1665
|
+
const stream = audioData;
|
|
1666
|
+
stream.on("data", (chunk) => {
|
|
1667
|
+
try {
|
|
1668
|
+
const base64Audio = this.audioStreamManager.processAudioChunk(chunk);
|
|
1669
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "realtime");
|
|
1670
|
+
this.sendEvent("realtime_input", message);
|
|
1671
|
+
} catch (error) {
|
|
1672
|
+
this.log("Failed to process audio chunk", error);
|
|
1673
|
+
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to process audio chunk", error);
|
|
1674
|
+
}
|
|
1675
|
+
});
|
|
1676
|
+
stream.on("error", (error) => {
|
|
1677
|
+
this.log("Audio stream error", error);
|
|
1678
|
+
this.createAndEmitError("audio_stream_error" /* AUDIO_STREAM_ERROR */, "Audio stream error", error);
|
|
1679
|
+
});
|
|
1680
|
+
stream.on("end", () => {
|
|
1681
|
+
this.log("Audio stream ended");
|
|
1682
|
+
});
|
|
1683
|
+
} else {
|
|
1684
|
+
const validateAudio = this.audioStreamManager.validateAndConvertAudioInput(audioData);
|
|
1685
|
+
const base64Audio = this.audioStreamManager.int16ArrayToBase64(validateAudio);
|
|
1686
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "realtime");
|
|
1687
|
+
this.sendEvent("realtime_input", message);
|
|
1688
|
+
}
|
|
1689
|
+
}, "gemini-live.send")();
|
|
1684
1690
|
}
|
|
1685
1691
|
/**
|
|
1686
1692
|
* Process speech from audio stream (traditional STT interface)
|
|
1687
1693
|
*/
|
|
1688
1694
|
async listen(audioStream, _options) {
|
|
1689
|
-
this.
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
};
|
|
1697
|
-
const onError = (error) => {
|
|
1698
|
-
throw new Error(`Transcription failed: ${error.message}`);
|
|
1699
|
-
};
|
|
1700
|
-
const onSession = (data) => {
|
|
1701
|
-
if (data.state === "disconnected") {
|
|
1702
|
-
throw new Error("Session disconnected during transcription");
|
|
1703
|
-
}
|
|
1704
|
-
};
|
|
1705
|
-
this.on("writing", onWriting);
|
|
1706
|
-
this.on("error", onError);
|
|
1707
|
-
this.on("session", onSession);
|
|
1708
|
-
try {
|
|
1709
|
-
const result = await this.audioStreamManager.handleAudioTranscription(
|
|
1710
|
-
audioStream,
|
|
1711
|
-
(base64Audio) => {
|
|
1712
|
-
return new Promise((resolve, reject) => {
|
|
1713
|
-
try {
|
|
1714
|
-
const message = this.audioStreamManager.createAudioMessage(base64Audio, "input");
|
|
1715
|
-
const cleanup = () => {
|
|
1716
|
-
this.off("turnComplete", onTurnComplete);
|
|
1717
|
-
this.off("error", onErr);
|
|
1718
|
-
};
|
|
1719
|
-
const onTurnComplete = () => {
|
|
1720
|
-
cleanup();
|
|
1721
|
-
resolve(transcriptionText.trim());
|
|
1722
|
-
};
|
|
1723
|
-
const onErr = (e) => {
|
|
1724
|
-
cleanup();
|
|
1725
|
-
reject(new Error(e.message));
|
|
1726
|
-
};
|
|
1727
|
-
this.on("turnComplete", onTurnComplete);
|
|
1728
|
-
this.on("error", onErr);
|
|
1729
|
-
this.sendEvent("client_content", message);
|
|
1730
|
-
this.log("Sent audio for transcription");
|
|
1731
|
-
} catch (err) {
|
|
1732
|
-
reject(err);
|
|
1733
|
-
}
|
|
1734
|
-
});
|
|
1735
|
-
},
|
|
1736
|
-
(error) => {
|
|
1737
|
-
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Audio transcription failed", error);
|
|
1695
|
+
return this.traced(async () => {
|
|
1696
|
+
this.validateConnectionState();
|
|
1697
|
+
let transcriptionText = "";
|
|
1698
|
+
const onWriting = (data) => {
|
|
1699
|
+
if (data.role === "user") {
|
|
1700
|
+
transcriptionText += data.text;
|
|
1701
|
+
this.log("Received transcription text:", { text: data.text, total: transcriptionText });
|
|
1738
1702
|
}
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1703
|
+
};
|
|
1704
|
+
const onError = (error) => {
|
|
1705
|
+
throw new Error(`Transcription failed: ${error.message}`);
|
|
1706
|
+
};
|
|
1707
|
+
const onSession = (data) => {
|
|
1708
|
+
if (data.state === "disconnected") {
|
|
1709
|
+
throw new Error("Session disconnected during transcription");
|
|
1710
|
+
}
|
|
1711
|
+
};
|
|
1712
|
+
this.on("writing", onWriting);
|
|
1713
|
+
this.on("error", onError);
|
|
1714
|
+
this.on("session", onSession);
|
|
1715
|
+
try {
|
|
1716
|
+
const result = await this.audioStreamManager.handleAudioTranscription(
|
|
1717
|
+
audioStream,
|
|
1718
|
+
(base64Audio) => {
|
|
1719
|
+
return new Promise((resolve, reject) => {
|
|
1720
|
+
try {
|
|
1721
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "input");
|
|
1722
|
+
const cleanup = () => {
|
|
1723
|
+
this.off("turnComplete", onTurnComplete);
|
|
1724
|
+
this.off("error", onErr);
|
|
1725
|
+
};
|
|
1726
|
+
const onTurnComplete = () => {
|
|
1727
|
+
cleanup();
|
|
1728
|
+
resolve(transcriptionText.trim());
|
|
1729
|
+
};
|
|
1730
|
+
const onErr = (e) => {
|
|
1731
|
+
cleanup();
|
|
1732
|
+
reject(new Error(e.message));
|
|
1733
|
+
};
|
|
1734
|
+
this.on("turnComplete", onTurnComplete);
|
|
1735
|
+
this.on("error", onErr);
|
|
1736
|
+
this.sendEvent("client_content", message);
|
|
1737
|
+
this.log("Sent audio for transcription");
|
|
1738
|
+
} catch (err) {
|
|
1739
|
+
reject(err);
|
|
1740
|
+
}
|
|
1741
|
+
});
|
|
1742
|
+
},
|
|
1743
|
+
(error) => {
|
|
1744
|
+
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Audio transcription failed", error);
|
|
1745
|
+
}
|
|
1746
|
+
);
|
|
1747
|
+
return result;
|
|
1748
|
+
} finally {
|
|
1749
|
+
this.off("writing", onWriting);
|
|
1750
|
+
this.off("error", onError);
|
|
1751
|
+
this.off("session", onSession);
|
|
1752
|
+
}
|
|
1753
|
+
}, "gemini-live.listen")();
|
|
1746
1754
|
}
|
|
1747
1755
|
/**
|
|
1748
1756
|
* Get available speakers/voices
|
|
1749
1757
|
*/
|
|
1750
1758
|
async getSpeakers() {
|
|
1751
|
-
return
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1759
|
+
return this.traced(async () => {
|
|
1760
|
+
return [
|
|
1761
|
+
{ voiceId: "Puck", description: "Conversational, friendly" },
|
|
1762
|
+
{ voiceId: "Charon", description: "Deep, authoritative" },
|
|
1763
|
+
{ voiceId: "Kore", description: "Neutral, professional" },
|
|
1764
|
+
{ voiceId: "Fenrir", description: "Warm, approachable" }
|
|
1765
|
+
];
|
|
1766
|
+
}, "gemini-live.getSpeakers")();
|
|
1757
1767
|
}
|
|
1758
1768
|
/**
|
|
1759
1769
|
* Resume a previous session using a session handle
|
|
@@ -2381,7 +2391,13 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
2381
2391
|
let result;
|
|
2382
2392
|
if (tool.execute) {
|
|
2383
2393
|
this.log("Executing tool", { toolName, toolArgs });
|
|
2384
|
-
result = await tool.execute(
|
|
2394
|
+
result = await tool.execute(
|
|
2395
|
+
{ context: toolArgs, runtimeContext: this.runtimeContext },
|
|
2396
|
+
{
|
|
2397
|
+
toolCallId: toolId,
|
|
2398
|
+
messages: []
|
|
2399
|
+
}
|
|
2400
|
+
);
|
|
2385
2401
|
this.log("Tool executed successfully", { toolName, result });
|
|
2386
2402
|
} else {
|
|
2387
2403
|
this.log("Tool has no execute function", { toolName });
|
|
@@ -2683,14 +2699,14 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
2683
2699
|
* inputSchema: z.object({
|
|
2684
2700
|
* location: z.string().describe("The city and state, e.g. San Francisco, CA"),
|
|
2685
2701
|
* }),
|
|
2686
|
-
* execute: async (
|
|
2702
|
+
* execute: async ({ context }) => {
|
|
2687
2703
|
* // Fetch weather data from an API
|
|
2688
2704
|
* const response = await fetch(
|
|
2689
|
-
* `https://api.weather.com?location=${encodeURIComponent(
|
|
2705
|
+
* `https://api.weather.com?location=${encodeURIComponent(context.location)}`,
|
|
2690
2706
|
* );
|
|
2691
2707
|
* const data = await response.json();
|
|
2692
2708
|
* return {
|
|
2693
|
-
* message: `The current temperature in ${
|
|
2709
|
+
* message: `The current temperature in ${context.location} is ${data.temperature}°F with ${data.conditions}.`,
|
|
2694
2710
|
* };
|
|
2695
2711
|
* },
|
|
2696
2712
|
* });
|
|
@@ -2708,7 +2724,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
2708
2724
|
* Get the current tools configured for this voice instance
|
|
2709
2725
|
* @returns Object containing the current tools
|
|
2710
2726
|
*/
|
|
2711
|
-
|
|
2727
|
+
getTools() {
|
|
2712
2728
|
return this.tools;
|
|
2713
2729
|
}
|
|
2714
2730
|
log(message, ...args) {
|