@mastra/voice-google-gemini-live 0.11.0-beta.1 → 0.11.1-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -58
- package/README.md +3 -3
- package/dist/index.cjs +238 -196
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +17 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +238 -196
- package/dist/index.js.map +1 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +13 -11
package/dist/index.d.ts
CHANGED
|
@@ -74,7 +74,7 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
74
74
|
private isResuming;
|
|
75
75
|
private sessionDurationTimeout?;
|
|
76
76
|
private tools?;
|
|
77
|
-
private
|
|
77
|
+
private runtimeContext?;
|
|
78
78
|
private options;
|
|
79
79
|
/**
|
|
80
80
|
* Normalize configuration to ensure proper VoiceConfig format
|
|
@@ -162,8 +162,8 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
162
162
|
/**
|
|
163
163
|
* Establish connection to the Gemini Live API
|
|
164
164
|
*/
|
|
165
|
-
connect({
|
|
166
|
-
|
|
165
|
+
connect({ runtimeContext }?: {
|
|
166
|
+
runtimeContext?: any;
|
|
167
167
|
}): Promise<void>;
|
|
168
168
|
/**
|
|
169
169
|
* Disconnect from the Gemini Live API
|
|
@@ -345,6 +345,16 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
345
345
|
* @private
|
|
346
346
|
*/
|
|
347
347
|
private determineModality;
|
|
348
|
+
/**
|
|
349
|
+
* Resolve Vertex AI location with sensible default
|
|
350
|
+
* @private
|
|
351
|
+
*/
|
|
352
|
+
private getVertexLocation;
|
|
353
|
+
/**
|
|
354
|
+
* Resolve the correct model identifier for Gemini API or Vertex AI
|
|
355
|
+
* @private
|
|
356
|
+
*/
|
|
357
|
+
private resolveModelIdentifier;
|
|
348
358
|
/**
|
|
349
359
|
* Send initial configuration to Gemini Live API
|
|
350
360
|
* @private
|
|
@@ -390,14 +400,14 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
390
400
|
* inputSchema: z.object({
|
|
391
401
|
* location: z.string().describe("The city and state, e.g. San Francisco, CA"),
|
|
392
402
|
* }),
|
|
393
|
-
* execute: async (
|
|
403
|
+
* execute: async ({ context }) => {
|
|
394
404
|
* // Fetch weather data from an API
|
|
395
405
|
* const response = await fetch(
|
|
396
|
-
* `https://api.weather.com?location=${encodeURIComponent(
|
|
406
|
+
* `https://api.weather.com?location=${encodeURIComponent(context.location)}`,
|
|
397
407
|
* );
|
|
398
408
|
* const data = await response.json();
|
|
399
409
|
* return {
|
|
400
|
-
* message: `The current temperature in ${
|
|
410
|
+
* message: `The current temperature in ${context.location} is ${data.temperature}°F with ${data.conditions}.`,
|
|
401
411
|
* };
|
|
402
412
|
* },
|
|
403
413
|
* });
|
|
@@ -412,7 +422,7 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
412
422
|
* Get the current tools configured for this voice instance
|
|
413
423
|
* @returns Object containing the current tools
|
|
414
424
|
*/
|
|
415
|
-
|
|
425
|
+
getTools(): ToolsInput | undefined;
|
|
416
426
|
private log;
|
|
417
427
|
/**
|
|
418
428
|
* Convert Zod schema to JSON Schema for tool parameters
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAKtE,OAAO,KAAK,EACV,qBAAqB,EACrB,sBAAsB,EACtB,kBAAkB,EAMlB,mBAAmB,EAEpB,MAAM,SAAS,CAAC;AAYjB;;GAEG;AAGH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,qBAAa,eAAgB,SAAQ,WAAW,CAC9C,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,UAAU,EACV,kBAAkB,CACnB;IACC,OAAO,CAAC,EAAE,CAAC,CAAS;IACpB,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,KAAK,CAAgD;IAC7D,OAAO,CAAC,aAAa,CAAC,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAU;IAChC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,KAAK,CAAiB;IAG9B,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,WAAW,CAAc;IAGjC,OAAO,CAAC,kBAAkB,CAAqB;IAG/C,OAAO,CAAC,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,gBAAgB,CAAC,CAAS;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,sBAAsB,CAAC,CAAiB;IAGhD,OAAO,CAAC,KAAK,CAAC,CAAa;IAC3B,OAAO,CAAC,cAAc,CAAC,CAAM;IAG7B,OAAO,CAAC,OAAO,CAAwB;IAEvC;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,eAAe;IAwB9B;;;;OAIG;gBACS,MAAM,GAAE,WAAW,CAAC,qBAAqB,CAAC,GAAG,qBAA0B;IA0DnF;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,EAAE,CAAC,CAAC,SAAS,cAAc,EACzB,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,cAAc,EAC1B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IASP;;;;OAIG;IACH,IAAI,CAAC,CAAC,SAAS,cAAc,EAC3B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;OAGG;IACH,OAAO,CAAC,IAAI;IAoCZ;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAoB7B;;;OAGG;IACH,oBAAoB,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAS9C;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAO1B;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAU/B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAU9B;;OAEG;IACG,OAAO,CAAC,EAAE,cAAc,EAAE,GAAE;QAAE,cAAc,CAAC,EAAE,GAAG,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAKtE,OAAO,KAAK,EACV,qBAAqB,EACrB,sBAAsB,EACtB,kBAAkB,EAMlB,mBAAmB,EAEpB,MAAM,SAAS,CAAC;AAYjB;;GAEG;AAGH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,qBAAa,eAAgB,SAAQ,WAAW,CAC9C,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,UAAU,EACV,kBAAkB,CACnB;IACC,OAAO,CAAC,EAAE,CAAC,CAAS;IACpB,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,KAAK,CAAgD;IAC7D,OAAO,CAAC,aAAa,CAAC,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAU;IAChC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,KAAK,CAAiB;IAG9B,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,WAAW,CAAc;IAGjC,OAAO,CAAC,kBAAkB,CAAqB;IAG/C,OAAO,CAAC,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,gBAAgB,CAAC,CAAS;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,sBAAsB,CAAC,CAAiB;IAGhD,OAAO,CAAC,KAAK,CAAC,CAAa;IAC3B,OAAO,CAAC,cAAc,CAAC,CAAM;IAG7B,OAAO,CAAC,OAAO,CAAwB;IAEvC;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,eAAe;IAwB9B;;;;OAIG;gBACS,MAAM,GAAE,WAAW,CAAC,qBAAqB,CAAC,GAAG,qBAA0B;IA0DnF;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,EAAE,CAAC,CAAC,SAAS,cAAc,EACzB,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,cAAc,EAC1B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IASP;;;;OAIG;IACH,IAAI,CAAC,CAAC,SAAS,cAAc,EAC3B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;OAGG;IACH,OAAO,CAAC,IAAI;IAoCZ;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAoB7B;;;OAGG;IACH,oBAAoB,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAS9C;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAO1B;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAU/B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAU9B;;OAEG;IACG,OAAO,CAAC,EAAE,cAAc,EAAE,GAAE;QAAE,cAAc,CAAC,EAAE,GAAG,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IA0F/E;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAiDjC;;OAEG;IACG,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,cAAc,EAAE,OAAO,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC;IA2EnG;;OAEG;IACG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,cAAc,GAAG,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAmCxE;;OAEG;IACG,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,cAAc,EAAE,QAAQ,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,MAAM,CAAC;IAuFpG;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAY9E;;OAEG;IACG,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BtG;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACG,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA6MhF;;OAEG;IACH,kBAAkB,IAAI,cAAc,GAAG,WAAW;IAIlD;;OAEG;IACH,WAAW,IAAI,OAAO;IAItB;;;OAGG;IACH,uBAAuB,IAAI,MAAM,CAAC,cAAc,GAAG,IAAI;IAIvD;;OAEG;IACH,gBAAgB,IAAI,MAAM,GAAG,SAAS;IAKtC;;OAEG;IACH,cAAc,IAAI;QAChB,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,IAAI,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,mBAAmB,CAAC;QAC7B,WAAW,EAAE,MAAM,CAAC;KACrB;IAYD;;OAEG;IACH,iBAAiB,IAAI,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IAIhF;;OAEG;IACH,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI;IAI/D;;OAEG;IACH,YAAY,IAAI,IAAI;IAKpB;;OAEG;IACH,gBAAgB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAQxC;;;OAGG;YACW,qBAAqB;IA4BnC;;;OAGG;IACH,OAAO,CAAC,2BAA2B;IAqCnC;;;OAGG;IACH,OAAO,CAAC,aAAa;IAmBrB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAKvB;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IA6C3B;;;OAGG;YACW,mBAAmB;IA0EjC;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAwB3B;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAY5B;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAsI3B;;;OAGG;YACW,cAAc;IA4B5B;;;OAGG;YACW,qBAAqB;IAoFnC;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAWzB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAMxB;;;OAGG;IACH,OAAO,CAAC,WAAW;IAcnB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAYzB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAIzB;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAkB9B;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAmIzB;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAqD7B;;;;OAIG;YACW,cAAc;IAO5B;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAI5B;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAI5B;;;OAGG;IACH,OAAO,CAAC,SAAS;IAkCjB;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,QAAQ,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAKjC;;;OAGG;IACH,QAAQ,IAAI,UAAU,GAAG,SAAS;IAIlC,OAAO,CAAC,GAAG;IAMX;;;OAGG;IACH,OAAO,CAAC,4BAA4B;IAiCpC;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IA2DjC;;OAEG;IACH,KAAK,IAAI,IAAI;IAIb;;OAEG;IACG,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAO/D;;;OAGG;IACH,eAAe,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI;CAM7C"}
|
package/dist/index.js
CHANGED
|
@@ -1260,7 +1260,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
1260
1260
|
sessionDurationTimeout;
|
|
1261
1261
|
// Tool integration properties
|
|
1262
1262
|
tools;
|
|
1263
|
-
|
|
1263
|
+
runtimeContext;
|
|
1264
1264
|
// Store the configuration options
|
|
1265
1265
|
options;
|
|
1266
1266
|
/**
|
|
@@ -1497,67 +1497,70 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
1497
1497
|
/**
|
|
1498
1498
|
* Establish connection to the Gemini Live API
|
|
1499
1499
|
*/
|
|
1500
|
-
async connect({
|
|
1501
|
-
|
|
1502
|
-
this.
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
this.requestContext = requestContext;
|
|
1506
|
-
this.emit("session", { state: "connecting" });
|
|
1507
|
-
try {
|
|
1508
|
-
let wsUrl;
|
|
1509
|
-
let headers = {};
|
|
1510
|
-
if (this.options.vertexAI) {
|
|
1511
|
-
wsUrl = `wss://${this.options.location}-aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1beta1.PredictionService.ServerStreamingPredict`;
|
|
1512
|
-
await this.authManager.initialize();
|
|
1513
|
-
const accessToken = await this.authManager.getAccessToken();
|
|
1514
|
-
headers = { headers: { Authorization: `Bearer ${accessToken}` } };
|
|
1515
|
-
this.log("Using Vertex AI authentication with OAuth token");
|
|
1516
|
-
} else {
|
|
1517
|
-
wsUrl = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent`;
|
|
1518
|
-
headers = {
|
|
1519
|
-
headers: {
|
|
1520
|
-
"x-goog-api-key": this.options.apiKey || "",
|
|
1521
|
-
"Content-Type": "application/json"
|
|
1522
|
-
}
|
|
1523
|
-
};
|
|
1524
|
-
this.log("Using Live API authentication with API key");
|
|
1525
|
-
}
|
|
1526
|
-
this.log("Connecting to:", wsUrl);
|
|
1527
|
-
this.ws = new WebSocket(wsUrl, void 0, headers);
|
|
1528
|
-
this.connectionManager.setWebSocket(this.ws);
|
|
1529
|
-
this.setupEventListeners();
|
|
1530
|
-
await this.connectionManager.waitForOpen();
|
|
1531
|
-
if (this.isResuming && this.sessionHandle) {
|
|
1532
|
-
await this.sendSessionResumption();
|
|
1533
|
-
} else {
|
|
1534
|
-
this.sendInitialConfig();
|
|
1535
|
-
this.sessionStartTime = Date.now();
|
|
1536
|
-
this.sessionId = randomUUID();
|
|
1500
|
+
async connect({ runtimeContext } = {}) {
|
|
1501
|
+
return this.traced(async () => {
|
|
1502
|
+
if (this.state === "connected") {
|
|
1503
|
+
this.log("Already connected to Gemini Live API");
|
|
1504
|
+
return;
|
|
1537
1505
|
}
|
|
1538
|
-
|
|
1539
|
-
this.state
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1506
|
+
this.runtimeContext = runtimeContext;
|
|
1507
|
+
this.emit("session", { state: "connecting" });
|
|
1508
|
+
try {
|
|
1509
|
+
let wsUrl;
|
|
1510
|
+
let headers = {};
|
|
1511
|
+
if (this.options.vertexAI) {
|
|
1512
|
+
const location = this.getVertexLocation();
|
|
1513
|
+
wsUrl = `wss://${location}-aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1beta1.LlmBidiService/BidiGenerateContent`;
|
|
1514
|
+
await this.authManager.initialize();
|
|
1515
|
+
const accessToken = await this.authManager.getAccessToken();
|
|
1516
|
+
headers = { headers: { Authorization: `Bearer ${accessToken}` } };
|
|
1517
|
+
this.log("Using Vertex AI authentication with OAuth token");
|
|
1518
|
+
} else {
|
|
1519
|
+
wsUrl = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent`;
|
|
1520
|
+
headers = {
|
|
1521
|
+
headers: {
|
|
1522
|
+
"x-goog-api-key": this.options.apiKey || "",
|
|
1523
|
+
"Content-Type": "application/json"
|
|
1524
|
+
}
|
|
1525
|
+
};
|
|
1526
|
+
this.log("Using Live API authentication with API key");
|
|
1527
|
+
}
|
|
1528
|
+
this.log("Connecting to:", wsUrl);
|
|
1529
|
+
this.ws = new WebSocket(wsUrl, void 0, headers);
|
|
1530
|
+
this.connectionManager.setWebSocket(this.ws);
|
|
1531
|
+
this.setupEventListeners();
|
|
1532
|
+
await this.connectionManager.waitForOpen();
|
|
1533
|
+
if (this.isResuming && this.sessionHandle) {
|
|
1534
|
+
await this.sendSessionResumption();
|
|
1535
|
+
} else {
|
|
1536
|
+
this.sendInitialConfig();
|
|
1537
|
+
this.sessionStartTime = Date.now();
|
|
1538
|
+
this.sessionId = randomUUID();
|
|
1539
|
+
}
|
|
1540
|
+
await this.waitForSessionCreated();
|
|
1541
|
+
this.state = "connected";
|
|
1542
|
+
this.emit("session", {
|
|
1543
|
+
state: "connected",
|
|
1544
|
+
config: {
|
|
1545
|
+
sessionId: this.sessionId,
|
|
1546
|
+
isResuming: this.isResuming,
|
|
1547
|
+
toolCount: Object.keys(this.tools || {}).length
|
|
1548
|
+
}
|
|
1549
|
+
});
|
|
1550
|
+
this.log("Successfully connected to Gemini Live API", {
|
|
1543
1551
|
sessionId: this.sessionId,
|
|
1544
1552
|
isResuming: this.isResuming,
|
|
1545
1553
|
toolCount: Object.keys(this.tools || {}).length
|
|
1554
|
+
});
|
|
1555
|
+
if (this.options.sessionConfig?.maxDuration) {
|
|
1556
|
+
this.startSessionDurationMonitor();
|
|
1546
1557
|
}
|
|
1547
|
-
})
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
toolCount: Object.keys(this.tools || {}).length
|
|
1552
|
-
});
|
|
1553
|
-
if (this.options.sessionConfig?.maxDuration) {
|
|
1554
|
-
this.startSessionDurationMonitor();
|
|
1558
|
+
} catch (error) {
|
|
1559
|
+
this.state = "disconnected";
|
|
1560
|
+
this.log("Connection failed", error);
|
|
1561
|
+
throw error;
|
|
1555
1562
|
}
|
|
1556
|
-
}
|
|
1557
|
-
this.state = "disconnected";
|
|
1558
|
-
this.log("Connection failed", error);
|
|
1559
|
-
throw error;
|
|
1560
|
-
}
|
|
1563
|
+
}, "gemini-live.connect")();
|
|
1561
1564
|
}
|
|
1562
1565
|
/**
|
|
1563
1566
|
* Disconnect from the Gemini Live API
|
|
@@ -1595,164 +1598,172 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
1595
1598
|
* Send text to be converted to speech
|
|
1596
1599
|
*/
|
|
1597
1600
|
async speak(input, options) {
|
|
1598
|
-
this.
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1601
|
+
return this.traced(async () => {
|
|
1602
|
+
this.validateConnectionState();
|
|
1603
|
+
if (typeof input !== "string") {
|
|
1604
|
+
const chunks = [];
|
|
1605
|
+
for await (const chunk of input) {
|
|
1606
|
+
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)));
|
|
1607
|
+
}
|
|
1608
|
+
input = Buffer.concat(chunks).toString("utf-8");
|
|
1603
1609
|
}
|
|
1604
|
-
input
|
|
1605
|
-
|
|
1606
|
-
if (input.trim().length === 0) {
|
|
1607
|
-
throw this.createAndEmitError("invalid_audio_format" /* INVALID_AUDIO_FORMAT */, "Input text is empty");
|
|
1608
|
-
}
|
|
1609
|
-
this.addToContext("user", input);
|
|
1610
|
-
const textMessage = {
|
|
1611
|
-
client_content: {
|
|
1612
|
-
turns: [
|
|
1613
|
-
{
|
|
1614
|
-
role: "user",
|
|
1615
|
-
parts: [
|
|
1616
|
-
{
|
|
1617
|
-
text: input
|
|
1618
|
-
}
|
|
1619
|
-
]
|
|
1620
|
-
}
|
|
1621
|
-
],
|
|
1622
|
-
turnComplete: true
|
|
1610
|
+
if (input.trim().length === 0) {
|
|
1611
|
+
throw this.createAndEmitError("invalid_audio_format" /* INVALID_AUDIO_FORMAT */, "Input text is empty");
|
|
1623
1612
|
}
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1613
|
+
this.addToContext("user", input);
|
|
1614
|
+
const textMessage = {
|
|
1615
|
+
client_content: {
|
|
1616
|
+
turns: [
|
|
1617
|
+
{
|
|
1618
|
+
role: "user",
|
|
1619
|
+
parts: [
|
|
1620
|
+
{
|
|
1621
|
+
text: input
|
|
1622
|
+
}
|
|
1623
|
+
]
|
|
1634
1624
|
}
|
|
1635
|
-
|
|
1625
|
+
],
|
|
1626
|
+
turnComplete: true
|
|
1636
1627
|
}
|
|
1637
1628
|
};
|
|
1629
|
+
if (options && (options.speaker || options.languageCode || options.responseModalities)) {
|
|
1630
|
+
const updateMessage = {
|
|
1631
|
+
type: "session.update",
|
|
1632
|
+
session: {
|
|
1633
|
+
generation_config: {
|
|
1634
|
+
...options.responseModalities ? { response_modalities: options.responseModalities } : {},
|
|
1635
|
+
speech_config: {
|
|
1636
|
+
...options.languageCode ? { language_code: options.languageCode } : {},
|
|
1637
|
+
...options.speaker ? { voice_config: { prebuilt_voice_config: { voice_name: options.speaker } } } : {}
|
|
1638
|
+
}
|
|
1639
|
+
}
|
|
1640
|
+
}
|
|
1641
|
+
};
|
|
1642
|
+
try {
|
|
1643
|
+
this.sendEvent("session.update", updateMessage);
|
|
1644
|
+
this.log("Applied per-turn runtime options", options);
|
|
1645
|
+
} catch (error) {
|
|
1646
|
+
this.log("Failed to apply per-turn runtime options", error);
|
|
1647
|
+
}
|
|
1648
|
+
}
|
|
1638
1649
|
try {
|
|
1639
|
-
this.sendEvent("
|
|
1640
|
-
this.log("
|
|
1650
|
+
this.sendEvent("client_content", textMessage);
|
|
1651
|
+
this.log("Text message sent", { text: input });
|
|
1641
1652
|
} catch (error) {
|
|
1642
|
-
this.log("Failed to
|
|
1653
|
+
this.log("Failed to send text message", error);
|
|
1654
|
+
throw this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to send text message", error);
|
|
1643
1655
|
}
|
|
1644
|
-
}
|
|
1645
|
-
try {
|
|
1646
|
-
this.sendEvent("client_content", textMessage);
|
|
1647
|
-
this.log("Text message sent", { text: input });
|
|
1648
|
-
} catch (error) {
|
|
1649
|
-
this.log("Failed to send text message", error);
|
|
1650
|
-
throw this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to send text message", error);
|
|
1651
|
-
}
|
|
1656
|
+
}, "gemini-live.speak")();
|
|
1652
1657
|
}
|
|
1653
1658
|
/**
|
|
1654
1659
|
* Send audio stream for processing
|
|
1655
1660
|
*/
|
|
1656
1661
|
async send(audioData) {
|
|
1657
|
-
this.
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1662
|
+
return this.traced(async () => {
|
|
1663
|
+
this.validateConnectionState();
|
|
1664
|
+
if ("readable" in audioData && typeof audioData.on === "function") {
|
|
1665
|
+
const stream = audioData;
|
|
1666
|
+
stream.on("data", (chunk) => {
|
|
1667
|
+
try {
|
|
1668
|
+
const base64Audio = this.audioStreamManager.processAudioChunk(chunk);
|
|
1669
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "realtime");
|
|
1670
|
+
this.sendEvent("realtime_input", message);
|
|
1671
|
+
} catch (error) {
|
|
1672
|
+
this.log("Failed to process audio chunk", error);
|
|
1673
|
+
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to process audio chunk", error);
|
|
1674
|
+
}
|
|
1675
|
+
});
|
|
1676
|
+
stream.on("error", (error) => {
|
|
1677
|
+
this.log("Audio stream error", error);
|
|
1678
|
+
this.createAndEmitError("audio_stream_error" /* AUDIO_STREAM_ERROR */, "Audio stream error", error);
|
|
1679
|
+
});
|
|
1680
|
+
stream.on("end", () => {
|
|
1681
|
+
this.log("Audio stream ended");
|
|
1682
|
+
});
|
|
1683
|
+
} else {
|
|
1684
|
+
const validateAudio = this.audioStreamManager.validateAndConvertAudioInput(audioData);
|
|
1685
|
+
const base64Audio = this.audioStreamManager.int16ArrayToBase64(validateAudio);
|
|
1686
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "realtime");
|
|
1687
|
+
this.sendEvent("realtime_input", message);
|
|
1688
|
+
}
|
|
1689
|
+
}, "gemini-live.send")();
|
|
1683
1690
|
}
|
|
1684
1691
|
/**
|
|
1685
1692
|
* Process speech from audio stream (traditional STT interface)
|
|
1686
1693
|
*/
|
|
1687
1694
|
async listen(audioStream, _options) {
|
|
1688
|
-
this.
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
};
|
|
1696
|
-
const onError = (error) => {
|
|
1697
|
-
throw new Error(`Transcription failed: ${error.message}`);
|
|
1698
|
-
};
|
|
1699
|
-
const onSession = (data) => {
|
|
1700
|
-
if (data.state === "disconnected") {
|
|
1701
|
-
throw new Error("Session disconnected during transcription");
|
|
1702
|
-
}
|
|
1703
|
-
};
|
|
1704
|
-
this.on("writing", onWriting);
|
|
1705
|
-
this.on("error", onError);
|
|
1706
|
-
this.on("session", onSession);
|
|
1707
|
-
try {
|
|
1708
|
-
const result = await this.audioStreamManager.handleAudioTranscription(
|
|
1709
|
-
audioStream,
|
|
1710
|
-
(base64Audio) => {
|
|
1711
|
-
return new Promise((resolve, reject) => {
|
|
1712
|
-
try {
|
|
1713
|
-
const message = this.audioStreamManager.createAudioMessage(base64Audio, "input");
|
|
1714
|
-
const cleanup = () => {
|
|
1715
|
-
this.off("turnComplete", onTurnComplete);
|
|
1716
|
-
this.off("error", onErr);
|
|
1717
|
-
};
|
|
1718
|
-
const onTurnComplete = () => {
|
|
1719
|
-
cleanup();
|
|
1720
|
-
resolve(transcriptionText.trim());
|
|
1721
|
-
};
|
|
1722
|
-
const onErr = (e) => {
|
|
1723
|
-
cleanup();
|
|
1724
|
-
reject(new Error(e.message));
|
|
1725
|
-
};
|
|
1726
|
-
this.on("turnComplete", onTurnComplete);
|
|
1727
|
-
this.on("error", onErr);
|
|
1728
|
-
this.sendEvent("client_content", message);
|
|
1729
|
-
this.log("Sent audio for transcription");
|
|
1730
|
-
} catch (err) {
|
|
1731
|
-
reject(err);
|
|
1732
|
-
}
|
|
1733
|
-
});
|
|
1734
|
-
},
|
|
1735
|
-
(error) => {
|
|
1736
|
-
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Audio transcription failed", error);
|
|
1695
|
+
return this.traced(async () => {
|
|
1696
|
+
this.validateConnectionState();
|
|
1697
|
+
let transcriptionText = "";
|
|
1698
|
+
const onWriting = (data) => {
|
|
1699
|
+
if (data.role === "user") {
|
|
1700
|
+
transcriptionText += data.text;
|
|
1701
|
+
this.log("Received transcription text:", { text: data.text, total: transcriptionText });
|
|
1737
1702
|
}
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1703
|
+
};
|
|
1704
|
+
const onError = (error) => {
|
|
1705
|
+
throw new Error(`Transcription failed: ${error.message}`);
|
|
1706
|
+
};
|
|
1707
|
+
const onSession = (data) => {
|
|
1708
|
+
if (data.state === "disconnected") {
|
|
1709
|
+
throw new Error("Session disconnected during transcription");
|
|
1710
|
+
}
|
|
1711
|
+
};
|
|
1712
|
+
this.on("writing", onWriting);
|
|
1713
|
+
this.on("error", onError);
|
|
1714
|
+
this.on("session", onSession);
|
|
1715
|
+
try {
|
|
1716
|
+
const result = await this.audioStreamManager.handleAudioTranscription(
|
|
1717
|
+
audioStream,
|
|
1718
|
+
(base64Audio) => {
|
|
1719
|
+
return new Promise((resolve, reject) => {
|
|
1720
|
+
try {
|
|
1721
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "input");
|
|
1722
|
+
const cleanup = () => {
|
|
1723
|
+
this.off("turnComplete", onTurnComplete);
|
|
1724
|
+
this.off("error", onErr);
|
|
1725
|
+
};
|
|
1726
|
+
const onTurnComplete = () => {
|
|
1727
|
+
cleanup();
|
|
1728
|
+
resolve(transcriptionText.trim());
|
|
1729
|
+
};
|
|
1730
|
+
const onErr = (e) => {
|
|
1731
|
+
cleanup();
|
|
1732
|
+
reject(new Error(e.message));
|
|
1733
|
+
};
|
|
1734
|
+
this.on("turnComplete", onTurnComplete);
|
|
1735
|
+
this.on("error", onErr);
|
|
1736
|
+
this.sendEvent("client_content", message);
|
|
1737
|
+
this.log("Sent audio for transcription");
|
|
1738
|
+
} catch (err) {
|
|
1739
|
+
reject(err);
|
|
1740
|
+
}
|
|
1741
|
+
});
|
|
1742
|
+
},
|
|
1743
|
+
(error) => {
|
|
1744
|
+
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Audio transcription failed", error);
|
|
1745
|
+
}
|
|
1746
|
+
);
|
|
1747
|
+
return result;
|
|
1748
|
+
} finally {
|
|
1749
|
+
this.off("writing", onWriting);
|
|
1750
|
+
this.off("error", onError);
|
|
1751
|
+
this.off("session", onSession);
|
|
1752
|
+
}
|
|
1753
|
+
}, "gemini-live.listen")();
|
|
1745
1754
|
}
|
|
1746
1755
|
/**
|
|
1747
1756
|
* Get available speakers/voices
|
|
1748
1757
|
*/
|
|
1749
1758
|
async getSpeakers() {
|
|
1750
|
-
return
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1759
|
+
return this.traced(async () => {
|
|
1760
|
+
return [
|
|
1761
|
+
{ voiceId: "Puck", description: "Conversational, friendly" },
|
|
1762
|
+
{ voiceId: "Charon", description: "Deep, authoritative" },
|
|
1763
|
+
{ voiceId: "Kore", description: "Neutral, professional" },
|
|
1764
|
+
{ voiceId: "Fenrir", description: "Warm, approachable" }
|
|
1765
|
+
];
|
|
1766
|
+
}, "gemini-live.getSpeakers")();
|
|
1756
1767
|
}
|
|
1757
1768
|
/**
|
|
1758
1769
|
* Resume a previous session using a session handle
|
|
@@ -2380,7 +2391,13 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
2380
2391
|
let result;
|
|
2381
2392
|
if (tool.execute) {
|
|
2382
2393
|
this.log("Executing tool", { toolName, toolArgs });
|
|
2383
|
-
result = await tool.execute(
|
|
2394
|
+
result = await tool.execute(
|
|
2395
|
+
{ context: toolArgs, runtimeContext: this.runtimeContext },
|
|
2396
|
+
{
|
|
2397
|
+
toolCallId: toolId,
|
|
2398
|
+
messages: []
|
|
2399
|
+
}
|
|
2400
|
+
);
|
|
2384
2401
|
this.log("Tool executed successfully", { toolName, result });
|
|
2385
2402
|
} else {
|
|
2386
2403
|
this.log("Tool has no execute function", { toolName });
|
|
@@ -2471,6 +2488,31 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
2471
2488
|
}
|
|
2472
2489
|
return "text";
|
|
2473
2490
|
}
|
|
2491
|
+
/**
|
|
2492
|
+
* Resolve Vertex AI location with sensible default
|
|
2493
|
+
* @private
|
|
2494
|
+
*/
|
|
2495
|
+
getVertexLocation() {
|
|
2496
|
+
return this.options.location?.trim() || "us-central1";
|
|
2497
|
+
}
|
|
2498
|
+
/**
|
|
2499
|
+
* Resolve the correct model identifier for Gemini API or Vertex AI
|
|
2500
|
+
* @private
|
|
2501
|
+
*/
|
|
2502
|
+
resolveModelIdentifier() {
|
|
2503
|
+
const model = this.options.model ?? DEFAULT_MODEL;
|
|
2504
|
+
if (!this.options.vertexAI) {
|
|
2505
|
+
return `models/${model}`;
|
|
2506
|
+
}
|
|
2507
|
+
if (!this.options.project) {
|
|
2508
|
+
throw this.createAndEmitError(
|
|
2509
|
+
"project_id_missing" /* PROJECT_ID_MISSING */,
|
|
2510
|
+
"Google Cloud project ID is required when using Vertex AI."
|
|
2511
|
+
);
|
|
2512
|
+
}
|
|
2513
|
+
const location = this.getVertexLocation();
|
|
2514
|
+
return `projects/${this.options.project}/locations/${location}/publishers/google/models/${model}`;
|
|
2515
|
+
}
|
|
2474
2516
|
/**
|
|
2475
2517
|
* Send initial configuration to Gemini Live API
|
|
2476
2518
|
* @private
|
|
@@ -2481,7 +2523,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
2481
2523
|
}
|
|
2482
2524
|
const setupMessage = {
|
|
2483
2525
|
setup: {
|
|
2484
|
-
model:
|
|
2526
|
+
model: this.resolveModelIdentifier()
|
|
2485
2527
|
}
|
|
2486
2528
|
};
|
|
2487
2529
|
if (this.options.instructions) {
|
|
@@ -2657,14 +2699,14 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
2657
2699
|
* inputSchema: z.object({
|
|
2658
2700
|
* location: z.string().describe("The city and state, e.g. San Francisco, CA"),
|
|
2659
2701
|
* }),
|
|
2660
|
-
* execute: async (
|
|
2702
|
+
* execute: async ({ context }) => {
|
|
2661
2703
|
* // Fetch weather data from an API
|
|
2662
2704
|
* const response = await fetch(
|
|
2663
|
-
* `https://api.weather.com?location=${encodeURIComponent(
|
|
2705
|
+
* `https://api.weather.com?location=${encodeURIComponent(context.location)}`,
|
|
2664
2706
|
* );
|
|
2665
2707
|
* const data = await response.json();
|
|
2666
2708
|
* return {
|
|
2667
|
-
* message: `The current temperature in ${
|
|
2709
|
+
* message: `The current temperature in ${context.location} is ${data.temperature}°F with ${data.conditions}.`,
|
|
2668
2710
|
* };
|
|
2669
2711
|
* },
|
|
2670
2712
|
* });
|
|
@@ -2682,7 +2724,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
2682
2724
|
* Get the current tools configured for this voice instance
|
|
2683
2725
|
* @returns Object containing the current tools
|
|
2684
2726
|
*/
|
|
2685
|
-
|
|
2727
|
+
getTools() {
|
|
2686
2728
|
return this.tools;
|
|
2687
2729
|
}
|
|
2688
2730
|
log(message, ...args) {
|