@mastra/voice-google-gemini-live 0.11.0-beta.1 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -58
- package/README.md +3 -3
- package/dist/index.cjs +211 -195
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +7 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +211 -195
- package/dist/index.js.map +1 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +13 -11
package/dist/index.d.ts
CHANGED
|
@@ -74,7 +74,7 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
74
74
|
private isResuming;
|
|
75
75
|
private sessionDurationTimeout?;
|
|
76
76
|
private tools?;
|
|
77
|
-
private
|
|
77
|
+
private runtimeContext?;
|
|
78
78
|
private options;
|
|
79
79
|
/**
|
|
80
80
|
* Normalize configuration to ensure proper VoiceConfig format
|
|
@@ -162,8 +162,8 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
162
162
|
/**
|
|
163
163
|
* Establish connection to the Gemini Live API
|
|
164
164
|
*/
|
|
165
|
-
connect({
|
|
166
|
-
|
|
165
|
+
connect({ runtimeContext }?: {
|
|
166
|
+
runtimeContext?: any;
|
|
167
167
|
}): Promise<void>;
|
|
168
168
|
/**
|
|
169
169
|
* Disconnect from the Gemini Live API
|
|
@@ -390,14 +390,14 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
390
390
|
* inputSchema: z.object({
|
|
391
391
|
* location: z.string().describe("The city and state, e.g. San Francisco, CA"),
|
|
392
392
|
* }),
|
|
393
|
-
* execute: async (
|
|
393
|
+
* execute: async ({ context }) => {
|
|
394
394
|
* // Fetch weather data from an API
|
|
395
395
|
* const response = await fetch(
|
|
396
|
-
* `https://api.weather.com?location=${encodeURIComponent(
|
|
396
|
+
* `https://api.weather.com?location=${encodeURIComponent(context.location)}`,
|
|
397
397
|
* );
|
|
398
398
|
* const data = await response.json();
|
|
399
399
|
* return {
|
|
400
|
-
* message: `The current temperature in ${
|
|
400
|
+
* message: `The current temperature in ${context.location} is ${data.temperature}°F with ${data.conditions}.`,
|
|
401
401
|
* };
|
|
402
402
|
* },
|
|
403
403
|
* });
|
|
@@ -412,7 +412,7 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
412
412
|
* Get the current tools configured for this voice instance
|
|
413
413
|
* @returns Object containing the current tools
|
|
414
414
|
*/
|
|
415
|
-
|
|
415
|
+
getTools(): ToolsInput | undefined;
|
|
416
416
|
private log;
|
|
417
417
|
/**
|
|
418
418
|
* Convert Zod schema to JSON Schema for tool parameters
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAKtE,OAAO,KAAK,EACV,qBAAqB,EACrB,sBAAsB,EACtB,kBAAkB,EAMlB,mBAAmB,EAEpB,MAAM,SAAS,CAAC;AAYjB;;GAEG;AAGH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,qBAAa,eAAgB,SAAQ,WAAW,CAC9C,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,UAAU,EACV,kBAAkB,CACnB;IACC,OAAO,CAAC,EAAE,CAAC,CAAS;IACpB,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,KAAK,CAAgD;IAC7D,OAAO,CAAC,aAAa,CAAC,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAU;IAChC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,KAAK,CAAiB;IAG9B,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,WAAW,CAAc;IAGjC,OAAO,CAAC,kBAAkB,CAAqB;IAG/C,OAAO,CAAC,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,gBAAgB,CAAC,CAAS;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,sBAAsB,CAAC,CAAiB;IAGhD,OAAO,CAAC,KAAK,CAAC,CAAa;IAC3B,OAAO,CAAC,cAAc,CAAC,CAAM;IAG7B,OAAO,CAAC,OAAO,CAAwB;IAEvC;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,eAAe;IAwB9B;;;;OAIG;gBACS,MAAM,GAAE,WAAW,CAAC,qBAAqB,CAAC,GAAG,qBAA0B;IA0DnF;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,EAAE,CAAC,CAAC,SAAS,cAAc,EACzB,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,cAAc,EAC1B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IASP;;;;OAIG;IACH,IAAI,CAAC,CAAC,SAAS,cAAc,EAC3B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;OAGG;IACH,OAAO,CAAC,IAAI;IAoCZ;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAoB7B;;;OAGG;IACH,oBAAoB,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAS9C;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAO1B;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAU/B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAU9B;;OAEG;IACG,OAAO,CAAC,EAAE,cAAc,EAAE,GAAE;QAAE,cAAc,CAAC,EAAE,GAAG,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAKtE,OAAO,KAAK,EACV,qBAAqB,EACrB,sBAAsB,EACtB,kBAAkB,EAMlB,mBAAmB,EAEpB,MAAM,SAAS,CAAC;AAYjB;;GAEG;AAGH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,qBAAa,eAAgB,SAAQ,WAAW,CAC9C,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,UAAU,EACV,kBAAkB,CACnB;IACC,OAAO,CAAC,EAAE,CAAC,CAAS;IACpB,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,KAAK,CAAgD;IAC7D,OAAO,CAAC,aAAa,CAAC,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAU;IAChC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,KAAK,CAAiB;IAG9B,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,WAAW,CAAc;IAGjC,OAAO,CAAC,kBAAkB,CAAqB;IAG/C,OAAO,CAAC,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,gBAAgB,CAAC,CAAS;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,sBAAsB,CAAC,CAAiB;IAGhD,OAAO,CAAC,KAAK,CAAC,CAAa;IAC3B,OAAO,CAAC,cAAc,CAAC,CAAM;IAG7B,OAAO,CAAC,OAAO,CAAwB;IAEvC;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,eAAe;IAwB9B;;;;OAIG;gBACS,MAAM,GAAE,WAAW,CAAC,qBAAqB,CAAC,GAAG,qBAA0B;IA0DnF;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,EAAE,CAAC,CAAC,SAAS,cAAc,EACzB,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,cAAc,EAC1B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IASP;;;;OAIG;IACH,IAAI,CAAC,CAAC,SAAS,cAAc,EAC3B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;OAGG;IACH,OAAO,CAAC,IAAI;IAoCZ;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAoB7B;;;OAGG;IACH,oBAAoB,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAS9C;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAO1B;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAU/B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAU9B;;OAEG;IACG,OAAO,CAAC,EAAE,cAAc,EAAE,GAAE;QAAE,cAAc,CAAC,EAAE,GAAG,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAyF/E;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAiDjC;;OAEG;IACG,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,cAAc,EAAE,OAAO,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC;IA2EnG;;OAEG;IACG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,cAAc,GAAG,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAmCxE;;OAEG;IACG,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,cAAc,EAAE,QAAQ,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,MAAM,CAAC;IAuFpG;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAY9E;;OAEG;IACG,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BtG;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACG,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA6MhF;;OAEG;IACH,kBAAkB,IAAI,cAAc,GAAG,WAAW;IAIlD;;OAEG;IACH,WAAW,IAAI,OAAO;IAItB;;;OAGG;IACH,uBAAuB,IAAI,MAAM,CAAC,cAAc,GAAG,IAAI;IAIvD;;OAEG;IACH,gBAAgB,IAAI,MAAM,GAAG,SAAS;IAKtC;;OAEG;IACH,cAAc,IAAI;QAChB,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,IAAI,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,mBAAmB,CAAC;QAC7B,WAAW,EAAE,MAAM,CAAC;KACrB;IAYD;;OAEG;IACH,iBAAiB,IAAI,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IAIhF;;OAEG;IACH,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI;IAI/D;;OAEG;IACH,YAAY,IAAI,IAAI;IAKpB;;OAEG;IACH,gBAAgB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAQxC;;;OAGG;YACW,qBAAqB;IA4BnC;;;OAGG;IACH,OAAO,CAAC,2BAA2B;IAqCnC;;;OAGG;IACH,OAAO,CAAC,aAAa;IAmBrB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAKvB;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IA6C3B;;;OAGG;YACW,mBAAmB;IA0EjC;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAwB3B;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAY5B;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAsI3B;;;OAGG;YACW,cAAc;IA4B5B;;;OAGG;YACW,qBAAqB;IAoFnC;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAWzB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAMxB;;;OAGG;IACH,OAAO,CAAC,WAAW;IAcnB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAYzB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAmIzB;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAqD7B;;;;OAIG;YACW,cAAc;IAO5B;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAI5B;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAI5B;;;OAGG;IACH,OAAO,CAAC,SAAS;IAkCjB;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,QAAQ,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAKjC;;;OAGG;IACH,QAAQ,IAAI,UAAU,GAAG,SAAS;IAIlC,OAAO,CAAC,GAAG;IAMX;;;OAGG;IACH,OAAO,CAAC,4BAA4B;IAiCpC;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IA2DjC;;OAEG;IACH,KAAK,IAAI,IAAI;IAIb;;OAEG;IACG,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAO/D;;;OAGG;IACH,eAAe,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI;CAM7C"}
|
package/dist/index.js
CHANGED
|
@@ -1260,7 +1260,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
1260
1260
|
sessionDurationTimeout;
|
|
1261
1261
|
// Tool integration properties
|
|
1262
1262
|
tools;
|
|
1263
|
-
|
|
1263
|
+
runtimeContext;
|
|
1264
1264
|
// Store the configuration options
|
|
1265
1265
|
options;
|
|
1266
1266
|
/**
|
|
@@ -1497,67 +1497,69 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
1497
1497
|
/**
|
|
1498
1498
|
* Establish connection to the Gemini Live API
|
|
1499
1499
|
*/
|
|
1500
|
-
async connect({
|
|
1501
|
-
|
|
1502
|
-
this.
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
this.requestContext = requestContext;
|
|
1506
|
-
this.emit("session", { state: "connecting" });
|
|
1507
|
-
try {
|
|
1508
|
-
let wsUrl;
|
|
1509
|
-
let headers = {};
|
|
1510
|
-
if (this.options.vertexAI) {
|
|
1511
|
-
wsUrl = `wss://${this.options.location}-aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1beta1.PredictionService.ServerStreamingPredict`;
|
|
1512
|
-
await this.authManager.initialize();
|
|
1513
|
-
const accessToken = await this.authManager.getAccessToken();
|
|
1514
|
-
headers = { headers: { Authorization: `Bearer ${accessToken}` } };
|
|
1515
|
-
this.log("Using Vertex AI authentication with OAuth token");
|
|
1516
|
-
} else {
|
|
1517
|
-
wsUrl = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent`;
|
|
1518
|
-
headers = {
|
|
1519
|
-
headers: {
|
|
1520
|
-
"x-goog-api-key": this.options.apiKey || "",
|
|
1521
|
-
"Content-Type": "application/json"
|
|
1522
|
-
}
|
|
1523
|
-
};
|
|
1524
|
-
this.log("Using Live API authentication with API key");
|
|
1525
|
-
}
|
|
1526
|
-
this.log("Connecting to:", wsUrl);
|
|
1527
|
-
this.ws = new WebSocket(wsUrl, void 0, headers);
|
|
1528
|
-
this.connectionManager.setWebSocket(this.ws);
|
|
1529
|
-
this.setupEventListeners();
|
|
1530
|
-
await this.connectionManager.waitForOpen();
|
|
1531
|
-
if (this.isResuming && this.sessionHandle) {
|
|
1532
|
-
await this.sendSessionResumption();
|
|
1533
|
-
} else {
|
|
1534
|
-
this.sendInitialConfig();
|
|
1535
|
-
this.sessionStartTime = Date.now();
|
|
1536
|
-
this.sessionId = randomUUID();
|
|
1500
|
+
async connect({ runtimeContext } = {}) {
|
|
1501
|
+
return this.traced(async () => {
|
|
1502
|
+
if (this.state === "connected") {
|
|
1503
|
+
this.log("Already connected to Gemini Live API");
|
|
1504
|
+
return;
|
|
1537
1505
|
}
|
|
1538
|
-
|
|
1539
|
-
this.state
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1506
|
+
this.runtimeContext = runtimeContext;
|
|
1507
|
+
this.emit("session", { state: "connecting" });
|
|
1508
|
+
try {
|
|
1509
|
+
let wsUrl;
|
|
1510
|
+
let headers = {};
|
|
1511
|
+
if (this.options.vertexAI) {
|
|
1512
|
+
wsUrl = `wss://${this.options.location}-aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1beta1.PredictionService.ServerStreamingPredict`;
|
|
1513
|
+
await this.authManager.initialize();
|
|
1514
|
+
const accessToken = await this.authManager.getAccessToken();
|
|
1515
|
+
headers = { headers: { Authorization: `Bearer ${accessToken}` } };
|
|
1516
|
+
this.log("Using Vertex AI authentication with OAuth token");
|
|
1517
|
+
} else {
|
|
1518
|
+
wsUrl = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent`;
|
|
1519
|
+
headers = {
|
|
1520
|
+
headers: {
|
|
1521
|
+
"x-goog-api-key": this.options.apiKey || "",
|
|
1522
|
+
"Content-Type": "application/json"
|
|
1523
|
+
}
|
|
1524
|
+
};
|
|
1525
|
+
this.log("Using Live API authentication with API key");
|
|
1526
|
+
}
|
|
1527
|
+
this.log("Connecting to:", wsUrl);
|
|
1528
|
+
this.ws = new WebSocket(wsUrl, void 0, headers);
|
|
1529
|
+
this.connectionManager.setWebSocket(this.ws);
|
|
1530
|
+
this.setupEventListeners();
|
|
1531
|
+
await this.connectionManager.waitForOpen();
|
|
1532
|
+
if (this.isResuming && this.sessionHandle) {
|
|
1533
|
+
await this.sendSessionResumption();
|
|
1534
|
+
} else {
|
|
1535
|
+
this.sendInitialConfig();
|
|
1536
|
+
this.sessionStartTime = Date.now();
|
|
1537
|
+
this.sessionId = randomUUID();
|
|
1538
|
+
}
|
|
1539
|
+
await this.waitForSessionCreated();
|
|
1540
|
+
this.state = "connected";
|
|
1541
|
+
this.emit("session", {
|
|
1542
|
+
state: "connected",
|
|
1543
|
+
config: {
|
|
1544
|
+
sessionId: this.sessionId,
|
|
1545
|
+
isResuming: this.isResuming,
|
|
1546
|
+
toolCount: Object.keys(this.tools || {}).length
|
|
1547
|
+
}
|
|
1548
|
+
});
|
|
1549
|
+
this.log("Successfully connected to Gemini Live API", {
|
|
1543
1550
|
sessionId: this.sessionId,
|
|
1544
1551
|
isResuming: this.isResuming,
|
|
1545
1552
|
toolCount: Object.keys(this.tools || {}).length
|
|
1553
|
+
});
|
|
1554
|
+
if (this.options.sessionConfig?.maxDuration) {
|
|
1555
|
+
this.startSessionDurationMonitor();
|
|
1546
1556
|
}
|
|
1547
|
-
})
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
toolCount: Object.keys(this.tools || {}).length
|
|
1552
|
-
});
|
|
1553
|
-
if (this.options.sessionConfig?.maxDuration) {
|
|
1554
|
-
this.startSessionDurationMonitor();
|
|
1557
|
+
} catch (error) {
|
|
1558
|
+
this.state = "disconnected";
|
|
1559
|
+
this.log("Connection failed", error);
|
|
1560
|
+
throw error;
|
|
1555
1561
|
}
|
|
1556
|
-
}
|
|
1557
|
-
this.state = "disconnected";
|
|
1558
|
-
this.log("Connection failed", error);
|
|
1559
|
-
throw error;
|
|
1560
|
-
}
|
|
1562
|
+
}, "gemini-live.connect")();
|
|
1561
1563
|
}
|
|
1562
1564
|
/**
|
|
1563
1565
|
* Disconnect from the Gemini Live API
|
|
@@ -1595,164 +1597,172 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
1595
1597
|
* Send text to be converted to speech
|
|
1596
1598
|
*/
|
|
1597
1599
|
async speak(input, options) {
|
|
1598
|
-
this.
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1600
|
+
return this.traced(async () => {
|
|
1601
|
+
this.validateConnectionState();
|
|
1602
|
+
if (typeof input !== "string") {
|
|
1603
|
+
const chunks = [];
|
|
1604
|
+
for await (const chunk of input) {
|
|
1605
|
+
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)));
|
|
1606
|
+
}
|
|
1607
|
+
input = Buffer.concat(chunks).toString("utf-8");
|
|
1603
1608
|
}
|
|
1604
|
-
input
|
|
1605
|
-
|
|
1606
|
-
if (input.trim().length === 0) {
|
|
1607
|
-
throw this.createAndEmitError("invalid_audio_format" /* INVALID_AUDIO_FORMAT */, "Input text is empty");
|
|
1608
|
-
}
|
|
1609
|
-
this.addToContext("user", input);
|
|
1610
|
-
const textMessage = {
|
|
1611
|
-
client_content: {
|
|
1612
|
-
turns: [
|
|
1613
|
-
{
|
|
1614
|
-
role: "user",
|
|
1615
|
-
parts: [
|
|
1616
|
-
{
|
|
1617
|
-
text: input
|
|
1618
|
-
}
|
|
1619
|
-
]
|
|
1620
|
-
}
|
|
1621
|
-
],
|
|
1622
|
-
turnComplete: true
|
|
1609
|
+
if (input.trim().length === 0) {
|
|
1610
|
+
throw this.createAndEmitError("invalid_audio_format" /* INVALID_AUDIO_FORMAT */, "Input text is empty");
|
|
1623
1611
|
}
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1612
|
+
this.addToContext("user", input);
|
|
1613
|
+
const textMessage = {
|
|
1614
|
+
client_content: {
|
|
1615
|
+
turns: [
|
|
1616
|
+
{
|
|
1617
|
+
role: "user",
|
|
1618
|
+
parts: [
|
|
1619
|
+
{
|
|
1620
|
+
text: input
|
|
1621
|
+
}
|
|
1622
|
+
]
|
|
1634
1623
|
}
|
|
1635
|
-
|
|
1624
|
+
],
|
|
1625
|
+
turnComplete: true
|
|
1636
1626
|
}
|
|
1637
1627
|
};
|
|
1628
|
+
if (options && (options.speaker || options.languageCode || options.responseModalities)) {
|
|
1629
|
+
const updateMessage = {
|
|
1630
|
+
type: "session.update",
|
|
1631
|
+
session: {
|
|
1632
|
+
generation_config: {
|
|
1633
|
+
...options.responseModalities ? { response_modalities: options.responseModalities } : {},
|
|
1634
|
+
speech_config: {
|
|
1635
|
+
...options.languageCode ? { language_code: options.languageCode } : {},
|
|
1636
|
+
...options.speaker ? { voice_config: { prebuilt_voice_config: { voice_name: options.speaker } } } : {}
|
|
1637
|
+
}
|
|
1638
|
+
}
|
|
1639
|
+
}
|
|
1640
|
+
};
|
|
1641
|
+
try {
|
|
1642
|
+
this.sendEvent("session.update", updateMessage);
|
|
1643
|
+
this.log("Applied per-turn runtime options", options);
|
|
1644
|
+
} catch (error) {
|
|
1645
|
+
this.log("Failed to apply per-turn runtime options", error);
|
|
1646
|
+
}
|
|
1647
|
+
}
|
|
1638
1648
|
try {
|
|
1639
|
-
this.sendEvent("
|
|
1640
|
-
this.log("
|
|
1649
|
+
this.sendEvent("client_content", textMessage);
|
|
1650
|
+
this.log("Text message sent", { text: input });
|
|
1641
1651
|
} catch (error) {
|
|
1642
|
-
this.log("Failed to
|
|
1652
|
+
this.log("Failed to send text message", error);
|
|
1653
|
+
throw this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to send text message", error);
|
|
1643
1654
|
}
|
|
1644
|
-
}
|
|
1645
|
-
try {
|
|
1646
|
-
this.sendEvent("client_content", textMessage);
|
|
1647
|
-
this.log("Text message sent", { text: input });
|
|
1648
|
-
} catch (error) {
|
|
1649
|
-
this.log("Failed to send text message", error);
|
|
1650
|
-
throw this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to send text message", error);
|
|
1651
|
-
}
|
|
1655
|
+
}, "gemini-live.speak")();
|
|
1652
1656
|
}
|
|
1653
1657
|
/**
|
|
1654
1658
|
* Send audio stream for processing
|
|
1655
1659
|
*/
|
|
1656
1660
|
async send(audioData) {
|
|
1657
|
-
this.
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1661
|
+
return this.traced(async () => {
|
|
1662
|
+
this.validateConnectionState();
|
|
1663
|
+
if ("readable" in audioData && typeof audioData.on === "function") {
|
|
1664
|
+
const stream = audioData;
|
|
1665
|
+
stream.on("data", (chunk) => {
|
|
1666
|
+
try {
|
|
1667
|
+
const base64Audio = this.audioStreamManager.processAudioChunk(chunk);
|
|
1668
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "realtime");
|
|
1669
|
+
this.sendEvent("realtime_input", message);
|
|
1670
|
+
} catch (error) {
|
|
1671
|
+
this.log("Failed to process audio chunk", error);
|
|
1672
|
+
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to process audio chunk", error);
|
|
1673
|
+
}
|
|
1674
|
+
});
|
|
1675
|
+
stream.on("error", (error) => {
|
|
1676
|
+
this.log("Audio stream error", error);
|
|
1677
|
+
this.createAndEmitError("audio_stream_error" /* AUDIO_STREAM_ERROR */, "Audio stream error", error);
|
|
1678
|
+
});
|
|
1679
|
+
stream.on("end", () => {
|
|
1680
|
+
this.log("Audio stream ended");
|
|
1681
|
+
});
|
|
1682
|
+
} else {
|
|
1683
|
+
const validateAudio = this.audioStreamManager.validateAndConvertAudioInput(audioData);
|
|
1684
|
+
const base64Audio = this.audioStreamManager.int16ArrayToBase64(validateAudio);
|
|
1685
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "realtime");
|
|
1686
|
+
this.sendEvent("realtime_input", message);
|
|
1687
|
+
}
|
|
1688
|
+
}, "gemini-live.send")();
|
|
1683
1689
|
}
|
|
1684
1690
|
/**
|
|
1685
1691
|
* Process speech from audio stream (traditional STT interface)
|
|
1686
1692
|
*/
|
|
1687
1693
|
async listen(audioStream, _options) {
|
|
1688
|
-
this.
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
};
|
|
1696
|
-
const onError = (error) => {
|
|
1697
|
-
throw new Error(`Transcription failed: ${error.message}`);
|
|
1698
|
-
};
|
|
1699
|
-
const onSession = (data) => {
|
|
1700
|
-
if (data.state === "disconnected") {
|
|
1701
|
-
throw new Error("Session disconnected during transcription");
|
|
1702
|
-
}
|
|
1703
|
-
};
|
|
1704
|
-
this.on("writing", onWriting);
|
|
1705
|
-
this.on("error", onError);
|
|
1706
|
-
this.on("session", onSession);
|
|
1707
|
-
try {
|
|
1708
|
-
const result = await this.audioStreamManager.handleAudioTranscription(
|
|
1709
|
-
audioStream,
|
|
1710
|
-
(base64Audio) => {
|
|
1711
|
-
return new Promise((resolve, reject) => {
|
|
1712
|
-
try {
|
|
1713
|
-
const message = this.audioStreamManager.createAudioMessage(base64Audio, "input");
|
|
1714
|
-
const cleanup = () => {
|
|
1715
|
-
this.off("turnComplete", onTurnComplete);
|
|
1716
|
-
this.off("error", onErr);
|
|
1717
|
-
};
|
|
1718
|
-
const onTurnComplete = () => {
|
|
1719
|
-
cleanup();
|
|
1720
|
-
resolve(transcriptionText.trim());
|
|
1721
|
-
};
|
|
1722
|
-
const onErr = (e) => {
|
|
1723
|
-
cleanup();
|
|
1724
|
-
reject(new Error(e.message));
|
|
1725
|
-
};
|
|
1726
|
-
this.on("turnComplete", onTurnComplete);
|
|
1727
|
-
this.on("error", onErr);
|
|
1728
|
-
this.sendEvent("client_content", message);
|
|
1729
|
-
this.log("Sent audio for transcription");
|
|
1730
|
-
} catch (err) {
|
|
1731
|
-
reject(err);
|
|
1732
|
-
}
|
|
1733
|
-
});
|
|
1734
|
-
},
|
|
1735
|
-
(error) => {
|
|
1736
|
-
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Audio transcription failed", error);
|
|
1694
|
+
return this.traced(async () => {
|
|
1695
|
+
this.validateConnectionState();
|
|
1696
|
+
let transcriptionText = "";
|
|
1697
|
+
const onWriting = (data) => {
|
|
1698
|
+
if (data.role === "user") {
|
|
1699
|
+
transcriptionText += data.text;
|
|
1700
|
+
this.log("Received transcription text:", { text: data.text, total: transcriptionText });
|
|
1737
1701
|
}
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1702
|
+
};
|
|
1703
|
+
const onError = (error) => {
|
|
1704
|
+
throw new Error(`Transcription failed: ${error.message}`);
|
|
1705
|
+
};
|
|
1706
|
+
const onSession = (data) => {
|
|
1707
|
+
if (data.state === "disconnected") {
|
|
1708
|
+
throw new Error("Session disconnected during transcription");
|
|
1709
|
+
}
|
|
1710
|
+
};
|
|
1711
|
+
this.on("writing", onWriting);
|
|
1712
|
+
this.on("error", onError);
|
|
1713
|
+
this.on("session", onSession);
|
|
1714
|
+
try {
|
|
1715
|
+
const result = await this.audioStreamManager.handleAudioTranscription(
|
|
1716
|
+
audioStream,
|
|
1717
|
+
(base64Audio) => {
|
|
1718
|
+
return new Promise((resolve, reject) => {
|
|
1719
|
+
try {
|
|
1720
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "input");
|
|
1721
|
+
const cleanup = () => {
|
|
1722
|
+
this.off("turnComplete", onTurnComplete);
|
|
1723
|
+
this.off("error", onErr);
|
|
1724
|
+
};
|
|
1725
|
+
const onTurnComplete = () => {
|
|
1726
|
+
cleanup();
|
|
1727
|
+
resolve(transcriptionText.trim());
|
|
1728
|
+
};
|
|
1729
|
+
const onErr = (e) => {
|
|
1730
|
+
cleanup();
|
|
1731
|
+
reject(new Error(e.message));
|
|
1732
|
+
};
|
|
1733
|
+
this.on("turnComplete", onTurnComplete);
|
|
1734
|
+
this.on("error", onErr);
|
|
1735
|
+
this.sendEvent("client_content", message);
|
|
1736
|
+
this.log("Sent audio for transcription");
|
|
1737
|
+
} catch (err) {
|
|
1738
|
+
reject(err);
|
|
1739
|
+
}
|
|
1740
|
+
});
|
|
1741
|
+
},
|
|
1742
|
+
(error) => {
|
|
1743
|
+
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Audio transcription failed", error);
|
|
1744
|
+
}
|
|
1745
|
+
);
|
|
1746
|
+
return result;
|
|
1747
|
+
} finally {
|
|
1748
|
+
this.off("writing", onWriting);
|
|
1749
|
+
this.off("error", onError);
|
|
1750
|
+
this.off("session", onSession);
|
|
1751
|
+
}
|
|
1752
|
+
}, "gemini-live.listen")();
|
|
1745
1753
|
}
|
|
1746
1754
|
/**
|
|
1747
1755
|
* Get available speakers/voices
|
|
1748
1756
|
*/
|
|
1749
1757
|
async getSpeakers() {
|
|
1750
|
-
return
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1758
|
+
return this.traced(async () => {
|
|
1759
|
+
return [
|
|
1760
|
+
{ voiceId: "Puck", description: "Conversational, friendly" },
|
|
1761
|
+
{ voiceId: "Charon", description: "Deep, authoritative" },
|
|
1762
|
+
{ voiceId: "Kore", description: "Neutral, professional" },
|
|
1763
|
+
{ voiceId: "Fenrir", description: "Warm, approachable" }
|
|
1764
|
+
];
|
|
1765
|
+
}, "gemini-live.getSpeakers")();
|
|
1756
1766
|
}
|
|
1757
1767
|
/**
|
|
1758
1768
|
* Resume a previous session using a session handle
|
|
@@ -2380,7 +2390,13 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
2380
2390
|
let result;
|
|
2381
2391
|
if (tool.execute) {
|
|
2382
2392
|
this.log("Executing tool", { toolName, toolArgs });
|
|
2383
|
-
result = await tool.execute(
|
|
2393
|
+
result = await tool.execute(
|
|
2394
|
+
{ context: toolArgs, runtimeContext: this.runtimeContext },
|
|
2395
|
+
{
|
|
2396
|
+
toolCallId: toolId,
|
|
2397
|
+
messages: []
|
|
2398
|
+
}
|
|
2399
|
+
);
|
|
2384
2400
|
this.log("Tool executed successfully", { toolName, result });
|
|
2385
2401
|
} else {
|
|
2386
2402
|
this.log("Tool has no execute function", { toolName });
|
|
@@ -2657,14 +2673,14 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
2657
2673
|
* inputSchema: z.object({
|
|
2658
2674
|
* location: z.string().describe("The city and state, e.g. San Francisco, CA"),
|
|
2659
2675
|
* }),
|
|
2660
|
-
* execute: async (
|
|
2676
|
+
* execute: async ({ context }) => {
|
|
2661
2677
|
* // Fetch weather data from an API
|
|
2662
2678
|
* const response = await fetch(
|
|
2663
|
-
* `https://api.weather.com?location=${encodeURIComponent(
|
|
2679
|
+
* `https://api.weather.com?location=${encodeURIComponent(context.location)}`,
|
|
2664
2680
|
* );
|
|
2665
2681
|
* const data = await response.json();
|
|
2666
2682
|
* return {
|
|
2667
|
-
* message: `The current temperature in ${
|
|
2683
|
+
* message: `The current temperature in ${context.location} is ${data.temperature}°F with ${data.conditions}.`,
|
|
2668
2684
|
* };
|
|
2669
2685
|
* },
|
|
2670
2686
|
* });
|
|
@@ -2682,7 +2698,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
2682
2698
|
* Get the current tools configured for this voice instance
|
|
2683
2699
|
* @returns Object containing the current tools
|
|
2684
2700
|
*/
|
|
2685
|
-
|
|
2701
|
+
getTools() {
|
|
2686
2702
|
return this.tools;
|
|
2687
2703
|
}
|
|
2688
2704
|
log(message, ...args) {
|