@mastra/voice-google-gemini-live 0.10.18 → 0.11.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +55 -17
- package/README.md +3 -3
- package/dist/index.cjs +195 -211
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +7 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +195 -211
- package/dist/index.js.map +1 -1
- package/package.json +7 -11
package/dist/index.d.ts
CHANGED
|
@@ -74,7 +74,7 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
74
74
|
private isResuming;
|
|
75
75
|
private sessionDurationTimeout?;
|
|
76
76
|
private tools?;
|
|
77
|
-
private
|
|
77
|
+
private requestContext?;
|
|
78
78
|
private options;
|
|
79
79
|
/**
|
|
80
80
|
* Normalize configuration to ensure proper VoiceConfig format
|
|
@@ -162,8 +162,8 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
162
162
|
/**
|
|
163
163
|
* Establish connection to the Gemini Live API
|
|
164
164
|
*/
|
|
165
|
-
connect({
|
|
166
|
-
|
|
165
|
+
connect({ requestContext }?: {
|
|
166
|
+
requestContext?: any;
|
|
167
167
|
}): Promise<void>;
|
|
168
168
|
/**
|
|
169
169
|
* Disconnect from the Gemini Live API
|
|
@@ -385,14 +385,14 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
385
385
|
* inputSchema: z.object({
|
|
386
386
|
* location: z.string().describe("The city and state, e.g. San Francisco, CA"),
|
|
387
387
|
* }),
|
|
388
|
-
* execute: async (
|
|
388
|
+
* execute: async (inputData) => {
|
|
389
389
|
* // Fetch weather data from an API
|
|
390
390
|
* const response = await fetch(
|
|
391
|
-
* `https://api.weather.com?location=${encodeURIComponent(
|
|
391
|
+
* `https://api.weather.com?location=${encodeURIComponent(inputData.location)}`,
|
|
392
392
|
* );
|
|
393
393
|
* const data = await response.json();
|
|
394
394
|
* return {
|
|
395
|
-
* message: `The current temperature in ${
|
|
395
|
+
* message: `The current temperature in ${inputData.location} is ${data.temperature}°F with ${data.conditions}.`,
|
|
396
396
|
* };
|
|
397
397
|
* },
|
|
398
398
|
* });
|
|
@@ -407,7 +407,7 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
407
407
|
* Get the current tools configured for this voice instance
|
|
408
408
|
* @returns Object containing the current tools
|
|
409
409
|
*/
|
|
410
|
-
|
|
410
|
+
listTools(): ToolsInput | undefined;
|
|
411
411
|
private log;
|
|
412
412
|
/**
|
|
413
413
|
* Convert Zod schema to JSON Schema for tool parameters
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAKtE,OAAO,KAAK,EACV,qBAAqB,EACrB,sBAAsB,EACtB,kBAAkB,EAMlB,mBAAmB,EAEpB,MAAM,SAAS,CAAC;AAYjB;;GAEG;AAGH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,qBAAa,eAAgB,SAAQ,WAAW,CAC9C,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,UAAU,EACV,kBAAkB,CACnB;IACC,OAAO,CAAC,EAAE,CAAC,CAAS;IACpB,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,KAAK,CAAgD;IAC7D,OAAO,CAAC,aAAa,CAAC,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAU;IAChC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,KAAK,CAAiB;IAG9B,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,WAAW,CAAc;IAGjC,OAAO,CAAC,kBAAkB,CAAqB;IAG/C,OAAO,CAAC,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,gBAAgB,CAAC,CAAS;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,sBAAsB,CAAC,CAAiB;IAGhD,OAAO,CAAC,KAAK,CAAC,CAAa;IAC3B,OAAO,CAAC,cAAc,CAAC,CAAM;IAG7B,OAAO,CAAC,OAAO,CAAwB;IAEvC;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,eAAe;IAwB9B;;;;OAIG;gBACS,MAAM,GAAE,WAAW,CAAC,qBAAqB,CAAC,GAAG,qBAA0B;IA0DnF;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,EAAE,CAAC,CAAC,SAAS,cAAc,EACzB,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,cAAc,EAC1B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IASP;;;;OAIG;IACH,IAAI,CAAC,CAAC,SAAS,cAAc,EAC3B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;OAGG;IACH,OAAO,CAAC,IAAI;IAoCZ;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAoB7B;;;OAGG;IACH,oBAAoB,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAS9C;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAO1B;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAU/B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAU9B;;OAEG;IACG,OAAO,CAAC,EAAE,cAAc,EAAE,GAAE;QAAE,cAAc,CAAC,EAAE,GAAG,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAKtE,OAAO,KAAK,EACV,qBAAqB,EACrB,sBAAsB,EACtB,kBAAkB,EAMlB,mBAAmB,EAEpB,MAAM,SAAS,CAAC;AAYjB;;GAEG;AAGH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,qBAAa,eAAgB,SAAQ,WAAW,CAC9C,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,UAAU,EACV,kBAAkB,CACnB;IACC,OAAO,CAAC,EAAE,CAAC,CAAS;IACpB,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,KAAK,CAAgD;IAC7D,OAAO,CAAC,aAAa,CAAC,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAU;IAChC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,KAAK,CAAiB;IAG9B,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,WAAW,CAAc;IAGjC,OAAO,CAAC,kBAAkB,CAAqB;IAG/C,OAAO,CAAC,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,gBAAgB,CAAC,CAAS;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,sBAAsB,CAAC,CAAiB;IAGhD,OAAO,CAAC,KAAK,CAAC,CAAa;IAC3B,OAAO,CAAC,cAAc,CAAC,CAAM;IAG7B,OAAO,CAAC,OAAO,CAAwB;IAEvC;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,eAAe;IAwB9B;;;;OAIG;gBACS,MAAM,GAAE,WAAW,CAAC,qBAAqB,CAAC,GAAG,qBAA0B;IA0DnF;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,EAAE,CAAC,CAAC,SAAS,cAAc,EACzB,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,cAAc,EAC1B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IASP;;;;OAIG;IACH,IAAI,CAAC,CAAC,SAAS,cAAc,EAC3B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;OAGG;IACH,OAAO,CAAC,IAAI;IAoCZ;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAoB7B;;;OAGG;IACH,oBAAoB,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAS9C;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAO1B;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAU/B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAU9B;;OAEG;IACG,OAAO,CAAC,EAAE,cAAc,EAAE,GAAE;QAAE,cAAc,CAAC,EAAE,GAAG,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAuF/E;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAiDjC;;OAEG;IACG,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,cAAc,EAAE,OAAO,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC;IAuEnG;;OAEG;IACG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,cAAc,GAAG,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAiCxE;;OAEG;IACG,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,cAAc,EAAE,QAAQ,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,MAAM,CAAC;IAqFpG;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAU9E;;OAEG;IACG,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BtG;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACG,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA6MhF;;OAEG;IACH,kBAAkB,IAAI,cAAc,GAAG,WAAW;IAIlD;;OAEG;IACH,WAAW,IAAI,OAAO;IAItB;;;OAGG;IACH,uBAAuB,IAAI,MAAM,CAAC,cAAc,GAAG,IAAI;IAIvD;;OAEG;IACH,gBAAgB,IAAI,MAAM,GAAG,SAAS;IAKtC;;OAEG;IACH,cAAc,IAAI;QAChB,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,IAAI,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,mBAAmB,CAAC;QAC7B,WAAW,EAAE,MAAM,CAAC;KACrB;IAYD;;OAEG;IACH,iBAAiB,IAAI,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IAIhF;;OAEG;IACH,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI;IAI/D;;OAEG;IACH,YAAY,IAAI,IAAI;IAKpB;;OAEG;IACH,gBAAgB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAQxC;;;OAGG;YACW,qBAAqB;IA4BnC;;;OAGG;IACH,OAAO,CAAC,2BAA2B;IAqCnC;;;OAGG;IACH,OAAO,CAAC,aAAa;IAmBrB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAKvB;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IA6C3B;;;OAGG;YACW,mBAAmB;IA0EjC;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAwB3B;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAY5B;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAiH3B;;;OAGG;YACW,cAAc;IA8E5B;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAWzB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAMxB;;;OAGG;IACH,OAAO,CAAC,WAAW;IAcnB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAYzB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAmIzB;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAqD7B;;;;OAIG;YACW,cAAc;IAO5B;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAI5B;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAI5B;;;OAGG;IACH,OAAO,CAAC,SAAS;IA+BjB;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,QAAQ,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAKjC;;;OAGG;IACH,SAAS,IAAI,UAAU,GAAG,SAAS;IAInC,OAAO,CAAC,GAAG;IAMX;;;OAGG;IACH,OAAO,CAAC,4BAA4B;IAiCpC;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IA2DjC;;OAEG;IACH,KAAK,IAAI,IAAI;IAIb;;OAEG;IACG,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAO/D;;;OAGG;IACH,eAAe,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI;CAM7C"}
|
package/dist/index.js
CHANGED
|
@@ -1260,7 +1260,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
1260
1260
|
sessionDurationTimeout;
|
|
1261
1261
|
// Tool integration properties
|
|
1262
1262
|
tools;
|
|
1263
|
-
|
|
1263
|
+
requestContext;
|
|
1264
1264
|
// Store the configuration options
|
|
1265
1265
|
options;
|
|
1266
1266
|
/**
|
|
@@ -1497,69 +1497,67 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
1497
1497
|
/**
|
|
1498
1498
|
* Establish connection to the Gemini Live API
|
|
1499
1499
|
*/
|
|
1500
|
-
async connect({
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
headers
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
"Content-Type": "application/json"
|
|
1523
|
-
}
|
|
1524
|
-
};
|
|
1525
|
-
this.log("Using Live API authentication with API key");
|
|
1526
|
-
}
|
|
1527
|
-
this.log("Connecting to:", wsUrl);
|
|
1528
|
-
this.ws = new WebSocket(wsUrl, void 0, headers);
|
|
1529
|
-
this.connectionManager.setWebSocket(this.ws);
|
|
1530
|
-
this.setupEventListeners();
|
|
1531
|
-
await this.connectionManager.waitForOpen();
|
|
1532
|
-
if (this.isResuming && this.sessionHandle) {
|
|
1533
|
-
await this.sendSessionResumption();
|
|
1534
|
-
} else {
|
|
1535
|
-
this.sendInitialConfig();
|
|
1536
|
-
this.sessionStartTime = Date.now();
|
|
1537
|
-
this.sessionId = randomUUID();
|
|
1538
|
-
}
|
|
1539
|
-
await this.waitForSessionCreated();
|
|
1540
|
-
this.state = "connected";
|
|
1541
|
-
this.emit("session", {
|
|
1542
|
-
state: "connected",
|
|
1543
|
-
config: {
|
|
1544
|
-
sessionId: this.sessionId,
|
|
1545
|
-
isResuming: this.isResuming,
|
|
1546
|
-
toolCount: Object.keys(this.tools || {}).length
|
|
1500
|
+
async connect({ requestContext } = {}) {
|
|
1501
|
+
if (this.state === "connected") {
|
|
1502
|
+
this.log("Already connected to Gemini Live API");
|
|
1503
|
+
return;
|
|
1504
|
+
}
|
|
1505
|
+
this.requestContext = requestContext;
|
|
1506
|
+
this.emit("session", { state: "connecting" });
|
|
1507
|
+
try {
|
|
1508
|
+
let wsUrl;
|
|
1509
|
+
let headers = {};
|
|
1510
|
+
if (this.options.vertexAI) {
|
|
1511
|
+
wsUrl = `wss://${this.options.location}-aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1beta1.PredictionService.ServerStreamingPredict`;
|
|
1512
|
+
await this.authManager.initialize();
|
|
1513
|
+
const accessToken = await this.authManager.getAccessToken();
|
|
1514
|
+
headers = { headers: { Authorization: `Bearer ${accessToken}` } };
|
|
1515
|
+
this.log("Using Vertex AI authentication with OAuth token");
|
|
1516
|
+
} else {
|
|
1517
|
+
wsUrl = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent`;
|
|
1518
|
+
headers = {
|
|
1519
|
+
headers: {
|
|
1520
|
+
"x-goog-api-key": this.options.apiKey || "",
|
|
1521
|
+
"Content-Type": "application/json"
|
|
1547
1522
|
}
|
|
1548
|
-
}
|
|
1549
|
-
this.log("
|
|
1523
|
+
};
|
|
1524
|
+
this.log("Using Live API authentication with API key");
|
|
1525
|
+
}
|
|
1526
|
+
this.log("Connecting to:", wsUrl);
|
|
1527
|
+
this.ws = new WebSocket(wsUrl, void 0, headers);
|
|
1528
|
+
this.connectionManager.setWebSocket(this.ws);
|
|
1529
|
+
this.setupEventListeners();
|
|
1530
|
+
await this.connectionManager.waitForOpen();
|
|
1531
|
+
if (this.isResuming && this.sessionHandle) {
|
|
1532
|
+
await this.sendSessionResumption();
|
|
1533
|
+
} else {
|
|
1534
|
+
this.sendInitialConfig();
|
|
1535
|
+
this.sessionStartTime = Date.now();
|
|
1536
|
+
this.sessionId = randomUUID();
|
|
1537
|
+
}
|
|
1538
|
+
await this.waitForSessionCreated();
|
|
1539
|
+
this.state = "connected";
|
|
1540
|
+
this.emit("session", {
|
|
1541
|
+
state: "connected",
|
|
1542
|
+
config: {
|
|
1550
1543
|
sessionId: this.sessionId,
|
|
1551
1544
|
isResuming: this.isResuming,
|
|
1552
1545
|
toolCount: Object.keys(this.tools || {}).length
|
|
1553
|
-
});
|
|
1554
|
-
if (this.options.sessionConfig?.maxDuration) {
|
|
1555
|
-
this.startSessionDurationMonitor();
|
|
1556
1546
|
}
|
|
1557
|
-
}
|
|
1558
|
-
|
|
1559
|
-
this.
|
|
1560
|
-
|
|
1547
|
+
});
|
|
1548
|
+
this.log("Successfully connected to Gemini Live API", {
|
|
1549
|
+
sessionId: this.sessionId,
|
|
1550
|
+
isResuming: this.isResuming,
|
|
1551
|
+
toolCount: Object.keys(this.tools || {}).length
|
|
1552
|
+
});
|
|
1553
|
+
if (this.options.sessionConfig?.maxDuration) {
|
|
1554
|
+
this.startSessionDurationMonitor();
|
|
1561
1555
|
}
|
|
1562
|
-
}
|
|
1556
|
+
} catch (error) {
|
|
1557
|
+
this.state = "disconnected";
|
|
1558
|
+
this.log("Connection failed", error);
|
|
1559
|
+
throw error;
|
|
1560
|
+
}
|
|
1563
1561
|
}
|
|
1564
1562
|
/**
|
|
1565
1563
|
* Disconnect from the Gemini Live API
|
|
@@ -1597,172 +1595,164 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
1597
1595
|
* Send text to be converted to speech
|
|
1598
1596
|
*/
|
|
1599
1597
|
async speak(input, options) {
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)));
|
|
1606
|
-
}
|
|
1607
|
-
input = Buffer.concat(chunks).toString("utf-8");
|
|
1608
|
-
}
|
|
1609
|
-
if (input.trim().length === 0) {
|
|
1610
|
-
throw this.createAndEmitError("invalid_audio_format" /* INVALID_AUDIO_FORMAT */, "Input text is empty");
|
|
1598
|
+
this.validateConnectionState();
|
|
1599
|
+
if (typeof input !== "string") {
|
|
1600
|
+
const chunks = [];
|
|
1601
|
+
for await (const chunk of input) {
|
|
1602
|
+
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)));
|
|
1611
1603
|
}
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
}
|
|
1627
|
-
};
|
|
1628
|
-
if (options && (options.speaker || options.languageCode || options.responseModalities)) {
|
|
1629
|
-
const updateMessage = {
|
|
1630
|
-
type: "session.update",
|
|
1631
|
-
session: {
|
|
1632
|
-
generation_config: {
|
|
1633
|
-
...options.responseModalities ? { response_modalities: options.responseModalities } : {},
|
|
1634
|
-
speech_config: {
|
|
1635
|
-
...options.languageCode ? { language_code: options.languageCode } : {},
|
|
1636
|
-
...options.speaker ? { voice_config: { prebuilt_voice_config: { voice_name: options.speaker } } } : {}
|
|
1604
|
+
input = Buffer.concat(chunks).toString("utf-8");
|
|
1605
|
+
}
|
|
1606
|
+
if (input.trim().length === 0) {
|
|
1607
|
+
throw this.createAndEmitError("invalid_audio_format" /* INVALID_AUDIO_FORMAT */, "Input text is empty");
|
|
1608
|
+
}
|
|
1609
|
+
this.addToContext("user", input);
|
|
1610
|
+
const textMessage = {
|
|
1611
|
+
client_content: {
|
|
1612
|
+
turns: [
|
|
1613
|
+
{
|
|
1614
|
+
role: "user",
|
|
1615
|
+
parts: [
|
|
1616
|
+
{
|
|
1617
|
+
text: input
|
|
1637
1618
|
}
|
|
1619
|
+
]
|
|
1620
|
+
}
|
|
1621
|
+
],
|
|
1622
|
+
turnComplete: true
|
|
1623
|
+
}
|
|
1624
|
+
};
|
|
1625
|
+
if (options && (options.speaker || options.languageCode || options.responseModalities)) {
|
|
1626
|
+
const updateMessage = {
|
|
1627
|
+
type: "session.update",
|
|
1628
|
+
session: {
|
|
1629
|
+
generation_config: {
|
|
1630
|
+
...options.responseModalities ? { response_modalities: options.responseModalities } : {},
|
|
1631
|
+
speech_config: {
|
|
1632
|
+
...options.languageCode ? { language_code: options.languageCode } : {},
|
|
1633
|
+
...options.speaker ? { voice_config: { prebuilt_voice_config: { voice_name: options.speaker } } } : {}
|
|
1638
1634
|
}
|
|
1639
1635
|
}
|
|
1640
|
-
};
|
|
1641
|
-
try {
|
|
1642
|
-
this.sendEvent("session.update", updateMessage);
|
|
1643
|
-
this.log("Applied per-turn runtime options", options);
|
|
1644
|
-
} catch (error) {
|
|
1645
|
-
this.log("Failed to apply per-turn runtime options", error);
|
|
1646
1636
|
}
|
|
1647
|
-
}
|
|
1637
|
+
};
|
|
1648
1638
|
try {
|
|
1649
|
-
this.sendEvent("
|
|
1650
|
-
this.log("
|
|
1639
|
+
this.sendEvent("session.update", updateMessage);
|
|
1640
|
+
this.log("Applied per-turn runtime options", options);
|
|
1651
1641
|
} catch (error) {
|
|
1652
|
-
this.log("Failed to
|
|
1653
|
-
throw this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to send text message", error);
|
|
1642
|
+
this.log("Failed to apply per-turn runtime options", error);
|
|
1654
1643
|
}
|
|
1655
|
-
}
|
|
1644
|
+
}
|
|
1645
|
+
try {
|
|
1646
|
+
this.sendEvent("client_content", textMessage);
|
|
1647
|
+
this.log("Text message sent", { text: input });
|
|
1648
|
+
} catch (error) {
|
|
1649
|
+
this.log("Failed to send text message", error);
|
|
1650
|
+
throw this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to send text message", error);
|
|
1651
|
+
}
|
|
1656
1652
|
}
|
|
1657
1653
|
/**
|
|
1658
1654
|
* Send audio stream for processing
|
|
1659
1655
|
*/
|
|
1660
1656
|
async send(audioData) {
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
}
|
|
1688
|
-
}, "gemini-live.send")();
|
|
1657
|
+
this.validateConnectionState();
|
|
1658
|
+
if ("readable" in audioData && typeof audioData.on === "function") {
|
|
1659
|
+
const stream = audioData;
|
|
1660
|
+
stream.on("data", (chunk) => {
|
|
1661
|
+
try {
|
|
1662
|
+
const base64Audio = this.audioStreamManager.processAudioChunk(chunk);
|
|
1663
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "realtime");
|
|
1664
|
+
this.sendEvent("realtime_input", message);
|
|
1665
|
+
} catch (error) {
|
|
1666
|
+
this.log("Failed to process audio chunk", error);
|
|
1667
|
+
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to process audio chunk", error);
|
|
1668
|
+
}
|
|
1669
|
+
});
|
|
1670
|
+
stream.on("error", (error) => {
|
|
1671
|
+
this.log("Audio stream error", error);
|
|
1672
|
+
this.createAndEmitError("audio_stream_error" /* AUDIO_STREAM_ERROR */, "Audio stream error", error);
|
|
1673
|
+
});
|
|
1674
|
+
stream.on("end", () => {
|
|
1675
|
+
this.log("Audio stream ended");
|
|
1676
|
+
});
|
|
1677
|
+
} else {
|
|
1678
|
+
const validateAudio = this.audioStreamManager.validateAndConvertAudioInput(audioData);
|
|
1679
|
+
const base64Audio = this.audioStreamManager.int16ArrayToBase64(validateAudio);
|
|
1680
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "realtime");
|
|
1681
|
+
this.sendEvent("realtime_input", message);
|
|
1682
|
+
}
|
|
1689
1683
|
}
|
|
1690
1684
|
/**
|
|
1691
1685
|
* Process speech from audio stream (traditional STT interface)
|
|
1692
1686
|
*/
|
|
1693
1687
|
async listen(audioStream, _options) {
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
this.log("Received transcription text:", { text: data.text, total: transcriptionText });
|
|
1701
|
-
}
|
|
1702
|
-
};
|
|
1703
|
-
const onError = (error) => {
|
|
1704
|
-
throw new Error(`Transcription failed: ${error.message}`);
|
|
1705
|
-
};
|
|
1706
|
-
const onSession = (data) => {
|
|
1707
|
-
if (data.state === "disconnected") {
|
|
1708
|
-
throw new Error("Session disconnected during transcription");
|
|
1709
|
-
}
|
|
1710
|
-
};
|
|
1711
|
-
this.on("writing", onWriting);
|
|
1712
|
-
this.on("error", onError);
|
|
1713
|
-
this.on("session", onSession);
|
|
1714
|
-
try {
|
|
1715
|
-
const result = await this.audioStreamManager.handleAudioTranscription(
|
|
1716
|
-
audioStream,
|
|
1717
|
-
(base64Audio) => {
|
|
1718
|
-
return new Promise((resolve, reject) => {
|
|
1719
|
-
try {
|
|
1720
|
-
const message = this.audioStreamManager.createAudioMessage(base64Audio, "input");
|
|
1721
|
-
const cleanup = () => {
|
|
1722
|
-
this.off("turnComplete", onTurnComplete);
|
|
1723
|
-
this.off("error", onErr);
|
|
1724
|
-
};
|
|
1725
|
-
const onTurnComplete = () => {
|
|
1726
|
-
cleanup();
|
|
1727
|
-
resolve(transcriptionText.trim());
|
|
1728
|
-
};
|
|
1729
|
-
const onErr = (e) => {
|
|
1730
|
-
cleanup();
|
|
1731
|
-
reject(new Error(e.message));
|
|
1732
|
-
};
|
|
1733
|
-
this.on("turnComplete", onTurnComplete);
|
|
1734
|
-
this.on("error", onErr);
|
|
1735
|
-
this.sendEvent("client_content", message);
|
|
1736
|
-
this.log("Sent audio for transcription");
|
|
1737
|
-
} catch (err) {
|
|
1738
|
-
reject(err);
|
|
1739
|
-
}
|
|
1740
|
-
});
|
|
1741
|
-
},
|
|
1742
|
-
(error) => {
|
|
1743
|
-
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Audio transcription failed", error);
|
|
1744
|
-
}
|
|
1745
|
-
);
|
|
1746
|
-
return result;
|
|
1747
|
-
} finally {
|
|
1748
|
-
this.off("writing", onWriting);
|
|
1749
|
-
this.off("error", onError);
|
|
1750
|
-
this.off("session", onSession);
|
|
1688
|
+
this.validateConnectionState();
|
|
1689
|
+
let transcriptionText = "";
|
|
1690
|
+
const onWriting = (data) => {
|
|
1691
|
+
if (data.role === "user") {
|
|
1692
|
+
transcriptionText += data.text;
|
|
1693
|
+
this.log("Received transcription text:", { text: data.text, total: transcriptionText });
|
|
1751
1694
|
}
|
|
1752
|
-
}
|
|
1695
|
+
};
|
|
1696
|
+
const onError = (error) => {
|
|
1697
|
+
throw new Error(`Transcription failed: ${error.message}`);
|
|
1698
|
+
};
|
|
1699
|
+
const onSession = (data) => {
|
|
1700
|
+
if (data.state === "disconnected") {
|
|
1701
|
+
throw new Error("Session disconnected during transcription");
|
|
1702
|
+
}
|
|
1703
|
+
};
|
|
1704
|
+
this.on("writing", onWriting);
|
|
1705
|
+
this.on("error", onError);
|
|
1706
|
+
this.on("session", onSession);
|
|
1707
|
+
try {
|
|
1708
|
+
const result = await this.audioStreamManager.handleAudioTranscription(
|
|
1709
|
+
audioStream,
|
|
1710
|
+
(base64Audio) => {
|
|
1711
|
+
return new Promise((resolve, reject) => {
|
|
1712
|
+
try {
|
|
1713
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "input");
|
|
1714
|
+
const cleanup = () => {
|
|
1715
|
+
this.off("turnComplete", onTurnComplete);
|
|
1716
|
+
this.off("error", onErr);
|
|
1717
|
+
};
|
|
1718
|
+
const onTurnComplete = () => {
|
|
1719
|
+
cleanup();
|
|
1720
|
+
resolve(transcriptionText.trim());
|
|
1721
|
+
};
|
|
1722
|
+
const onErr = (e) => {
|
|
1723
|
+
cleanup();
|
|
1724
|
+
reject(new Error(e.message));
|
|
1725
|
+
};
|
|
1726
|
+
this.on("turnComplete", onTurnComplete);
|
|
1727
|
+
this.on("error", onErr);
|
|
1728
|
+
this.sendEvent("client_content", message);
|
|
1729
|
+
this.log("Sent audio for transcription");
|
|
1730
|
+
} catch (err) {
|
|
1731
|
+
reject(err);
|
|
1732
|
+
}
|
|
1733
|
+
});
|
|
1734
|
+
},
|
|
1735
|
+
(error) => {
|
|
1736
|
+
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Audio transcription failed", error);
|
|
1737
|
+
}
|
|
1738
|
+
);
|
|
1739
|
+
return result;
|
|
1740
|
+
} finally {
|
|
1741
|
+
this.off("writing", onWriting);
|
|
1742
|
+
this.off("error", onError);
|
|
1743
|
+
this.off("session", onSession);
|
|
1744
|
+
}
|
|
1753
1745
|
}
|
|
1754
1746
|
/**
|
|
1755
1747
|
* Get available speakers/voices
|
|
1756
1748
|
*/
|
|
1757
1749
|
async getSpeakers() {
|
|
1758
|
-
return
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
];
|
|
1765
|
-
}, "gemini-live.getSpeakers")();
|
|
1750
|
+
return [
|
|
1751
|
+
{ voiceId: "Puck", description: "Conversational, friendly" },
|
|
1752
|
+
{ voiceId: "Charon", description: "Deep, authoritative" },
|
|
1753
|
+
{ voiceId: "Kore", description: "Neutral, professional" },
|
|
1754
|
+
{ voiceId: "Fenrir", description: "Warm, approachable" }
|
|
1755
|
+
];
|
|
1766
1756
|
}
|
|
1767
1757
|
/**
|
|
1768
1758
|
* Resume a previous session using a session handle
|
|
@@ -2363,13 +2353,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
2363
2353
|
let result;
|
|
2364
2354
|
if (tool.execute) {
|
|
2365
2355
|
this.log("Executing tool", { toolName, toolArgs });
|
|
2366
|
-
result = await tool.execute(
|
|
2367
|
-
{ context: toolArgs, runtimeContext: this.runtimeContext },
|
|
2368
|
-
{
|
|
2369
|
-
toolCallId: toolId,
|
|
2370
|
-
messages: []
|
|
2371
|
-
}
|
|
2372
|
-
);
|
|
2356
|
+
result = await tool.execute(toolArgs, { requestContext: this.requestContext });
|
|
2373
2357
|
this.log("Tool executed successfully", { toolName, result });
|
|
2374
2358
|
} else {
|
|
2375
2359
|
this.log("Tool has no execute function", { toolName });
|
|
@@ -2636,14 +2620,14 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
2636
2620
|
* inputSchema: z.object({
|
|
2637
2621
|
* location: z.string().describe("The city and state, e.g. San Francisco, CA"),
|
|
2638
2622
|
* }),
|
|
2639
|
-
* execute: async (
|
|
2623
|
+
* execute: async (inputData) => {
|
|
2640
2624
|
* // Fetch weather data from an API
|
|
2641
2625
|
* const response = await fetch(
|
|
2642
|
-
* `https://api.weather.com?location=${encodeURIComponent(
|
|
2626
|
+
* `https://api.weather.com?location=${encodeURIComponent(inputData.location)}`,
|
|
2643
2627
|
* );
|
|
2644
2628
|
* const data = await response.json();
|
|
2645
2629
|
* return {
|
|
2646
|
-
* message: `The current temperature in ${
|
|
2630
|
+
* message: `The current temperature in ${inputData.location} is ${data.temperature}°F with ${data.conditions}.`,
|
|
2647
2631
|
* };
|
|
2648
2632
|
* },
|
|
2649
2633
|
* });
|
|
@@ -2661,7 +2645,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
2661
2645
|
* Get the current tools configured for this voice instance
|
|
2662
2646
|
* @returns Object containing the current tools
|
|
2663
2647
|
*/
|
|
2664
|
-
|
|
2648
|
+
listTools() {
|
|
2665
2649
|
return this.tools;
|
|
2666
2650
|
}
|
|
2667
2651
|
log(message, ...args) {
|