@mastra/voice-google-gemini-live 0.11.0-alpha.0 → 0.11.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +59 -21
- package/README.md +3 -3
- package/dist/index.cjs +195 -211
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +7 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +195 -211
- package/dist/index.js.map +1 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +11 -13
package/dist/index.d.ts
CHANGED
|
@@ -74,7 +74,7 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
74
74
|
private isResuming;
|
|
75
75
|
private sessionDurationTimeout?;
|
|
76
76
|
private tools?;
|
|
77
|
-
private
|
|
77
|
+
private requestContext?;
|
|
78
78
|
private options;
|
|
79
79
|
/**
|
|
80
80
|
* Normalize configuration to ensure proper VoiceConfig format
|
|
@@ -162,8 +162,8 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
162
162
|
/**
|
|
163
163
|
* Establish connection to the Gemini Live API
|
|
164
164
|
*/
|
|
165
|
-
connect({
|
|
166
|
-
|
|
165
|
+
connect({ requestContext }?: {
|
|
166
|
+
requestContext?: any;
|
|
167
167
|
}): Promise<void>;
|
|
168
168
|
/**
|
|
169
169
|
* Disconnect from the Gemini Live API
|
|
@@ -390,14 +390,14 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
390
390
|
* inputSchema: z.object({
|
|
391
391
|
* location: z.string().describe("The city and state, e.g. San Francisco, CA"),
|
|
392
392
|
* }),
|
|
393
|
-
* execute: async (
|
|
393
|
+
* execute: async (inputData) => {
|
|
394
394
|
* // Fetch weather data from an API
|
|
395
395
|
* const response = await fetch(
|
|
396
|
-
* `https://api.weather.com?location=${encodeURIComponent(
|
|
396
|
+
* `https://api.weather.com?location=${encodeURIComponent(inputData.location)}`,
|
|
397
397
|
* );
|
|
398
398
|
* const data = await response.json();
|
|
399
399
|
* return {
|
|
400
|
-
* message: `The current temperature in ${
|
|
400
|
+
* message: `The current temperature in ${inputData.location} is ${data.temperature}°F with ${data.conditions}.`,
|
|
401
401
|
* };
|
|
402
402
|
* },
|
|
403
403
|
* });
|
|
@@ -412,7 +412,7 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
412
412
|
* Get the current tools configured for this voice instance
|
|
413
413
|
* @returns Object containing the current tools
|
|
414
414
|
*/
|
|
415
|
-
|
|
415
|
+
listTools(): ToolsInput | undefined;
|
|
416
416
|
private log;
|
|
417
417
|
/**
|
|
418
418
|
* Convert Zod schema to JSON Schema for tool parameters
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAKtE,OAAO,KAAK,EACV,qBAAqB,EACrB,sBAAsB,EACtB,kBAAkB,EAMlB,mBAAmB,EAEpB,MAAM,SAAS,CAAC;AAYjB;;GAEG;AAGH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,qBAAa,eAAgB,SAAQ,WAAW,CAC9C,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,UAAU,EACV,kBAAkB,CACnB;IACC,OAAO,CAAC,EAAE,CAAC,CAAS;IACpB,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,KAAK,CAAgD;IAC7D,OAAO,CAAC,aAAa,CAAC,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAU;IAChC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,KAAK,CAAiB;IAG9B,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,WAAW,CAAc;IAGjC,OAAO,CAAC,kBAAkB,CAAqB;IAG/C,OAAO,CAAC,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,gBAAgB,CAAC,CAAS;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,sBAAsB,CAAC,CAAiB;IAGhD,OAAO,CAAC,KAAK,CAAC,CAAa;IAC3B,OAAO,CAAC,cAAc,CAAC,CAAM;IAG7B,OAAO,CAAC,OAAO,CAAwB;IAEvC;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,eAAe;IAwB9B;;;;OAIG;gBACS,MAAM,GAAE,WAAW,CAAC,qBAAqB,CAAC,GAAG,qBAA0B;IA0DnF;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,EAAE,CAAC,CAAC,SAAS,cAAc,EACzB,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,cAAc,EAC1B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IASP;;;;OAIG;IACH,IAAI,CAAC,CAAC,SAAS,cAAc,EAC3B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;OAGG;IACH,OAAO,CAAC,IAAI;IAoCZ;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAoB7B;;;OAGG;IACH,oBAAoB,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAS9C;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAO1B;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAU/B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAU9B;;OAEG;IACG,OAAO,CAAC,EAAE,cAAc,EAAE,GAAE;QAAE,cAAc,CAAC,EAAE,GAAG,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAKtE,OAAO,KAAK,EACV,qBAAqB,EACrB,sBAAsB,EACtB,kBAAkB,EAMlB,mBAAmB,EAEpB,MAAM,SAAS,CAAC;AAYjB;;GAEG;AAGH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,qBAAa,eAAgB,SAAQ,WAAW,CAC9C,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,UAAU,EACV,kBAAkB,CACnB;IACC,OAAO,CAAC,EAAE,CAAC,CAAS;IACpB,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,KAAK,CAAgD;IAC7D,OAAO,CAAC,aAAa,CAAC,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAU;IAChC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,KAAK,CAAiB;IAG9B,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,WAAW,CAAc;IAGjC,OAAO,CAAC,kBAAkB,CAAqB;IAG/C,OAAO,CAAC,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,gBAAgB,CAAC,CAAS;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,sBAAsB,CAAC,CAAiB;IAGhD,OAAO,CAAC,KAAK,CAAC,CAAa;IAC3B,OAAO,CAAC,cAAc,CAAC,CAAM;IAG7B,OAAO,CAAC,OAAO,CAAwB;IAEvC;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,eAAe;IAwB9B;;;;OAIG;gBACS,MAAM,GAAE,WAAW,CAAC,qBAAqB,CAAC,GAAG,qBAA0B;IA0DnF;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,EAAE,CAAC,CAAC,SAAS,cAAc,EACzB,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,cAAc,EAC1B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IASP;;;;OAIG;IACH,IAAI,CAAC,CAAC,SAAS,cAAc,EAC3B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;OAGG;IACH,OAAO,CAAC,IAAI;IAoCZ;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAoB7B;;;OAGG;IACH,oBAAoB,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAS9C;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAO1B;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAU/B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAU9B;;OAEG;IACG,OAAO,CAAC,EAAE,cAAc,EAAE,GAAE;QAAE,cAAc,CAAC,EAAE,GAAG,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAuF/E;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAiDjC;;OAEG;IACG,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,cAAc,EAAE,OAAO,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC;IAuEnG;;OAEG;IACG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,cAAc,GAAG,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAiCxE;;OAEG;IACG,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,cAAc,EAAE,QAAQ,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,MAAM,CAAC;IAqFpG;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAU9E;;OAEG;IACG,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BtG;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACG,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA6MhF;;OAEG;IACH,kBAAkB,IAAI,cAAc,GAAG,WAAW;IAIlD;;OAEG;IACH,WAAW,IAAI,OAAO;IAItB;;;OAGG;IACH,uBAAuB,IAAI,MAAM,CAAC,cAAc,GAAG,IAAI;IAIvD;;OAEG;IACH,gBAAgB,IAAI,MAAM,GAAG,SAAS;IAKtC;;OAEG;IACH,cAAc,IAAI;QAChB,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,IAAI,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,mBAAmB,CAAC;QAC7B,WAAW,EAAE,MAAM,CAAC;KACrB;IAYD;;OAEG;IACH,iBAAiB,IAAI,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IAIhF;;OAEG;IACH,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI;IAI/D;;OAEG;IACH,YAAY,IAAI,IAAI;IAKpB;;OAEG;IACH,gBAAgB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAQxC;;;OAGG;YACW,qBAAqB;IA4BnC;;;OAGG;IACH,OAAO,CAAC,2BAA2B;IAqCnC;;;OAGG;IACH,OAAO,CAAC,aAAa;IAmBrB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAKvB;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IA6C3B;;;OAGG;YACW,mBAAmB;IA0EjC;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAwB3B;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAY5B;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAsI3B;;;OAGG;YACW,cAAc;IA4B5B;;;OAGG;YACW,qBAAqB;IA8EnC;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAWzB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAMxB;;;OAGG;IACH,OAAO,CAAC,WAAW;IAcnB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAYzB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAmIzB;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAqD7B;;;;OAIG;YACW,cAAc;IAO5B;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAI5B;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAI5B;;;OAGG;IACH,OAAO,CAAC,SAAS;IAkCjB;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,QAAQ,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAKjC;;;OAGG;IACH,SAAS,IAAI,UAAU,GAAG,SAAS;IAInC,OAAO,CAAC,GAAG;IAMX;;;OAGG;IACH,OAAO,CAAC,4BAA4B;IAiCpC;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IA2DjC;;OAEG;IACH,KAAK,IAAI,IAAI;IAIb;;OAEG;IACG,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAO/D;;;OAGG;IACH,eAAe,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI;CAM7C"}
|
package/dist/index.js
CHANGED
|
@@ -1260,7 +1260,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
1260
1260
|
sessionDurationTimeout;
|
|
1261
1261
|
// Tool integration properties
|
|
1262
1262
|
tools;
|
|
1263
|
-
|
|
1263
|
+
requestContext;
|
|
1264
1264
|
// Store the configuration options
|
|
1265
1265
|
options;
|
|
1266
1266
|
/**
|
|
@@ -1497,69 +1497,67 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
1497
1497
|
/**
|
|
1498
1498
|
* Establish connection to the Gemini Live API
|
|
1499
1499
|
*/
|
|
1500
|
-
async connect({
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
headers
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
"Content-Type": "application/json"
|
|
1523
|
-
}
|
|
1524
|
-
};
|
|
1525
|
-
this.log("Using Live API authentication with API key");
|
|
1526
|
-
}
|
|
1527
|
-
this.log("Connecting to:", wsUrl);
|
|
1528
|
-
this.ws = new WebSocket(wsUrl, void 0, headers);
|
|
1529
|
-
this.connectionManager.setWebSocket(this.ws);
|
|
1530
|
-
this.setupEventListeners();
|
|
1531
|
-
await this.connectionManager.waitForOpen();
|
|
1532
|
-
if (this.isResuming && this.sessionHandle) {
|
|
1533
|
-
await this.sendSessionResumption();
|
|
1534
|
-
} else {
|
|
1535
|
-
this.sendInitialConfig();
|
|
1536
|
-
this.sessionStartTime = Date.now();
|
|
1537
|
-
this.sessionId = randomUUID();
|
|
1538
|
-
}
|
|
1539
|
-
await this.waitForSessionCreated();
|
|
1540
|
-
this.state = "connected";
|
|
1541
|
-
this.emit("session", {
|
|
1542
|
-
state: "connected",
|
|
1543
|
-
config: {
|
|
1544
|
-
sessionId: this.sessionId,
|
|
1545
|
-
isResuming: this.isResuming,
|
|
1546
|
-
toolCount: Object.keys(this.tools || {}).length
|
|
1500
|
+
async connect({ requestContext } = {}) {
|
|
1501
|
+
if (this.state === "connected") {
|
|
1502
|
+
this.log("Already connected to Gemini Live API");
|
|
1503
|
+
return;
|
|
1504
|
+
}
|
|
1505
|
+
this.requestContext = requestContext;
|
|
1506
|
+
this.emit("session", { state: "connecting" });
|
|
1507
|
+
try {
|
|
1508
|
+
let wsUrl;
|
|
1509
|
+
let headers = {};
|
|
1510
|
+
if (this.options.vertexAI) {
|
|
1511
|
+
wsUrl = `wss://${this.options.location}-aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1beta1.PredictionService.ServerStreamingPredict`;
|
|
1512
|
+
await this.authManager.initialize();
|
|
1513
|
+
const accessToken = await this.authManager.getAccessToken();
|
|
1514
|
+
headers = { headers: { Authorization: `Bearer ${accessToken}` } };
|
|
1515
|
+
this.log("Using Vertex AI authentication with OAuth token");
|
|
1516
|
+
} else {
|
|
1517
|
+
wsUrl = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent`;
|
|
1518
|
+
headers = {
|
|
1519
|
+
headers: {
|
|
1520
|
+
"x-goog-api-key": this.options.apiKey || "",
|
|
1521
|
+
"Content-Type": "application/json"
|
|
1547
1522
|
}
|
|
1548
|
-
}
|
|
1549
|
-
this.log("
|
|
1523
|
+
};
|
|
1524
|
+
this.log("Using Live API authentication with API key");
|
|
1525
|
+
}
|
|
1526
|
+
this.log("Connecting to:", wsUrl);
|
|
1527
|
+
this.ws = new WebSocket(wsUrl, void 0, headers);
|
|
1528
|
+
this.connectionManager.setWebSocket(this.ws);
|
|
1529
|
+
this.setupEventListeners();
|
|
1530
|
+
await this.connectionManager.waitForOpen();
|
|
1531
|
+
if (this.isResuming && this.sessionHandle) {
|
|
1532
|
+
await this.sendSessionResumption();
|
|
1533
|
+
} else {
|
|
1534
|
+
this.sendInitialConfig();
|
|
1535
|
+
this.sessionStartTime = Date.now();
|
|
1536
|
+
this.sessionId = randomUUID();
|
|
1537
|
+
}
|
|
1538
|
+
await this.waitForSessionCreated();
|
|
1539
|
+
this.state = "connected";
|
|
1540
|
+
this.emit("session", {
|
|
1541
|
+
state: "connected",
|
|
1542
|
+
config: {
|
|
1550
1543
|
sessionId: this.sessionId,
|
|
1551
1544
|
isResuming: this.isResuming,
|
|
1552
1545
|
toolCount: Object.keys(this.tools || {}).length
|
|
1553
|
-
});
|
|
1554
|
-
if (this.options.sessionConfig?.maxDuration) {
|
|
1555
|
-
this.startSessionDurationMonitor();
|
|
1556
1546
|
}
|
|
1557
|
-
}
|
|
1558
|
-
|
|
1559
|
-
this.
|
|
1560
|
-
|
|
1547
|
+
});
|
|
1548
|
+
this.log("Successfully connected to Gemini Live API", {
|
|
1549
|
+
sessionId: this.sessionId,
|
|
1550
|
+
isResuming: this.isResuming,
|
|
1551
|
+
toolCount: Object.keys(this.tools || {}).length
|
|
1552
|
+
});
|
|
1553
|
+
if (this.options.sessionConfig?.maxDuration) {
|
|
1554
|
+
this.startSessionDurationMonitor();
|
|
1561
1555
|
}
|
|
1562
|
-
}
|
|
1556
|
+
} catch (error) {
|
|
1557
|
+
this.state = "disconnected";
|
|
1558
|
+
this.log("Connection failed", error);
|
|
1559
|
+
throw error;
|
|
1560
|
+
}
|
|
1563
1561
|
}
|
|
1564
1562
|
/**
|
|
1565
1563
|
* Disconnect from the Gemini Live API
|
|
@@ -1597,172 +1595,164 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
1597
1595
|
* Send text to be converted to speech
|
|
1598
1596
|
*/
|
|
1599
1597
|
async speak(input, options) {
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)));
|
|
1606
|
-
}
|
|
1607
|
-
input = Buffer.concat(chunks).toString("utf-8");
|
|
1608
|
-
}
|
|
1609
|
-
if (input.trim().length === 0) {
|
|
1610
|
-
throw this.createAndEmitError("invalid_audio_format" /* INVALID_AUDIO_FORMAT */, "Input text is empty");
|
|
1598
|
+
this.validateConnectionState();
|
|
1599
|
+
if (typeof input !== "string") {
|
|
1600
|
+
const chunks = [];
|
|
1601
|
+
for await (const chunk of input) {
|
|
1602
|
+
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)));
|
|
1611
1603
|
}
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
}
|
|
1627
|
-
};
|
|
1628
|
-
if (options && (options.speaker || options.languageCode || options.responseModalities)) {
|
|
1629
|
-
const updateMessage = {
|
|
1630
|
-
type: "session.update",
|
|
1631
|
-
session: {
|
|
1632
|
-
generation_config: {
|
|
1633
|
-
...options.responseModalities ? { response_modalities: options.responseModalities } : {},
|
|
1634
|
-
speech_config: {
|
|
1635
|
-
...options.languageCode ? { language_code: options.languageCode } : {},
|
|
1636
|
-
...options.speaker ? { voice_config: { prebuilt_voice_config: { voice_name: options.speaker } } } : {}
|
|
1604
|
+
input = Buffer.concat(chunks).toString("utf-8");
|
|
1605
|
+
}
|
|
1606
|
+
if (input.trim().length === 0) {
|
|
1607
|
+
throw this.createAndEmitError("invalid_audio_format" /* INVALID_AUDIO_FORMAT */, "Input text is empty");
|
|
1608
|
+
}
|
|
1609
|
+
this.addToContext("user", input);
|
|
1610
|
+
const textMessage = {
|
|
1611
|
+
client_content: {
|
|
1612
|
+
turns: [
|
|
1613
|
+
{
|
|
1614
|
+
role: "user",
|
|
1615
|
+
parts: [
|
|
1616
|
+
{
|
|
1617
|
+
text: input
|
|
1637
1618
|
}
|
|
1619
|
+
]
|
|
1620
|
+
}
|
|
1621
|
+
],
|
|
1622
|
+
turnComplete: true
|
|
1623
|
+
}
|
|
1624
|
+
};
|
|
1625
|
+
if (options && (options.speaker || options.languageCode || options.responseModalities)) {
|
|
1626
|
+
const updateMessage = {
|
|
1627
|
+
type: "session.update",
|
|
1628
|
+
session: {
|
|
1629
|
+
generation_config: {
|
|
1630
|
+
...options.responseModalities ? { response_modalities: options.responseModalities } : {},
|
|
1631
|
+
speech_config: {
|
|
1632
|
+
...options.languageCode ? { language_code: options.languageCode } : {},
|
|
1633
|
+
...options.speaker ? { voice_config: { prebuilt_voice_config: { voice_name: options.speaker } } } : {}
|
|
1638
1634
|
}
|
|
1639
1635
|
}
|
|
1640
|
-
};
|
|
1641
|
-
try {
|
|
1642
|
-
this.sendEvent("session.update", updateMessage);
|
|
1643
|
-
this.log("Applied per-turn runtime options", options);
|
|
1644
|
-
} catch (error) {
|
|
1645
|
-
this.log("Failed to apply per-turn runtime options", error);
|
|
1646
1636
|
}
|
|
1647
|
-
}
|
|
1637
|
+
};
|
|
1648
1638
|
try {
|
|
1649
|
-
this.sendEvent("
|
|
1650
|
-
this.log("
|
|
1639
|
+
this.sendEvent("session.update", updateMessage);
|
|
1640
|
+
this.log("Applied per-turn runtime options", options);
|
|
1651
1641
|
} catch (error) {
|
|
1652
|
-
this.log("Failed to
|
|
1653
|
-
throw this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to send text message", error);
|
|
1642
|
+
this.log("Failed to apply per-turn runtime options", error);
|
|
1654
1643
|
}
|
|
1655
|
-
}
|
|
1644
|
+
}
|
|
1645
|
+
try {
|
|
1646
|
+
this.sendEvent("client_content", textMessage);
|
|
1647
|
+
this.log("Text message sent", { text: input });
|
|
1648
|
+
} catch (error) {
|
|
1649
|
+
this.log("Failed to send text message", error);
|
|
1650
|
+
throw this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to send text message", error);
|
|
1651
|
+
}
|
|
1656
1652
|
}
|
|
1657
1653
|
/**
|
|
1658
1654
|
* Send audio stream for processing
|
|
1659
1655
|
*/
|
|
1660
1656
|
async send(audioData) {
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
}
|
|
1688
|
-
}, "gemini-live.send")();
|
|
1657
|
+
this.validateConnectionState();
|
|
1658
|
+
if ("readable" in audioData && typeof audioData.on === "function") {
|
|
1659
|
+
const stream = audioData;
|
|
1660
|
+
stream.on("data", (chunk) => {
|
|
1661
|
+
try {
|
|
1662
|
+
const base64Audio = this.audioStreamManager.processAudioChunk(chunk);
|
|
1663
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "realtime");
|
|
1664
|
+
this.sendEvent("realtime_input", message);
|
|
1665
|
+
} catch (error) {
|
|
1666
|
+
this.log("Failed to process audio chunk", error);
|
|
1667
|
+
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to process audio chunk", error);
|
|
1668
|
+
}
|
|
1669
|
+
});
|
|
1670
|
+
stream.on("error", (error) => {
|
|
1671
|
+
this.log("Audio stream error", error);
|
|
1672
|
+
this.createAndEmitError("audio_stream_error" /* AUDIO_STREAM_ERROR */, "Audio stream error", error);
|
|
1673
|
+
});
|
|
1674
|
+
stream.on("end", () => {
|
|
1675
|
+
this.log("Audio stream ended");
|
|
1676
|
+
});
|
|
1677
|
+
} else {
|
|
1678
|
+
const validateAudio = this.audioStreamManager.validateAndConvertAudioInput(audioData);
|
|
1679
|
+
const base64Audio = this.audioStreamManager.int16ArrayToBase64(validateAudio);
|
|
1680
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "realtime");
|
|
1681
|
+
this.sendEvent("realtime_input", message);
|
|
1682
|
+
}
|
|
1689
1683
|
}
|
|
1690
1684
|
/**
|
|
1691
1685
|
* Process speech from audio stream (traditional STT interface)
|
|
1692
1686
|
*/
|
|
1693
1687
|
async listen(audioStream, _options) {
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
this.log("Received transcription text:", { text: data.text, total: transcriptionText });
|
|
1701
|
-
}
|
|
1702
|
-
};
|
|
1703
|
-
const onError = (error) => {
|
|
1704
|
-
throw new Error(`Transcription failed: ${error.message}`);
|
|
1705
|
-
};
|
|
1706
|
-
const onSession = (data) => {
|
|
1707
|
-
if (data.state === "disconnected") {
|
|
1708
|
-
throw new Error("Session disconnected during transcription");
|
|
1709
|
-
}
|
|
1710
|
-
};
|
|
1711
|
-
this.on("writing", onWriting);
|
|
1712
|
-
this.on("error", onError);
|
|
1713
|
-
this.on("session", onSession);
|
|
1714
|
-
try {
|
|
1715
|
-
const result = await this.audioStreamManager.handleAudioTranscription(
|
|
1716
|
-
audioStream,
|
|
1717
|
-
(base64Audio) => {
|
|
1718
|
-
return new Promise((resolve, reject) => {
|
|
1719
|
-
try {
|
|
1720
|
-
const message = this.audioStreamManager.createAudioMessage(base64Audio, "input");
|
|
1721
|
-
const cleanup = () => {
|
|
1722
|
-
this.off("turnComplete", onTurnComplete);
|
|
1723
|
-
this.off("error", onErr);
|
|
1724
|
-
};
|
|
1725
|
-
const onTurnComplete = () => {
|
|
1726
|
-
cleanup();
|
|
1727
|
-
resolve(transcriptionText.trim());
|
|
1728
|
-
};
|
|
1729
|
-
const onErr = (e) => {
|
|
1730
|
-
cleanup();
|
|
1731
|
-
reject(new Error(e.message));
|
|
1732
|
-
};
|
|
1733
|
-
this.on("turnComplete", onTurnComplete);
|
|
1734
|
-
this.on("error", onErr);
|
|
1735
|
-
this.sendEvent("client_content", message);
|
|
1736
|
-
this.log("Sent audio for transcription");
|
|
1737
|
-
} catch (err) {
|
|
1738
|
-
reject(err);
|
|
1739
|
-
}
|
|
1740
|
-
});
|
|
1741
|
-
},
|
|
1742
|
-
(error) => {
|
|
1743
|
-
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Audio transcription failed", error);
|
|
1744
|
-
}
|
|
1745
|
-
);
|
|
1746
|
-
return result;
|
|
1747
|
-
} finally {
|
|
1748
|
-
this.off("writing", onWriting);
|
|
1749
|
-
this.off("error", onError);
|
|
1750
|
-
this.off("session", onSession);
|
|
1688
|
+
this.validateConnectionState();
|
|
1689
|
+
let transcriptionText = "";
|
|
1690
|
+
const onWriting = (data) => {
|
|
1691
|
+
if (data.role === "user") {
|
|
1692
|
+
transcriptionText += data.text;
|
|
1693
|
+
this.log("Received transcription text:", { text: data.text, total: transcriptionText });
|
|
1751
1694
|
}
|
|
1752
|
-
}
|
|
1695
|
+
};
|
|
1696
|
+
const onError = (error) => {
|
|
1697
|
+
throw new Error(`Transcription failed: ${error.message}`);
|
|
1698
|
+
};
|
|
1699
|
+
const onSession = (data) => {
|
|
1700
|
+
if (data.state === "disconnected") {
|
|
1701
|
+
throw new Error("Session disconnected during transcription");
|
|
1702
|
+
}
|
|
1703
|
+
};
|
|
1704
|
+
this.on("writing", onWriting);
|
|
1705
|
+
this.on("error", onError);
|
|
1706
|
+
this.on("session", onSession);
|
|
1707
|
+
try {
|
|
1708
|
+
const result = await this.audioStreamManager.handleAudioTranscription(
|
|
1709
|
+
audioStream,
|
|
1710
|
+
(base64Audio) => {
|
|
1711
|
+
return new Promise((resolve, reject) => {
|
|
1712
|
+
try {
|
|
1713
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "input");
|
|
1714
|
+
const cleanup = () => {
|
|
1715
|
+
this.off("turnComplete", onTurnComplete);
|
|
1716
|
+
this.off("error", onErr);
|
|
1717
|
+
};
|
|
1718
|
+
const onTurnComplete = () => {
|
|
1719
|
+
cleanup();
|
|
1720
|
+
resolve(transcriptionText.trim());
|
|
1721
|
+
};
|
|
1722
|
+
const onErr = (e) => {
|
|
1723
|
+
cleanup();
|
|
1724
|
+
reject(new Error(e.message));
|
|
1725
|
+
};
|
|
1726
|
+
this.on("turnComplete", onTurnComplete);
|
|
1727
|
+
this.on("error", onErr);
|
|
1728
|
+
this.sendEvent("client_content", message);
|
|
1729
|
+
this.log("Sent audio for transcription");
|
|
1730
|
+
} catch (err) {
|
|
1731
|
+
reject(err);
|
|
1732
|
+
}
|
|
1733
|
+
});
|
|
1734
|
+
},
|
|
1735
|
+
(error) => {
|
|
1736
|
+
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Audio transcription failed", error);
|
|
1737
|
+
}
|
|
1738
|
+
);
|
|
1739
|
+
return result;
|
|
1740
|
+
} finally {
|
|
1741
|
+
this.off("writing", onWriting);
|
|
1742
|
+
this.off("error", onError);
|
|
1743
|
+
this.off("session", onSession);
|
|
1744
|
+
}
|
|
1753
1745
|
}
|
|
1754
1746
|
/**
|
|
1755
1747
|
* Get available speakers/voices
|
|
1756
1748
|
*/
|
|
1757
1749
|
async getSpeakers() {
|
|
1758
|
-
return
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
];
|
|
1765
|
-
}, "gemini-live.getSpeakers")();
|
|
1750
|
+
return [
|
|
1751
|
+
{ voiceId: "Puck", description: "Conversational, friendly" },
|
|
1752
|
+
{ voiceId: "Charon", description: "Deep, authoritative" },
|
|
1753
|
+
{ voiceId: "Kore", description: "Neutral, professional" },
|
|
1754
|
+
{ voiceId: "Fenrir", description: "Warm, approachable" }
|
|
1755
|
+
];
|
|
1766
1756
|
}
|
|
1767
1757
|
/**
|
|
1768
1758
|
* Resume a previous session using a session handle
|
|
@@ -2390,13 +2380,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
2390
2380
|
let result;
|
|
2391
2381
|
if (tool.execute) {
|
|
2392
2382
|
this.log("Executing tool", { toolName, toolArgs });
|
|
2393
|
-
result = await tool.execute(
|
|
2394
|
-
{ context: toolArgs, runtimeContext: this.runtimeContext },
|
|
2395
|
-
{
|
|
2396
|
-
toolCallId: toolId,
|
|
2397
|
-
messages: []
|
|
2398
|
-
}
|
|
2399
|
-
);
|
|
2383
|
+
result = await tool.execute(toolArgs, { requestContext: this.requestContext });
|
|
2400
2384
|
this.log("Tool executed successfully", { toolName, result });
|
|
2401
2385
|
} else {
|
|
2402
2386
|
this.log("Tool has no execute function", { toolName });
|
|
@@ -2673,14 +2657,14 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
2673
2657
|
* inputSchema: z.object({
|
|
2674
2658
|
* location: z.string().describe("The city and state, e.g. San Francisco, CA"),
|
|
2675
2659
|
* }),
|
|
2676
|
-
* execute: async (
|
|
2660
|
+
* execute: async (inputData) => {
|
|
2677
2661
|
* // Fetch weather data from an API
|
|
2678
2662
|
* const response = await fetch(
|
|
2679
|
-
* `https://api.weather.com?location=${encodeURIComponent(
|
|
2663
|
+
* `https://api.weather.com?location=${encodeURIComponent(inputData.location)}`,
|
|
2680
2664
|
* );
|
|
2681
2665
|
* const data = await response.json();
|
|
2682
2666
|
* return {
|
|
2683
|
-
* message: `The current temperature in ${
|
|
2667
|
+
* message: `The current temperature in ${inputData.location} is ${data.temperature}°F with ${data.conditions}.`,
|
|
2684
2668
|
* };
|
|
2685
2669
|
* },
|
|
2686
2670
|
* });
|
|
@@ -2698,7 +2682,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
2698
2682
|
* Get the current tools configured for this voice instance
|
|
2699
2683
|
* @returns Object containing the current tools
|
|
2700
2684
|
*/
|
|
2701
|
-
|
|
2685
|
+
listTools() {
|
|
2702
2686
|
return this.tools;
|
|
2703
2687
|
}
|
|
2704
2688
|
log(message, ...args) {
|