@mastra/voice-google-gemini-live 0.0.0-fix-backport-setserver-20251201151948 → 0.0.0-fix-request-context-as-query-key-20251209093005
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +118 -38
- package/README.md +3 -3
- package/dist/index.cjs +196 -212
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +7 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +196 -212
- package/dist/index.js.map +1 -1
- package/dist/managers/AudioStreamManager.d.ts +1 -1
- package/dist/managers/AudioStreamManager.d.ts.map +1 -1
- package/dist/managers/EventManager.d.ts +1 -1
- package/dist/managers/EventManager.d.ts.map +1 -1
- package/dist/managers/SessionManager.d.ts +1 -1
- package/dist/managers/SessionManager.d.ts.map +1 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +11 -13
package/dist/index.d.ts
CHANGED
|
@@ -74,7 +74,7 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
74
74
|
private isResuming;
|
|
75
75
|
private sessionDurationTimeout?;
|
|
76
76
|
private tools?;
|
|
77
|
-
private
|
|
77
|
+
private requestContext?;
|
|
78
78
|
private options;
|
|
79
79
|
/**
|
|
80
80
|
* Normalize configuration to ensure proper VoiceConfig format
|
|
@@ -162,8 +162,8 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
162
162
|
/**
|
|
163
163
|
* Establish connection to the Gemini Live API
|
|
164
164
|
*/
|
|
165
|
-
connect({
|
|
166
|
-
|
|
165
|
+
connect({ requestContext }?: {
|
|
166
|
+
requestContext?: any;
|
|
167
167
|
}): Promise<void>;
|
|
168
168
|
/**
|
|
169
169
|
* Disconnect from the Gemini Live API
|
|
@@ -400,14 +400,14 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
400
400
|
* inputSchema: z.object({
|
|
401
401
|
* location: z.string().describe("The city and state, e.g. San Francisco, CA"),
|
|
402
402
|
* }),
|
|
403
|
-
* execute: async (
|
|
403
|
+
* execute: async (inputData) => {
|
|
404
404
|
* // Fetch weather data from an API
|
|
405
405
|
* const response = await fetch(
|
|
406
|
-
* `https://api.weather.com?location=${encodeURIComponent(
|
|
406
|
+
* `https://api.weather.com?location=${encodeURIComponent(inputData.location)}`,
|
|
407
407
|
* );
|
|
408
408
|
* const data = await response.json();
|
|
409
409
|
* return {
|
|
410
|
-
* message: `The current temperature in ${
|
|
410
|
+
* message: `The current temperature in ${inputData.location} is ${data.temperature}°F with ${data.conditions}.`,
|
|
411
411
|
* };
|
|
412
412
|
* },
|
|
413
413
|
* });
|
|
@@ -422,7 +422,7 @@ export declare class GeminiLiveVoice extends MastraVoice<GeminiLiveVoiceConfig,
|
|
|
422
422
|
* Get the current tools configured for this voice instance
|
|
423
423
|
* @returns Object containing the current tools
|
|
424
424
|
*/
|
|
425
|
-
|
|
425
|
+
listTools(): ToolsInput | undefined;
|
|
426
426
|
private log;
|
|
427
427
|
/**
|
|
428
428
|
* Convert Zod schema to JSON Schema for tool parameters
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAKtE,OAAO,KAAK,EACV,qBAAqB,EACrB,sBAAsB,EACtB,kBAAkB,EAMlB,mBAAmB,EAEpB,MAAM,SAAS,CAAC;AAYjB;;GAEG;AAGH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,qBAAa,eAAgB,SAAQ,WAAW,CAC9C,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,UAAU,EACV,kBAAkB,CACnB;IACC,OAAO,CAAC,EAAE,CAAC,CAAS;IACpB,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,KAAK,CAAgD;IAC7D,OAAO,CAAC,aAAa,CAAC,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAU;IAChC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,KAAK,CAAiB;IAG9B,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,WAAW,CAAc;IAGjC,OAAO,CAAC,kBAAkB,CAAqB;IAG/C,OAAO,CAAC,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,gBAAgB,CAAC,CAAS;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,sBAAsB,CAAC,CAAiB;IAGhD,OAAO,CAAC,KAAK,CAAC,CAAa;IAC3B,OAAO,CAAC,cAAc,CAAC,CAAM;IAG7B,OAAO,CAAC,OAAO,CAAwB;IAEvC;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,eAAe;IAwB9B;;;;OAIG;gBACS,MAAM,GAAE,WAAW,CAAC,qBAAqB,CAAC,GAAG,qBAA0B;IA0DnF;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,EAAE,CAAC,CAAC,SAAS,cAAc,EACzB,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,cAAc,EAC1B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IASP;;;;OAIG;IACH,IAAI,CAAC,CAAC,SAAS,cAAc,EAC3B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;OAGG;IACH,OAAO,CAAC,IAAI;IAoCZ;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAoB7B;;;OAGG;IACH,oBAAoB,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAS9C;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAO1B;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAU/B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAU9B;;OAEG;IACG,OAAO,CAAC,EAAE,cAAc,EAAE,GAAE;QAAE,cAAc,CAAC,EAAE,GAAG,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAKtE,OAAO,KAAK,EACV,qBAAqB,EACrB,sBAAsB,EACtB,kBAAkB,EAMlB,mBAAmB,EAEpB,MAAM,SAAS,CAAC;AAYjB;;GAEG;AAGH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,qBAAa,eAAgB,SAAQ,WAAW,CAC9C,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,UAAU,EACV,kBAAkB,CACnB;IACC,OAAO,CAAC,EAAE,CAAC,CAAS;IACpB,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,KAAK,CAAgD;IAC7D,OAAO,CAAC,aAAa,CAAC,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAU;IAChC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,KAAK,CAAiB;IAG9B,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,WAAW,CAAc;IAGjC,OAAO,CAAC,kBAAkB,CAAqB;IAG/C,OAAO,CAAC,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,gBAAgB,CAAC,CAAS;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,sBAAsB,CAAC,CAAiB;IAGhD,OAAO,CAAC,KAAK,CAAC,CAAa;IAC3B,OAAO,CAAC,cAAc,CAAC,CAAM;IAG7B,OAAO,CAAC,OAAO,CAAwB;IAEvC;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,eAAe;IAwB9B;;;;OAIG;gBACS,MAAM,GAAE,WAAW,CAAC,qBAAqB,CAAC,GAAG,qBAA0B;IA0DnF;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,EAAE,CAAC,CAAC,SAAS,cAAc,EACzB,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,cAAc,EAC1B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IASP;;;;OAIG;IACH,IAAI,CAAC,CAAC,SAAS,cAAc,EAC3B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;OAGG;IACH,OAAO,CAAC,IAAI;IAoCZ;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAoB7B;;;OAGG;IACH,oBAAoB,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAS9C;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAO1B;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAU/B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAU9B;;OAEG;IACG,OAAO,CAAC,EAAE,cAAc,EAAE,GAAE;QAAE,cAAc,CAAC,EAAE,GAAG,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAwF/E;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAiDjC;;OAEG;IACG,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,cAAc,EAAE,OAAO,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC;IAuEnG;;OAEG;IACG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,cAAc,GAAG,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAiCxE;;OAEG;IACG,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,cAAc,EAAE,QAAQ,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,MAAM,CAAC;IAqFpG;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAU9E;;OAEG;IACG,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BtG;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACG,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA6MhF;;OAEG;IACH,kBAAkB,IAAI,cAAc,GAAG,WAAW;IAIlD;;OAEG;IACH,WAAW,IAAI,OAAO;IAItB;;;OAGG;IACH,uBAAuB,IAAI,MAAM,CAAC,cAAc,GAAG,IAAI;IAIvD;;OAEG;IACH,gBAAgB,IAAI,MAAM,GAAG,SAAS;IAKtC;;OAEG;IACH,cAAc,IAAI;QAChB,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,IAAI,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,mBAAmB,CAAC;QAC7B,WAAW,EAAE,MAAM,CAAC;KACrB;IAYD;;OAEG;IACH,iBAAiB,IAAI,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IAIhF;;OAEG;IACH,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI;IAI/D;;OAEG;IACH,YAAY,IAAI,IAAI;IAKpB;;OAEG;IACH,gBAAgB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAQxC;;;OAGG;YACW,qBAAqB;IA4BnC;;;OAGG;IACH,OAAO,CAAC,2BAA2B;IAqCnC;;;OAGG;IACH,OAAO,CAAC,aAAa;IAmBrB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAKvB;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IA6C3B;;;OAGG;YACW,mBAAmB;IA0EjC;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAwB3B;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAY5B;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAsI3B;;;OAGG;YACW,cAAc;IA4B5B;;;OAGG;YACW,qBAAqB;IA8EnC;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAWzB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAMxB;;;OAGG;IACH,OAAO,CAAC,WAAW;IAcnB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAYzB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAIzB;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAkB9B;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAmIzB;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAqD7B;;;;OAIG;YACW,cAAc;IAO5B;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAI5B;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAI5B;;;OAGG;IACH,OAAO,CAAC,SAAS;IAkCjB;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,QAAQ,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAKjC;;;OAGG;IACH,SAAS,IAAI,UAAU,GAAG,SAAS;IAInC,OAAO,CAAC,GAAG;IAMX;;;OAGG;IACH,OAAO,CAAC,4BAA4B;IAiCpC;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IA2DjC;;OAEG;IACH,KAAK,IAAI,IAAI;IAIb;;OAEG;IACG,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAO/D;;;OAGG;IACH,eAAe,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI;CAM7C"}
|
package/dist/index.js
CHANGED
|
@@ -1260,7 +1260,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
1260
1260
|
sessionDurationTimeout;
|
|
1261
1261
|
// Tool integration properties
|
|
1262
1262
|
tools;
|
|
1263
|
-
|
|
1263
|
+
requestContext;
|
|
1264
1264
|
// Store the configuration options
|
|
1265
1265
|
options;
|
|
1266
1266
|
/**
|
|
@@ -1497,70 +1497,68 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
1497
1497
|
/**
|
|
1498
1498
|
* Establish connection to the Gemini Live API
|
|
1499
1499
|
*/
|
|
1500
|
-
async connect({
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
headers
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
"Content-Type": "application/json"
|
|
1524
|
-
}
|
|
1525
|
-
};
|
|
1526
|
-
this.log("Using Live API authentication with API key");
|
|
1527
|
-
}
|
|
1528
|
-
this.log("Connecting to:", wsUrl);
|
|
1529
|
-
this.ws = new WebSocket(wsUrl, void 0, headers);
|
|
1530
|
-
this.connectionManager.setWebSocket(this.ws);
|
|
1531
|
-
this.setupEventListeners();
|
|
1532
|
-
await this.connectionManager.waitForOpen();
|
|
1533
|
-
if (this.isResuming && this.sessionHandle) {
|
|
1534
|
-
await this.sendSessionResumption();
|
|
1535
|
-
} else {
|
|
1536
|
-
this.sendInitialConfig();
|
|
1537
|
-
this.sessionStartTime = Date.now();
|
|
1538
|
-
this.sessionId = randomUUID();
|
|
1539
|
-
}
|
|
1540
|
-
await this.waitForSessionCreated();
|
|
1541
|
-
this.state = "connected";
|
|
1542
|
-
this.emit("session", {
|
|
1543
|
-
state: "connected",
|
|
1544
|
-
config: {
|
|
1545
|
-
sessionId: this.sessionId,
|
|
1546
|
-
isResuming: this.isResuming,
|
|
1547
|
-
toolCount: Object.keys(this.tools || {}).length
|
|
1500
|
+
async connect({ requestContext } = {}) {
|
|
1501
|
+
if (this.state === "connected") {
|
|
1502
|
+
this.log("Already connected to Gemini Live API");
|
|
1503
|
+
return;
|
|
1504
|
+
}
|
|
1505
|
+
this.requestContext = requestContext;
|
|
1506
|
+
this.emit("session", { state: "connecting" });
|
|
1507
|
+
try {
|
|
1508
|
+
let wsUrl;
|
|
1509
|
+
let headers = {};
|
|
1510
|
+
if (this.options.vertexAI) {
|
|
1511
|
+
const location = this.getVertexLocation();
|
|
1512
|
+
wsUrl = `wss://${location}-aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1beta1.LlmBidiService/BidiGenerateContent`;
|
|
1513
|
+
await this.authManager.initialize();
|
|
1514
|
+
const accessToken = await this.authManager.getAccessToken();
|
|
1515
|
+
headers = { headers: { Authorization: `Bearer ${accessToken}` } };
|
|
1516
|
+
this.log("Using Vertex AI authentication with OAuth token");
|
|
1517
|
+
} else {
|
|
1518
|
+
wsUrl = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent`;
|
|
1519
|
+
headers = {
|
|
1520
|
+
headers: {
|
|
1521
|
+
"x-goog-api-key": this.options.apiKey || "",
|
|
1522
|
+
"Content-Type": "application/json"
|
|
1548
1523
|
}
|
|
1549
|
-
}
|
|
1550
|
-
this.log("
|
|
1524
|
+
};
|
|
1525
|
+
this.log("Using Live API authentication with API key");
|
|
1526
|
+
}
|
|
1527
|
+
this.log("Connecting to:", wsUrl);
|
|
1528
|
+
this.ws = new WebSocket(wsUrl, void 0, headers);
|
|
1529
|
+
this.connectionManager.setWebSocket(this.ws);
|
|
1530
|
+
this.setupEventListeners();
|
|
1531
|
+
await this.connectionManager.waitForOpen();
|
|
1532
|
+
if (this.isResuming && this.sessionHandle) {
|
|
1533
|
+
await this.sendSessionResumption();
|
|
1534
|
+
} else {
|
|
1535
|
+
this.sendInitialConfig();
|
|
1536
|
+
this.sessionStartTime = Date.now();
|
|
1537
|
+
this.sessionId = randomUUID();
|
|
1538
|
+
}
|
|
1539
|
+
await this.waitForSessionCreated();
|
|
1540
|
+
this.state = "connected";
|
|
1541
|
+
this.emit("session", {
|
|
1542
|
+
state: "connected",
|
|
1543
|
+
config: {
|
|
1551
1544
|
sessionId: this.sessionId,
|
|
1552
1545
|
isResuming: this.isResuming,
|
|
1553
1546
|
toolCount: Object.keys(this.tools || {}).length
|
|
1554
|
-
});
|
|
1555
|
-
if (this.options.sessionConfig?.maxDuration) {
|
|
1556
|
-
this.startSessionDurationMonitor();
|
|
1557
1547
|
}
|
|
1558
|
-
}
|
|
1559
|
-
|
|
1560
|
-
this.
|
|
1561
|
-
|
|
1548
|
+
});
|
|
1549
|
+
this.log("Successfully connected to Gemini Live API", {
|
|
1550
|
+
sessionId: this.sessionId,
|
|
1551
|
+
isResuming: this.isResuming,
|
|
1552
|
+
toolCount: Object.keys(this.tools || {}).length
|
|
1553
|
+
});
|
|
1554
|
+
if (this.options.sessionConfig?.maxDuration) {
|
|
1555
|
+
this.startSessionDurationMonitor();
|
|
1562
1556
|
}
|
|
1563
|
-
}
|
|
1557
|
+
} catch (error) {
|
|
1558
|
+
this.state = "disconnected";
|
|
1559
|
+
this.log("Connection failed", error);
|
|
1560
|
+
throw error;
|
|
1561
|
+
}
|
|
1564
1562
|
}
|
|
1565
1563
|
/**
|
|
1566
1564
|
* Disconnect from the Gemini Live API
|
|
@@ -1598,172 +1596,164 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
1598
1596
|
* Send text to be converted to speech
|
|
1599
1597
|
*/
|
|
1600
1598
|
async speak(input, options) {
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)));
|
|
1607
|
-
}
|
|
1608
|
-
input = Buffer.concat(chunks).toString("utf-8");
|
|
1609
|
-
}
|
|
1610
|
-
if (input.trim().length === 0) {
|
|
1611
|
-
throw this.createAndEmitError("invalid_audio_format" /* INVALID_AUDIO_FORMAT */, "Input text is empty");
|
|
1599
|
+
this.validateConnectionState();
|
|
1600
|
+
if (typeof input !== "string") {
|
|
1601
|
+
const chunks = [];
|
|
1602
|
+
for await (const chunk of input) {
|
|
1603
|
+
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)));
|
|
1612
1604
|
}
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
}
|
|
1628
|
-
};
|
|
1629
|
-
if (options && (options.speaker || options.languageCode || options.responseModalities)) {
|
|
1630
|
-
const updateMessage = {
|
|
1631
|
-
type: "session.update",
|
|
1632
|
-
session: {
|
|
1633
|
-
generation_config: {
|
|
1634
|
-
...options.responseModalities ? { response_modalities: options.responseModalities } : {},
|
|
1635
|
-
speech_config: {
|
|
1636
|
-
...options.languageCode ? { language_code: options.languageCode } : {},
|
|
1637
|
-
...options.speaker ? { voice_config: { prebuilt_voice_config: { voice_name: options.speaker } } } : {}
|
|
1605
|
+
input = Buffer.concat(chunks).toString("utf-8");
|
|
1606
|
+
}
|
|
1607
|
+
if (input.trim().length === 0) {
|
|
1608
|
+
throw this.createAndEmitError("invalid_audio_format" /* INVALID_AUDIO_FORMAT */, "Input text is empty");
|
|
1609
|
+
}
|
|
1610
|
+
this.addToContext("user", input);
|
|
1611
|
+
const textMessage = {
|
|
1612
|
+
client_content: {
|
|
1613
|
+
turns: [
|
|
1614
|
+
{
|
|
1615
|
+
role: "user",
|
|
1616
|
+
parts: [
|
|
1617
|
+
{
|
|
1618
|
+
text: input
|
|
1638
1619
|
}
|
|
1620
|
+
]
|
|
1621
|
+
}
|
|
1622
|
+
],
|
|
1623
|
+
turnComplete: true
|
|
1624
|
+
}
|
|
1625
|
+
};
|
|
1626
|
+
if (options && (options.speaker || options.languageCode || options.responseModalities)) {
|
|
1627
|
+
const updateMessage = {
|
|
1628
|
+
type: "session.update",
|
|
1629
|
+
session: {
|
|
1630
|
+
generation_config: {
|
|
1631
|
+
...options.responseModalities ? { response_modalities: options.responseModalities } : {},
|
|
1632
|
+
speech_config: {
|
|
1633
|
+
...options.languageCode ? { language_code: options.languageCode } : {},
|
|
1634
|
+
...options.speaker ? { voice_config: { prebuilt_voice_config: { voice_name: options.speaker } } } : {}
|
|
1639
1635
|
}
|
|
1640
1636
|
}
|
|
1641
|
-
};
|
|
1642
|
-
try {
|
|
1643
|
-
this.sendEvent("session.update", updateMessage);
|
|
1644
|
-
this.log("Applied per-turn runtime options", options);
|
|
1645
|
-
} catch (error) {
|
|
1646
|
-
this.log("Failed to apply per-turn runtime options", error);
|
|
1647
1637
|
}
|
|
1648
|
-
}
|
|
1638
|
+
};
|
|
1649
1639
|
try {
|
|
1650
|
-
this.sendEvent("
|
|
1651
|
-
this.log("
|
|
1640
|
+
this.sendEvent("session.update", updateMessage);
|
|
1641
|
+
this.log("Applied per-turn runtime options", options);
|
|
1652
1642
|
} catch (error) {
|
|
1653
|
-
this.log("Failed to
|
|
1654
|
-
throw this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to send text message", error);
|
|
1643
|
+
this.log("Failed to apply per-turn runtime options", error);
|
|
1655
1644
|
}
|
|
1656
|
-
}
|
|
1645
|
+
}
|
|
1646
|
+
try {
|
|
1647
|
+
this.sendEvent("client_content", textMessage);
|
|
1648
|
+
this.log("Text message sent", { text: input });
|
|
1649
|
+
} catch (error) {
|
|
1650
|
+
this.log("Failed to send text message", error);
|
|
1651
|
+
throw this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to send text message", error);
|
|
1652
|
+
}
|
|
1657
1653
|
}
|
|
1658
1654
|
/**
|
|
1659
1655
|
* Send audio stream for processing
|
|
1660
1656
|
*/
|
|
1661
1657
|
async send(audioData) {
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
}
|
|
1689
|
-
}, "gemini-live.send")();
|
|
1658
|
+
this.validateConnectionState();
|
|
1659
|
+
if ("readable" in audioData && typeof audioData.on === "function") {
|
|
1660
|
+
const stream = audioData;
|
|
1661
|
+
stream.on("data", (chunk) => {
|
|
1662
|
+
try {
|
|
1663
|
+
const base64Audio = this.audioStreamManager.processAudioChunk(chunk);
|
|
1664
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "realtime");
|
|
1665
|
+
this.sendEvent("realtime_input", message);
|
|
1666
|
+
} catch (error) {
|
|
1667
|
+
this.log("Failed to process audio chunk", error);
|
|
1668
|
+
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to process audio chunk", error);
|
|
1669
|
+
}
|
|
1670
|
+
});
|
|
1671
|
+
stream.on("error", (error) => {
|
|
1672
|
+
this.log("Audio stream error", error);
|
|
1673
|
+
this.createAndEmitError("audio_stream_error" /* AUDIO_STREAM_ERROR */, "Audio stream error", error);
|
|
1674
|
+
});
|
|
1675
|
+
stream.on("end", () => {
|
|
1676
|
+
this.log("Audio stream ended");
|
|
1677
|
+
});
|
|
1678
|
+
} else {
|
|
1679
|
+
const validateAudio = this.audioStreamManager.validateAndConvertAudioInput(audioData);
|
|
1680
|
+
const base64Audio = this.audioStreamManager.int16ArrayToBase64(validateAudio);
|
|
1681
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "realtime");
|
|
1682
|
+
this.sendEvent("realtime_input", message);
|
|
1683
|
+
}
|
|
1690
1684
|
}
|
|
1691
1685
|
/**
|
|
1692
1686
|
* Process speech from audio stream (traditional STT interface)
|
|
1693
1687
|
*/
|
|
1694
1688
|
async listen(audioStream, _options) {
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
this.log("Received transcription text:", { text: data.text, total: transcriptionText });
|
|
1702
|
-
}
|
|
1703
|
-
};
|
|
1704
|
-
const onError = (error) => {
|
|
1705
|
-
throw new Error(`Transcription failed: ${error.message}`);
|
|
1706
|
-
};
|
|
1707
|
-
const onSession = (data) => {
|
|
1708
|
-
if (data.state === "disconnected") {
|
|
1709
|
-
throw new Error("Session disconnected during transcription");
|
|
1710
|
-
}
|
|
1711
|
-
};
|
|
1712
|
-
this.on("writing", onWriting);
|
|
1713
|
-
this.on("error", onError);
|
|
1714
|
-
this.on("session", onSession);
|
|
1715
|
-
try {
|
|
1716
|
-
const result = await this.audioStreamManager.handleAudioTranscription(
|
|
1717
|
-
audioStream,
|
|
1718
|
-
(base64Audio) => {
|
|
1719
|
-
return new Promise((resolve, reject) => {
|
|
1720
|
-
try {
|
|
1721
|
-
const message = this.audioStreamManager.createAudioMessage(base64Audio, "input");
|
|
1722
|
-
const cleanup = () => {
|
|
1723
|
-
this.off("turnComplete", onTurnComplete);
|
|
1724
|
-
this.off("error", onErr);
|
|
1725
|
-
};
|
|
1726
|
-
const onTurnComplete = () => {
|
|
1727
|
-
cleanup();
|
|
1728
|
-
resolve(transcriptionText.trim());
|
|
1729
|
-
};
|
|
1730
|
-
const onErr = (e) => {
|
|
1731
|
-
cleanup();
|
|
1732
|
-
reject(new Error(e.message));
|
|
1733
|
-
};
|
|
1734
|
-
this.on("turnComplete", onTurnComplete);
|
|
1735
|
-
this.on("error", onErr);
|
|
1736
|
-
this.sendEvent("client_content", message);
|
|
1737
|
-
this.log("Sent audio for transcription");
|
|
1738
|
-
} catch (err) {
|
|
1739
|
-
reject(err);
|
|
1740
|
-
}
|
|
1741
|
-
});
|
|
1742
|
-
},
|
|
1743
|
-
(error) => {
|
|
1744
|
-
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Audio transcription failed", error);
|
|
1745
|
-
}
|
|
1746
|
-
);
|
|
1747
|
-
return result;
|
|
1748
|
-
} finally {
|
|
1749
|
-
this.off("writing", onWriting);
|
|
1750
|
-
this.off("error", onError);
|
|
1751
|
-
this.off("session", onSession);
|
|
1689
|
+
this.validateConnectionState();
|
|
1690
|
+
let transcriptionText = "";
|
|
1691
|
+
const onWriting = (data) => {
|
|
1692
|
+
if (data.role === "user") {
|
|
1693
|
+
transcriptionText += data.text;
|
|
1694
|
+
this.log("Received transcription text:", { text: data.text, total: transcriptionText });
|
|
1752
1695
|
}
|
|
1753
|
-
}
|
|
1696
|
+
};
|
|
1697
|
+
const onError = (error) => {
|
|
1698
|
+
throw new Error(`Transcription failed: ${error.message}`);
|
|
1699
|
+
};
|
|
1700
|
+
const onSession = (data) => {
|
|
1701
|
+
if (data.state === "disconnected") {
|
|
1702
|
+
throw new Error("Session disconnected during transcription");
|
|
1703
|
+
}
|
|
1704
|
+
};
|
|
1705
|
+
this.on("writing", onWriting);
|
|
1706
|
+
this.on("error", onError);
|
|
1707
|
+
this.on("session", onSession);
|
|
1708
|
+
try {
|
|
1709
|
+
const result = await this.audioStreamManager.handleAudioTranscription(
|
|
1710
|
+
audioStream,
|
|
1711
|
+
(base64Audio) => {
|
|
1712
|
+
return new Promise((resolve, reject) => {
|
|
1713
|
+
try {
|
|
1714
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "input");
|
|
1715
|
+
const cleanup = () => {
|
|
1716
|
+
this.off("turnComplete", onTurnComplete);
|
|
1717
|
+
this.off("error", onErr);
|
|
1718
|
+
};
|
|
1719
|
+
const onTurnComplete = () => {
|
|
1720
|
+
cleanup();
|
|
1721
|
+
resolve(transcriptionText.trim());
|
|
1722
|
+
};
|
|
1723
|
+
const onErr = (e) => {
|
|
1724
|
+
cleanup();
|
|
1725
|
+
reject(new Error(e.message));
|
|
1726
|
+
};
|
|
1727
|
+
this.on("turnComplete", onTurnComplete);
|
|
1728
|
+
this.on("error", onErr);
|
|
1729
|
+
this.sendEvent("client_content", message);
|
|
1730
|
+
this.log("Sent audio for transcription");
|
|
1731
|
+
} catch (err) {
|
|
1732
|
+
reject(err);
|
|
1733
|
+
}
|
|
1734
|
+
});
|
|
1735
|
+
},
|
|
1736
|
+
(error) => {
|
|
1737
|
+
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Audio transcription failed", error);
|
|
1738
|
+
}
|
|
1739
|
+
);
|
|
1740
|
+
return result;
|
|
1741
|
+
} finally {
|
|
1742
|
+
this.off("writing", onWriting);
|
|
1743
|
+
this.off("error", onError);
|
|
1744
|
+
this.off("session", onSession);
|
|
1745
|
+
}
|
|
1754
1746
|
}
|
|
1755
1747
|
/**
|
|
1756
1748
|
* Get available speakers/voices
|
|
1757
1749
|
*/
|
|
1758
1750
|
async getSpeakers() {
|
|
1759
|
-
return
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
];
|
|
1766
|
-
}, "gemini-live.getSpeakers")();
|
|
1751
|
+
return [
|
|
1752
|
+
{ voiceId: "Puck", description: "Conversational, friendly" },
|
|
1753
|
+
{ voiceId: "Charon", description: "Deep, authoritative" },
|
|
1754
|
+
{ voiceId: "Kore", description: "Neutral, professional" },
|
|
1755
|
+
{ voiceId: "Fenrir", description: "Warm, approachable" }
|
|
1756
|
+
];
|
|
1767
1757
|
}
|
|
1768
1758
|
/**
|
|
1769
1759
|
* Resume a previous session using a session handle
|
|
@@ -2391,13 +2381,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
2391
2381
|
let result;
|
|
2392
2382
|
if (tool.execute) {
|
|
2393
2383
|
this.log("Executing tool", { toolName, toolArgs });
|
|
2394
|
-
result = await tool.execute(
|
|
2395
|
-
{ context: toolArgs, runtimeContext: this.runtimeContext },
|
|
2396
|
-
{
|
|
2397
|
-
toolCallId: toolId,
|
|
2398
|
-
messages: []
|
|
2399
|
-
}
|
|
2400
|
-
);
|
|
2384
|
+
result = await tool.execute(toolArgs, { requestContext: this.requestContext });
|
|
2401
2385
|
this.log("Tool executed successfully", { toolName, result });
|
|
2402
2386
|
} else {
|
|
2403
2387
|
this.log("Tool has no execute function", { toolName });
|
|
@@ -2699,14 +2683,14 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
2699
2683
|
* inputSchema: z.object({
|
|
2700
2684
|
* location: z.string().describe("The city and state, e.g. San Francisco, CA"),
|
|
2701
2685
|
* }),
|
|
2702
|
-
* execute: async (
|
|
2686
|
+
* execute: async (inputData) => {
|
|
2703
2687
|
* // Fetch weather data from an API
|
|
2704
2688
|
* const response = await fetch(
|
|
2705
|
-
* `https://api.weather.com?location=${encodeURIComponent(
|
|
2689
|
+
* `https://api.weather.com?location=${encodeURIComponent(inputData.location)}`,
|
|
2706
2690
|
* );
|
|
2707
2691
|
* const data = await response.json();
|
|
2708
2692
|
* return {
|
|
2709
|
-
* message: `The current temperature in ${
|
|
2693
|
+
* message: `The current temperature in ${inputData.location} is ${data.temperature}°F with ${data.conditions}.`,
|
|
2710
2694
|
* };
|
|
2711
2695
|
* },
|
|
2712
2696
|
* });
|
|
@@ -2724,7 +2708,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
2724
2708
|
* Get the current tools configured for this voice instance
|
|
2725
2709
|
* @returns Object containing the current tools
|
|
2726
2710
|
*/
|
|
2727
|
-
|
|
2711
|
+
listTools() {
|
|
2728
2712
|
return this.tools;
|
|
2729
2713
|
}
|
|
2730
2714
|
log(message, ...args) {
|