@mastra/voice-google-gemini-live 0.0.0-add-save-score-validation-on-stores-20250911031242 → 0.0.0-add-crumb-action-20251028111500
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +142 -3
- package/dist/index.cjs +193 -203
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +193 -203
- package/dist/index.js.map +1 -1
- package/dist/managers/AudioStreamManager.d.ts +1 -1
- package/dist/managers/index.d.ts +11 -11
- package/dist/utils/errors.d.ts +1 -1
- package/package.json +6 -5
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { ToolsInput } from '@mastra/core/agent';
|
|
2
2
|
import { MastraVoice } from '@mastra/core/voice';
|
|
3
3
|
import type { VoiceEventType, VoiceConfig } from '@mastra/core/voice';
|
|
4
|
-
import type { GeminiLiveVoiceConfig, GeminiLiveVoiceOptions, GeminiLiveEventMap, GeminiSessionConfig } from './types';
|
|
4
|
+
import type { GeminiLiveVoiceConfig, GeminiLiveVoiceOptions, GeminiLiveEventMap, GeminiSessionConfig } from './types.js';
|
|
5
5
|
/**
|
|
6
6
|
* Helper class for consistent error handling
|
|
7
7
|
*/
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAKtE,OAAO,KAAK,EACV,qBAAqB,EACrB,sBAAsB,EACtB,kBAAkB,EAMlB,mBAAmB,EAEpB,MAAM,SAAS,CAAC;AAYjB;;GAEG;AAGH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,qBAAa,eAAgB,SAAQ,WAAW,CAC9C,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,UAAU,EACV,kBAAkB,CACnB;IACC,OAAO,CAAC,EAAE,CAAC,CAAS;IACpB,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,KAAK,CAAgD;IAC7D,OAAO,CAAC,aAAa,CAAC,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAU;IAChC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,KAAK,CAAiB;IAG9B,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,WAAW,CAAc;IAGjC,OAAO,CAAC,kBAAkB,CAAqB;IAG/C,OAAO,CAAC,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,gBAAgB,CAAC,CAAS;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,sBAAsB,CAAC,CAAiB;IAGhD,OAAO,CAAC,KAAK,CAAC,CAAa;IAC3B,OAAO,CAAC,cAAc,CAAC,CAAM;IAG7B,OAAO,CAAC,OAAO,CAAwB;IAEvC;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,eAAe;IAwB9B;;;;OAIG;gBACS,MAAM,GAAE,WAAW,CAAC,qBAAqB,CAAC,GAAG,qBAA0B;IA0DnF;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,EAAE,CAAC,CAAC,SAAS,cAAc,EACzB,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,cAAc,EAC1B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IASP;;;;OAIG;IACH,IAAI,CAAC,CAAC,SAAS,cAAc,EAC3B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;OAGG;IACH,OAAO,CAAC,IAAI;IAoCZ;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAoB7B;;;OAGG;IACH,oBAAoB,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAS9C;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAO1B;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAU/B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAU9B;;OAEG;IACG,OAAO,CAAC,EAAE,cAAc,EAAE,GAAE;QAAE,cAAc,CAAC,EAAE,GAAG,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAKtE,OAAO,KAAK,EACV,qBAAqB,EACrB,sBAAsB,EACtB,kBAAkB,EAMlB,mBAAmB,EAEpB,MAAM,SAAS,CAAC;AAYjB;;GAEG;AAGH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,qBAAa,eAAgB,SAAQ,WAAW,CAC9C,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,UAAU,EACV,kBAAkB,CACnB;IACC,OAAO,CAAC,EAAE,CAAC,CAAS;IACpB,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,KAAK,CAAgD;IAC7D,OAAO,CAAC,aAAa,CAAC,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAU;IAChC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,KAAK,CAAiB;IAG9B,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,WAAW,CAAc;IAGjC,OAAO,CAAC,kBAAkB,CAAqB;IAG/C,OAAO,CAAC,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,gBAAgB,CAAC,CAAS;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,sBAAsB,CAAC,CAAiB;IAGhD,OAAO,CAAC,KAAK,CAAC,CAAa;IAC3B,OAAO,CAAC,cAAc,CAAC,CAAM;IAG7B,OAAO,CAAC,OAAO,CAAwB;IAEvC;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,eAAe;IAwB9B;;;;OAIG;gBACS,MAAM,GAAE,WAAW,CAAC,qBAAqB,CAAC,GAAG,qBAA0B;IA0DnF;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,EAAE,CAAC,CAAC,SAAS,cAAc,EACzB,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,cAAc,EAC1B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IASP;;;;OAIG;IACH,IAAI,CAAC,CAAC,SAAS,cAAc,EAC3B,KAAK,EAAE,CAAC,EACR,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,MAAM,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,IAAI,GAC7F,IAAI;IAUP;;;OAGG;IACH,OAAO,CAAC,IAAI;IAoCZ;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAoB7B;;;OAGG;IACH,oBAAoB,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAS9C;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAO1B;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAU/B;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAU9B;;OAEG;IACG,OAAO,CAAC,EAAE,cAAc,EAAE,GAAE;QAAE,cAAc,CAAC,EAAE,GAAG,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAuF/E;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAiDjC;;OAEG;IACG,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,cAAc,EAAE,OAAO,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC;IAuEnG;;OAEG;IACG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,cAAc,GAAG,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAiCxE;;OAEG;IACG,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,cAAc,EAAE,QAAQ,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,MAAM,CAAC;IAqFpG;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAU9E;;OAEG;IACG,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BtG;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACG,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,qBAAqB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA6MhF;;OAEG;IACH,kBAAkB,IAAI,cAAc,GAAG,WAAW;IAIlD;;OAEG;IACH,WAAW,IAAI,OAAO;IAItB;;;OAGG;IACH,uBAAuB,IAAI,MAAM,CAAC,cAAc,GAAG,IAAI;IAIvD;;OAEG;IACH,gBAAgB,IAAI,MAAM,GAAG,SAAS;IAKtC;;OAEG;IACH,cAAc,IAAI;QAChB,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,IAAI,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,mBAAmB,CAAC;QAC7B,WAAW,EAAE,MAAM,CAAC;KACrB;IAYD;;OAEG;IACH,iBAAiB,IAAI,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IAIhF;;OAEG;IACH,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI;IAI/D;;OAEG;IACH,YAAY,IAAI,IAAI;IAKpB;;OAEG;IACH,gBAAgB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAQxC;;;OAGG;YACW,qBAAqB;IA4BnC;;;OAGG;IACH,OAAO,CAAC,2BAA2B;IAqCnC;;;OAGG;IACH,OAAO,CAAC,aAAa;IAmBrB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAKvB;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IA6C3B;;;OAGG;YACW,mBAAmB;IA0EjC;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAwB3B;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAY5B;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAiH3B;;;OAGG;YACW,cAAc;IAoF5B;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAWzB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAMxB;;;OAGG;IACH,OAAO,CAAC,WAAW;IAcnB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAYzB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAmIzB;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAqD7B;;;;OAIG;YACW,cAAc;IAO5B;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAI5B;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAI5B;;;OAGG;IACH,OAAO,CAAC,SAAS;IA+BjB;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,QAAQ,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAKjC;;;OAGG;IACH,QAAQ,IAAI,UAAU,GAAG,SAAS;IAIlC,OAAO,CAAC,GAAG;IAMX;;;OAGG;IACH,OAAO,CAAC,4BAA4B;IAiCpC;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IA2DjC;;OAEG;IACH,KAAK,IAAI,IAAI;IAIb;;OAEG;IACG,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAO/D;;;OAGG;IACH,eAAe,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI;CAM7C"}
|
package/dist/index.js
CHANGED
|
@@ -562,7 +562,7 @@ var AudioStreamManager = class {
|
|
|
562
562
|
*/
|
|
563
563
|
log(message, ...args) {
|
|
564
564
|
if (this.debug) {
|
|
565
|
-
console.
|
|
565
|
+
console.info(`[AudioStreamManager] ${message}`, ...args);
|
|
566
566
|
}
|
|
567
567
|
}
|
|
568
568
|
/**
|
|
@@ -841,7 +841,7 @@ var ConnectionManager = class {
|
|
|
841
841
|
*/
|
|
842
842
|
log(message, ...args) {
|
|
843
843
|
if (this.debug) {
|
|
844
|
-
console.
|
|
844
|
+
console.info(`[ConnectionManager] ${message}`, ...args);
|
|
845
845
|
}
|
|
846
846
|
}
|
|
847
847
|
};
|
|
@@ -1115,7 +1115,7 @@ var AuthManager = class {
|
|
|
1115
1115
|
*/
|
|
1116
1116
|
log(message, ...args) {
|
|
1117
1117
|
if (this.config.debug) {
|
|
1118
|
-
console.
|
|
1118
|
+
console.info(`[AuthManager] ${message}`, ...args);
|
|
1119
1119
|
}
|
|
1120
1120
|
}
|
|
1121
1121
|
};
|
|
@@ -1231,7 +1231,7 @@ var EventManager = class {
|
|
|
1231
1231
|
*/
|
|
1232
1232
|
log(message, ...args) {
|
|
1233
1233
|
if (this.debug) {
|
|
1234
|
-
console.
|
|
1234
|
+
console.info(`[EventManager] ${message}`, ...args);
|
|
1235
1235
|
}
|
|
1236
1236
|
}
|
|
1237
1237
|
};
|
|
@@ -1498,68 +1498,66 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
1498
1498
|
* Establish connection to the Gemini Live API
|
|
1499
1499
|
*/
|
|
1500
1500
|
async connect({ runtimeContext } = {}) {
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
headers
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
"Content-Type": "application/json"
|
|
1523
|
-
}
|
|
1524
|
-
};
|
|
1525
|
-
this.log("Using Live API authentication with API key");
|
|
1526
|
-
}
|
|
1527
|
-
this.log("Connecting to:", wsUrl);
|
|
1528
|
-
this.ws = new WebSocket(wsUrl, void 0, headers);
|
|
1529
|
-
this.connectionManager.setWebSocket(this.ws);
|
|
1530
|
-
this.setupEventListeners();
|
|
1531
|
-
await this.connectionManager.waitForOpen();
|
|
1532
|
-
if (this.isResuming && this.sessionHandle) {
|
|
1533
|
-
await this.sendSessionResumption();
|
|
1534
|
-
} else {
|
|
1535
|
-
this.sendInitialConfig();
|
|
1536
|
-
this.sessionStartTime = Date.now();
|
|
1537
|
-
this.sessionId = randomUUID();
|
|
1538
|
-
}
|
|
1539
|
-
await this.waitForSessionCreated();
|
|
1540
|
-
this.state = "connected";
|
|
1541
|
-
this.emit("session", {
|
|
1542
|
-
state: "connected",
|
|
1543
|
-
config: {
|
|
1544
|
-
sessionId: this.sessionId,
|
|
1545
|
-
isResuming: this.isResuming,
|
|
1546
|
-
toolCount: Object.keys(this.tools || {}).length
|
|
1501
|
+
if (this.state === "connected") {
|
|
1502
|
+
this.log("Already connected to Gemini Live API");
|
|
1503
|
+
return;
|
|
1504
|
+
}
|
|
1505
|
+
this.runtimeContext = runtimeContext;
|
|
1506
|
+
this.emit("session", { state: "connecting" });
|
|
1507
|
+
try {
|
|
1508
|
+
let wsUrl;
|
|
1509
|
+
let headers = {};
|
|
1510
|
+
if (this.options.vertexAI) {
|
|
1511
|
+
wsUrl = `wss://${this.options.location}-aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1beta1.PredictionService.ServerStreamingPredict`;
|
|
1512
|
+
await this.authManager.initialize();
|
|
1513
|
+
const accessToken = await this.authManager.getAccessToken();
|
|
1514
|
+
headers = { headers: { Authorization: `Bearer ${accessToken}` } };
|
|
1515
|
+
this.log("Using Vertex AI authentication with OAuth token");
|
|
1516
|
+
} else {
|
|
1517
|
+
wsUrl = `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent`;
|
|
1518
|
+
headers = {
|
|
1519
|
+
headers: {
|
|
1520
|
+
"x-goog-api-key": this.options.apiKey || "",
|
|
1521
|
+
"Content-Type": "application/json"
|
|
1547
1522
|
}
|
|
1548
|
-
}
|
|
1549
|
-
this.log("
|
|
1523
|
+
};
|
|
1524
|
+
this.log("Using Live API authentication with API key");
|
|
1525
|
+
}
|
|
1526
|
+
this.log("Connecting to:", wsUrl);
|
|
1527
|
+
this.ws = new WebSocket(wsUrl, void 0, headers);
|
|
1528
|
+
this.connectionManager.setWebSocket(this.ws);
|
|
1529
|
+
this.setupEventListeners();
|
|
1530
|
+
await this.connectionManager.waitForOpen();
|
|
1531
|
+
if (this.isResuming && this.sessionHandle) {
|
|
1532
|
+
await this.sendSessionResumption();
|
|
1533
|
+
} else {
|
|
1534
|
+
this.sendInitialConfig();
|
|
1535
|
+
this.sessionStartTime = Date.now();
|
|
1536
|
+
this.sessionId = randomUUID();
|
|
1537
|
+
}
|
|
1538
|
+
await this.waitForSessionCreated();
|
|
1539
|
+
this.state = "connected";
|
|
1540
|
+
this.emit("session", {
|
|
1541
|
+
state: "connected",
|
|
1542
|
+
config: {
|
|
1550
1543
|
sessionId: this.sessionId,
|
|
1551
1544
|
isResuming: this.isResuming,
|
|
1552
1545
|
toolCount: Object.keys(this.tools || {}).length
|
|
1553
|
-
});
|
|
1554
|
-
if (this.options.sessionConfig?.maxDuration) {
|
|
1555
|
-
this.startSessionDurationMonitor();
|
|
1556
1546
|
}
|
|
1557
|
-
}
|
|
1558
|
-
|
|
1559
|
-
this.
|
|
1560
|
-
|
|
1547
|
+
});
|
|
1548
|
+
this.log("Successfully connected to Gemini Live API", {
|
|
1549
|
+
sessionId: this.sessionId,
|
|
1550
|
+
isResuming: this.isResuming,
|
|
1551
|
+
toolCount: Object.keys(this.tools || {}).length
|
|
1552
|
+
});
|
|
1553
|
+
if (this.options.sessionConfig?.maxDuration) {
|
|
1554
|
+
this.startSessionDurationMonitor();
|
|
1561
1555
|
}
|
|
1562
|
-
}
|
|
1556
|
+
} catch (error) {
|
|
1557
|
+
this.state = "disconnected";
|
|
1558
|
+
this.log("Connection failed", error);
|
|
1559
|
+
throw error;
|
|
1560
|
+
}
|
|
1563
1561
|
}
|
|
1564
1562
|
/**
|
|
1565
1563
|
* Disconnect from the Gemini Live API
|
|
@@ -1597,172 +1595,164 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
1597
1595
|
* Send text to be converted to speech
|
|
1598
1596
|
*/
|
|
1599
1597
|
async speak(input, options) {
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)));
|
|
1606
|
-
}
|
|
1607
|
-
input = Buffer.concat(chunks).toString("utf-8");
|
|
1608
|
-
}
|
|
1609
|
-
if (input.trim().length === 0) {
|
|
1610
|
-
throw this.createAndEmitError("invalid_audio_format" /* INVALID_AUDIO_FORMAT */, "Input text is empty");
|
|
1598
|
+
this.validateConnectionState();
|
|
1599
|
+
if (typeof input !== "string") {
|
|
1600
|
+
const chunks = [];
|
|
1601
|
+
for await (const chunk of input) {
|
|
1602
|
+
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)));
|
|
1611
1603
|
}
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
}
|
|
1627
|
-
};
|
|
1628
|
-
if (options && (options.speaker || options.languageCode || options.responseModalities)) {
|
|
1629
|
-
const updateMessage = {
|
|
1630
|
-
type: "session.update",
|
|
1631
|
-
session: {
|
|
1632
|
-
generation_config: {
|
|
1633
|
-
...options.responseModalities ? { response_modalities: options.responseModalities } : {},
|
|
1634
|
-
speech_config: {
|
|
1635
|
-
...options.languageCode ? { language_code: options.languageCode } : {},
|
|
1636
|
-
...options.speaker ? { voice_config: { prebuilt_voice_config: { voice_name: options.speaker } } } : {}
|
|
1604
|
+
input = Buffer.concat(chunks).toString("utf-8");
|
|
1605
|
+
}
|
|
1606
|
+
if (input.trim().length === 0) {
|
|
1607
|
+
throw this.createAndEmitError("invalid_audio_format" /* INVALID_AUDIO_FORMAT */, "Input text is empty");
|
|
1608
|
+
}
|
|
1609
|
+
this.addToContext("user", input);
|
|
1610
|
+
const textMessage = {
|
|
1611
|
+
client_content: {
|
|
1612
|
+
turns: [
|
|
1613
|
+
{
|
|
1614
|
+
role: "user",
|
|
1615
|
+
parts: [
|
|
1616
|
+
{
|
|
1617
|
+
text: input
|
|
1637
1618
|
}
|
|
1619
|
+
]
|
|
1620
|
+
}
|
|
1621
|
+
],
|
|
1622
|
+
turnComplete: true
|
|
1623
|
+
}
|
|
1624
|
+
};
|
|
1625
|
+
if (options && (options.speaker || options.languageCode || options.responseModalities)) {
|
|
1626
|
+
const updateMessage = {
|
|
1627
|
+
type: "session.update",
|
|
1628
|
+
session: {
|
|
1629
|
+
generation_config: {
|
|
1630
|
+
...options.responseModalities ? { response_modalities: options.responseModalities } : {},
|
|
1631
|
+
speech_config: {
|
|
1632
|
+
...options.languageCode ? { language_code: options.languageCode } : {},
|
|
1633
|
+
...options.speaker ? { voice_config: { prebuilt_voice_config: { voice_name: options.speaker } } } : {}
|
|
1638
1634
|
}
|
|
1639
1635
|
}
|
|
1640
|
-
};
|
|
1641
|
-
try {
|
|
1642
|
-
this.sendEvent("session.update", updateMessage);
|
|
1643
|
-
this.log("Applied per-turn runtime options", options);
|
|
1644
|
-
} catch (error) {
|
|
1645
|
-
this.log("Failed to apply per-turn runtime options", error);
|
|
1646
1636
|
}
|
|
1647
|
-
}
|
|
1637
|
+
};
|
|
1648
1638
|
try {
|
|
1649
|
-
this.sendEvent("
|
|
1650
|
-
this.log("
|
|
1639
|
+
this.sendEvent("session.update", updateMessage);
|
|
1640
|
+
this.log("Applied per-turn runtime options", options);
|
|
1651
1641
|
} catch (error) {
|
|
1652
|
-
this.log("Failed to
|
|
1653
|
-
throw this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to send text message", error);
|
|
1642
|
+
this.log("Failed to apply per-turn runtime options", error);
|
|
1654
1643
|
}
|
|
1655
|
-
}
|
|
1644
|
+
}
|
|
1645
|
+
try {
|
|
1646
|
+
this.sendEvent("client_content", textMessage);
|
|
1647
|
+
this.log("Text message sent", { text: input });
|
|
1648
|
+
} catch (error) {
|
|
1649
|
+
this.log("Failed to send text message", error);
|
|
1650
|
+
throw this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to send text message", error);
|
|
1651
|
+
}
|
|
1656
1652
|
}
|
|
1657
1653
|
/**
|
|
1658
1654
|
* Send audio stream for processing
|
|
1659
1655
|
*/
|
|
1660
1656
|
async send(audioData) {
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
}
|
|
1688
|
-
}, "gemini-live.send")();
|
|
1657
|
+
this.validateConnectionState();
|
|
1658
|
+
if ("readable" in audioData && typeof audioData.on === "function") {
|
|
1659
|
+
const stream = audioData;
|
|
1660
|
+
stream.on("data", (chunk) => {
|
|
1661
|
+
try {
|
|
1662
|
+
const base64Audio = this.audioStreamManager.processAudioChunk(chunk);
|
|
1663
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "realtime");
|
|
1664
|
+
this.sendEvent("realtime_input", message);
|
|
1665
|
+
} catch (error) {
|
|
1666
|
+
this.log("Failed to process audio chunk", error);
|
|
1667
|
+
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Failed to process audio chunk", error);
|
|
1668
|
+
}
|
|
1669
|
+
});
|
|
1670
|
+
stream.on("error", (error) => {
|
|
1671
|
+
this.log("Audio stream error", error);
|
|
1672
|
+
this.createAndEmitError("audio_stream_error" /* AUDIO_STREAM_ERROR */, "Audio stream error", error);
|
|
1673
|
+
});
|
|
1674
|
+
stream.on("end", () => {
|
|
1675
|
+
this.log("Audio stream ended");
|
|
1676
|
+
});
|
|
1677
|
+
} else {
|
|
1678
|
+
const validateAudio = this.audioStreamManager.validateAndConvertAudioInput(audioData);
|
|
1679
|
+
const base64Audio = this.audioStreamManager.int16ArrayToBase64(validateAudio);
|
|
1680
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "realtime");
|
|
1681
|
+
this.sendEvent("realtime_input", message);
|
|
1682
|
+
}
|
|
1689
1683
|
}
|
|
1690
1684
|
/**
|
|
1691
1685
|
* Process speech from audio stream (traditional STT interface)
|
|
1692
1686
|
*/
|
|
1693
1687
|
async listen(audioStream, _options) {
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
this.log("Received transcription text:", { text: data.text, total: transcriptionText });
|
|
1701
|
-
}
|
|
1702
|
-
};
|
|
1703
|
-
const onError = (error) => {
|
|
1704
|
-
throw new Error(`Transcription failed: ${error.message}`);
|
|
1705
|
-
};
|
|
1706
|
-
const onSession = (data) => {
|
|
1707
|
-
if (data.state === "disconnected") {
|
|
1708
|
-
throw new Error("Session disconnected during transcription");
|
|
1709
|
-
}
|
|
1710
|
-
};
|
|
1711
|
-
this.on("writing", onWriting);
|
|
1712
|
-
this.on("error", onError);
|
|
1713
|
-
this.on("session", onSession);
|
|
1714
|
-
try {
|
|
1715
|
-
const result = await this.audioStreamManager.handleAudioTranscription(
|
|
1716
|
-
audioStream,
|
|
1717
|
-
(base64Audio) => {
|
|
1718
|
-
return new Promise((resolve, reject) => {
|
|
1719
|
-
try {
|
|
1720
|
-
const message = this.audioStreamManager.createAudioMessage(base64Audio, "input");
|
|
1721
|
-
const cleanup = () => {
|
|
1722
|
-
this.off("turnComplete", onTurnComplete);
|
|
1723
|
-
this.off("error", onErr);
|
|
1724
|
-
};
|
|
1725
|
-
const onTurnComplete = () => {
|
|
1726
|
-
cleanup();
|
|
1727
|
-
resolve(transcriptionText.trim());
|
|
1728
|
-
};
|
|
1729
|
-
const onErr = (e) => {
|
|
1730
|
-
cleanup();
|
|
1731
|
-
reject(new Error(e.message));
|
|
1732
|
-
};
|
|
1733
|
-
this.on("turnComplete", onTurnComplete);
|
|
1734
|
-
this.on("error", onErr);
|
|
1735
|
-
this.sendEvent("client_content", message);
|
|
1736
|
-
this.log("Sent audio for transcription");
|
|
1737
|
-
} catch (err) {
|
|
1738
|
-
reject(err);
|
|
1739
|
-
}
|
|
1740
|
-
});
|
|
1741
|
-
},
|
|
1742
|
-
(error) => {
|
|
1743
|
-
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Audio transcription failed", error);
|
|
1744
|
-
}
|
|
1745
|
-
);
|
|
1746
|
-
return result;
|
|
1747
|
-
} finally {
|
|
1748
|
-
this.off("writing", onWriting);
|
|
1749
|
-
this.off("error", onError);
|
|
1750
|
-
this.off("session", onSession);
|
|
1688
|
+
this.validateConnectionState();
|
|
1689
|
+
let transcriptionText = "";
|
|
1690
|
+
const onWriting = (data) => {
|
|
1691
|
+
if (data.role === "user") {
|
|
1692
|
+
transcriptionText += data.text;
|
|
1693
|
+
this.log("Received transcription text:", { text: data.text, total: transcriptionText });
|
|
1751
1694
|
}
|
|
1752
|
-
}
|
|
1695
|
+
};
|
|
1696
|
+
const onError = (error) => {
|
|
1697
|
+
throw new Error(`Transcription failed: ${error.message}`);
|
|
1698
|
+
};
|
|
1699
|
+
const onSession = (data) => {
|
|
1700
|
+
if (data.state === "disconnected") {
|
|
1701
|
+
throw new Error("Session disconnected during transcription");
|
|
1702
|
+
}
|
|
1703
|
+
};
|
|
1704
|
+
this.on("writing", onWriting);
|
|
1705
|
+
this.on("error", onError);
|
|
1706
|
+
this.on("session", onSession);
|
|
1707
|
+
try {
|
|
1708
|
+
const result = await this.audioStreamManager.handleAudioTranscription(
|
|
1709
|
+
audioStream,
|
|
1710
|
+
(base64Audio) => {
|
|
1711
|
+
return new Promise((resolve, reject) => {
|
|
1712
|
+
try {
|
|
1713
|
+
const message = this.audioStreamManager.createAudioMessage(base64Audio, "input");
|
|
1714
|
+
const cleanup = () => {
|
|
1715
|
+
this.off("turnComplete", onTurnComplete);
|
|
1716
|
+
this.off("error", onErr);
|
|
1717
|
+
};
|
|
1718
|
+
const onTurnComplete = () => {
|
|
1719
|
+
cleanup();
|
|
1720
|
+
resolve(transcriptionText.trim());
|
|
1721
|
+
};
|
|
1722
|
+
const onErr = (e) => {
|
|
1723
|
+
cleanup();
|
|
1724
|
+
reject(new Error(e.message));
|
|
1725
|
+
};
|
|
1726
|
+
this.on("turnComplete", onTurnComplete);
|
|
1727
|
+
this.on("error", onErr);
|
|
1728
|
+
this.sendEvent("client_content", message);
|
|
1729
|
+
this.log("Sent audio for transcription");
|
|
1730
|
+
} catch (err) {
|
|
1731
|
+
reject(err);
|
|
1732
|
+
}
|
|
1733
|
+
});
|
|
1734
|
+
},
|
|
1735
|
+
(error) => {
|
|
1736
|
+
this.createAndEmitError("audio_processing_error" /* AUDIO_PROCESSING_ERROR */, "Audio transcription failed", error);
|
|
1737
|
+
}
|
|
1738
|
+
);
|
|
1739
|
+
return result;
|
|
1740
|
+
} finally {
|
|
1741
|
+
this.off("writing", onWriting);
|
|
1742
|
+
this.off("error", onError);
|
|
1743
|
+
this.off("session", onSession);
|
|
1744
|
+
}
|
|
1753
1745
|
}
|
|
1754
1746
|
/**
|
|
1755
1747
|
* Get available speakers/voices
|
|
1756
1748
|
*/
|
|
1757
1749
|
async getSpeakers() {
|
|
1758
|
-
return
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
];
|
|
1765
|
-
}, "gemini-live.getSpeakers")();
|
|
1750
|
+
return [
|
|
1751
|
+
{ voiceId: "Puck", description: "Conversational, friendly" },
|
|
1752
|
+
{ voiceId: "Charon", description: "Deep, authoritative" },
|
|
1753
|
+
{ voiceId: "Kore", description: "Neutral, professional" },
|
|
1754
|
+
{ voiceId: "Fenrir", description: "Warm, approachable" }
|
|
1755
|
+
];
|
|
1766
1756
|
}
|
|
1767
1757
|
/**
|
|
1768
1758
|
* Resume a previous session using a session handle
|
|
@@ -2666,7 +2656,7 @@ var GeminiLiveVoice = class _GeminiLiveVoice extends MastraVoice {
|
|
|
2666
2656
|
}
|
|
2667
2657
|
log(message, ...args) {
|
|
2668
2658
|
if (this.debug) {
|
|
2669
|
-
console.
|
|
2659
|
+
console.info(`[GeminiLiveVoice] ${message}`, ...args);
|
|
2670
2660
|
}
|
|
2671
2661
|
}
|
|
2672
2662
|
/**
|