@mastra/voice-aws-nova-sonic 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/_types/@internal_voice/dist/_types/@internal_ai-sdk-v5/dist/index.d.ts +110 -221
- package/dist/_types/@internal_voice/dist/_types/@internal_core/dist/base/index.d.ts +27 -27
- package/dist/_types/@internal_voice/dist/_types/@internal_core/dist/index-S1lgaKO7.d.ts +218 -0
- package/dist/_types/@internal_voice/dist/_types/@internal_core/dist/request-context/index.d.ts +122 -95
- package/dist/_types/@internal_voice/dist/_types/@internal_core/dist/types/index.d.ts +4 -2
- package/dist/docs/SKILL.md +6 -6
- package/dist/docs/assets/SOURCE_MAP.json +1 -1
- package/dist/docs/references/{docs-voice-overview.md → guides-voice-overview.md} +90 -134
- package/dist/docs/references/{docs-voice-speech-to-speech.md → guides-voice-speech-to-speech.md} +90 -8
- package/dist/docs/references/{reference-voice-aws-nova-sonic.md → integrations-voice-aws-nova-sonic.md} +4 -2
- package/dist/index.cjs +1679 -1860
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1678 -1858
- package/dist/index.js.map +1 -1
- package/package.json +17 -16
- package/dist/_types/@internal_voice/dist/_types/@internal_core/dist/logger/index.d.ts +0 -217
package/dist/index.js
CHANGED
|
@@ -1,1873 +1,1693 @@
|
|
|
1
|
-
import { randomUUID } from
|
|
2
|
-
import { PassThrough } from
|
|
3
|
-
import { BedrockRuntimeClient, InvokeModelWithBidirectionalStreamCommand } from
|
|
4
|
-
import { NodeHttp2Handler } from
|
|
5
|
-
import { defaultProvider } from
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
1
|
+
import { randomUUID } from "crypto";
|
|
2
|
+
import { PassThrough } from "stream";
|
|
3
|
+
import { BedrockRuntimeClient, InvokeModelWithBidirectionalStreamCommand } from "@aws-sdk/client-bedrock-runtime";
|
|
4
|
+
import { NodeHttp2Handler } from "@smithy/node-http-handler";
|
|
5
|
+
import { defaultProvider } from "@aws-sdk/credential-provider-node";
|
|
6
|
+
//#region ../../packages/_internal-core/dist/logger/index.js
|
|
7
|
+
const RegisteredLogger = {
|
|
8
|
+
AGENT: "AGENT",
|
|
9
|
+
OBSERVABILITY: "OBSERVABILITY",
|
|
10
|
+
AUTH: "AUTH",
|
|
11
|
+
BROWSER: "BROWSER",
|
|
12
|
+
NETWORK: "NETWORK",
|
|
13
|
+
WORKFLOW: "WORKFLOW",
|
|
14
|
+
LLM: "LLM",
|
|
15
|
+
TTS: "TTS",
|
|
16
|
+
VOICE: "VOICE",
|
|
17
|
+
VECTOR: "VECTOR",
|
|
18
|
+
BUNDLER: "BUNDLER",
|
|
19
|
+
DEPLOYER: "DEPLOYER",
|
|
20
|
+
MEMORY: "MEMORY",
|
|
21
|
+
STORAGE: "STORAGE",
|
|
22
|
+
EMBEDDINGS: "EMBEDDINGS",
|
|
23
|
+
MCP_SERVER: "MCP_SERVER",
|
|
24
|
+
SERVER_CACHE: "SERVER_CACHE",
|
|
25
|
+
SERVER: "SERVER",
|
|
26
|
+
WORKSPACE: "WORKSPACE",
|
|
27
|
+
CHANNEL: "CHANNEL"
|
|
28
|
+
};
|
|
29
|
+
const LogLevel = {
|
|
30
|
+
DEBUG: "debug",
|
|
31
|
+
INFO: "info",
|
|
32
|
+
WARN: "warn",
|
|
33
|
+
ERROR: "error",
|
|
34
|
+
NONE: "silent"
|
|
35
|
+
};
|
|
17
36
|
var MastraLogger = class {
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
37
|
+
name;
|
|
38
|
+
level;
|
|
39
|
+
transports;
|
|
40
|
+
constructor(options = {}) {
|
|
41
|
+
this.name = options.name || "Mastra";
|
|
42
|
+
this.level = options.level || LogLevel.ERROR;
|
|
43
|
+
this.transports = new Map(Object.entries(options.transports || {}));
|
|
44
|
+
}
|
|
45
|
+
getTransports() {
|
|
46
|
+
return this.transports;
|
|
47
|
+
}
|
|
48
|
+
trackException(_error, _metadata) {}
|
|
49
|
+
async listLogs(transportId, params) {
|
|
50
|
+
if (!transportId || !this.transports.has(transportId)) return {
|
|
51
|
+
logs: [],
|
|
52
|
+
total: 0,
|
|
53
|
+
page: params?.page ?? 1,
|
|
54
|
+
perPage: params?.perPage ?? 100,
|
|
55
|
+
hasMore: false
|
|
56
|
+
};
|
|
57
|
+
return this.transports.get(transportId).listLogs?.(params) ?? {
|
|
58
|
+
logs: [],
|
|
59
|
+
total: 0,
|
|
60
|
+
page: params?.page ?? 1,
|
|
61
|
+
perPage: params?.perPage ?? 100,
|
|
62
|
+
hasMore: false
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
async listLogsByRunId({ transportId, runId, fromDate, toDate, logLevel, filters, page, perPage }) {
|
|
66
|
+
if (!transportId || !this.transports.has(transportId) || !runId) return {
|
|
67
|
+
logs: [],
|
|
68
|
+
total: 0,
|
|
69
|
+
page: page ?? 1,
|
|
70
|
+
perPage: perPage ?? 100,
|
|
71
|
+
hasMore: false
|
|
72
|
+
};
|
|
73
|
+
return this.transports.get(transportId).listLogsByRunId?.({
|
|
74
|
+
runId,
|
|
75
|
+
fromDate,
|
|
76
|
+
toDate,
|
|
77
|
+
logLevel,
|
|
78
|
+
filters,
|
|
79
|
+
page,
|
|
80
|
+
perPage
|
|
81
|
+
}) ?? {
|
|
82
|
+
logs: [],
|
|
83
|
+
total: 0,
|
|
84
|
+
page: page ?? 1,
|
|
85
|
+
perPage: perPage ?? 100,
|
|
86
|
+
hasMore: false
|
|
87
|
+
};
|
|
88
|
+
}
|
|
64
89
|
};
|
|
65
|
-
var ConsoleLogger = class
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
90
|
+
var ConsoleLogger = class ConsoleLogger extends MastraLogger {
|
|
91
|
+
component;
|
|
92
|
+
filter;
|
|
93
|
+
constructor(options = {}) {
|
|
94
|
+
super(options);
|
|
95
|
+
this.component = options.component;
|
|
96
|
+
this.filter = options.filter;
|
|
97
|
+
}
|
|
98
|
+
child(componentOrBindings) {
|
|
99
|
+
const component = typeof componentOrBindings === "string" ? componentOrBindings : componentOrBindings?.component ?? this.component;
|
|
100
|
+
return new ConsoleLogger({
|
|
101
|
+
name: this.name,
|
|
102
|
+
level: this.level,
|
|
103
|
+
component,
|
|
104
|
+
filter: this.filter
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
shouldLog(level, message, args) {
|
|
108
|
+
if (!this.filter) return true;
|
|
109
|
+
try {
|
|
110
|
+
return this.filter({
|
|
111
|
+
component: this.component,
|
|
112
|
+
level,
|
|
113
|
+
message,
|
|
114
|
+
args
|
|
115
|
+
});
|
|
116
|
+
} catch (e) {
|
|
117
|
+
console.error(`[Logger] Filter error for component=${this.component} level=${level}:`, e);
|
|
118
|
+
return true;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
prefix() {
|
|
122
|
+
return this.component ? `[${this.component}] ` : "";
|
|
123
|
+
}
|
|
124
|
+
debug(message, ...args) {
|
|
125
|
+
if (this.level === LogLevel.DEBUG && this.shouldLog(LogLevel.DEBUG, message, args)) console.info(`${this.prefix()}${message}`, ...args);
|
|
126
|
+
}
|
|
127
|
+
info(message, ...args) {
|
|
128
|
+
if ((this.level === LogLevel.INFO || this.level === LogLevel.DEBUG) && this.shouldLog(LogLevel.INFO, message, args)) console.info(`${this.prefix()}${message}`, ...args);
|
|
129
|
+
}
|
|
130
|
+
warn(message, ...args) {
|
|
131
|
+
if ((this.level === LogLevel.WARN || this.level === LogLevel.INFO || this.level === LogLevel.DEBUG) && this.shouldLog(LogLevel.WARN, message, args)) console.warn(`${this.prefix()}${message}`, ...args);
|
|
132
|
+
}
|
|
133
|
+
error(message, ...args) {
|
|
134
|
+
if ((this.level === LogLevel.ERROR || this.level === LogLevel.WARN || this.level === LogLevel.INFO || this.level === LogLevel.DEBUG) && this.shouldLog(LogLevel.ERROR, message, args)) console.error(`${this.prefix()}${message}`, ...args);
|
|
135
|
+
}
|
|
136
|
+
async listLogs(_transportId, _params) {
|
|
137
|
+
return {
|
|
138
|
+
logs: [],
|
|
139
|
+
total: 0,
|
|
140
|
+
page: _params?.page ?? 1,
|
|
141
|
+
perPage: _params?.perPage ?? 100,
|
|
142
|
+
hasMore: false
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
async listLogsByRunId(_args) {
|
|
146
|
+
return {
|
|
147
|
+
logs: [],
|
|
148
|
+
total: 0,
|
|
149
|
+
page: _args.page ?? 1,
|
|
150
|
+
perPage: _args.perPage ?? 100,
|
|
151
|
+
hasMore: false
|
|
152
|
+
};
|
|
153
|
+
}
|
|
120
154
|
};
|
|
121
|
-
|
|
122
|
-
|
|
155
|
+
//#endregion
|
|
156
|
+
//#region ../../packages/_internal-core/dist/base/index.js
|
|
123
157
|
var MastraBase = class {
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
*/
|
|
156
|
-
__setLogger(logger) {
|
|
157
|
-
this.logger = "child" in logger && typeof logger.child === "function" ? logger.child({ component: this.component }) : logger;
|
|
158
|
-
}
|
|
158
|
+
component = RegisteredLogger.LLM;
|
|
159
|
+
logger;
|
|
160
|
+
name;
|
|
161
|
+
#rawConfig;
|
|
162
|
+
constructor({ component, name, rawConfig }) {
|
|
163
|
+
this.component = component || RegisteredLogger.LLM;
|
|
164
|
+
this.name = name;
|
|
165
|
+
this.#rawConfig = rawConfig;
|
|
166
|
+
this.logger = new ConsoleLogger({ name: `${this.component} - ${this.name}` });
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Returns the raw storage configuration this primitive was created from,
|
|
170
|
+
* or undefined if it was created from code.
|
|
171
|
+
*/
|
|
172
|
+
toRawConfig() {
|
|
173
|
+
return this.#rawConfig;
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Sets the raw storage configuration for this primitive.
|
|
177
|
+
* @internal
|
|
178
|
+
*/
|
|
179
|
+
__setRawConfig(rawConfig) {
|
|
180
|
+
this.#rawConfig = rawConfig;
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Set the logger for the agent
|
|
184
|
+
* @param logger
|
|
185
|
+
*/
|
|
186
|
+
__setLogger(logger) {
|
|
187
|
+
this.logger = "child" in logger && typeof logger.child === "function" ? logger.child({ component: this.component }) : logger;
|
|
188
|
+
}
|
|
159
189
|
};
|
|
160
|
-
|
|
161
|
-
|
|
190
|
+
//#endregion
|
|
191
|
+
//#region ../../packages/_internals/voice/dist/aisdk-C_K-VCwq.js
|
|
162
192
|
var MastraVoice = class extends MastraBase {
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
return Promise.resolve({ enabled: false });
|
|
265
|
-
}
|
|
193
|
+
listeningModel;
|
|
194
|
+
speechModel;
|
|
195
|
+
speaker;
|
|
196
|
+
realtimeConfig;
|
|
197
|
+
constructor({ listeningModel, speechModel, speaker, realtimeConfig, name } = {}) {
|
|
198
|
+
super({
|
|
199
|
+
component: "VOICE",
|
|
200
|
+
name
|
|
201
|
+
});
|
|
202
|
+
this.listeningModel = listeningModel;
|
|
203
|
+
this.speechModel = speechModel;
|
|
204
|
+
this.speaker = speaker;
|
|
205
|
+
this.realtimeConfig = realtimeConfig;
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Custom serialization for tracing/observability spans.
|
|
209
|
+
* Excludes `apiKey` from listeningModel / speechModel / realtimeConfig
|
|
210
|
+
* and any provider-specific state held by subclasses. Subclasses that
|
|
211
|
+
* need to expose additional non-sensitive fields can override.
|
|
212
|
+
*/
|
|
213
|
+
serializeForSpan() {
|
|
214
|
+
return {
|
|
215
|
+
component: "VOICE",
|
|
216
|
+
name: this.name,
|
|
217
|
+
speaker: this.speaker,
|
|
218
|
+
listeningModel: this.listeningModel ? { name: this.listeningModel.name } : void 0,
|
|
219
|
+
speechModel: this.speechModel ? { name: this.speechModel.name } : void 0,
|
|
220
|
+
realtimeModel: this.realtimeConfig?.model
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
updateConfig(_options) {
|
|
224
|
+
this.logger.debug("updateConfig not implemented by this voice provider");
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* Initializes a WebSocket or WebRTC connection for real-time communication
|
|
228
|
+
* @returns Promise that resolves when the connection is established
|
|
229
|
+
*/
|
|
230
|
+
async connect(_options) {
|
|
231
|
+
this.logger.debug("connect not implemented by this voice provider");
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* Relay audio data to the voice provider for real-time processing
|
|
235
|
+
* @param audioData Audio data to relay
|
|
236
|
+
*/
|
|
237
|
+
async send(_audioData) {
|
|
238
|
+
this.logger.debug("relay not implemented by this voice provider");
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* Trigger voice providers to respond
|
|
242
|
+
*/
|
|
243
|
+
async answer(_options) {
|
|
244
|
+
this.logger.debug("answer not implemented by this voice provider");
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* Equip the voice provider with instructions
|
|
248
|
+
* @param instructions Instructions to add
|
|
249
|
+
*/
|
|
250
|
+
addInstructions(_instructions) {}
|
|
251
|
+
/**
|
|
252
|
+
* Equip the voice provider with tools
|
|
253
|
+
* @param tools Array of tools to add
|
|
254
|
+
*/
|
|
255
|
+
addTools(_tools) {}
|
|
256
|
+
/**
|
|
257
|
+
* Disconnect from the WebSocket or WebRTC connection
|
|
258
|
+
*/
|
|
259
|
+
close() {
|
|
260
|
+
this.logger.debug("close not implemented by this voice provider");
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* Register an event listener
|
|
264
|
+
* @param event Event name (e.g., 'speaking', 'writing', 'error')
|
|
265
|
+
* @param callback Callback function that receives event data
|
|
266
|
+
*/
|
|
267
|
+
on(_event, _callback) {
|
|
268
|
+
this.logger.debug("on not implemented by this voice provider");
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* Remove an event listener
|
|
272
|
+
* @param event Event name (e.g., 'speaking', 'writing', 'error')
|
|
273
|
+
* @param callback Callback function to remove
|
|
274
|
+
*/
|
|
275
|
+
off(_event, _callback) {
|
|
276
|
+
this.logger.debug("off not implemented by this voice provider");
|
|
277
|
+
}
|
|
278
|
+
/**
|
|
279
|
+
* Get available speakers/voices
|
|
280
|
+
* @returns Array of available voice IDs and their metadata
|
|
281
|
+
*/
|
|
282
|
+
getSpeakers() {
|
|
283
|
+
this.logger.debug("getSpeakers not implemented by this voice provider");
|
|
284
|
+
return Promise.resolve([]);
|
|
285
|
+
}
|
|
286
|
+
/**
|
|
287
|
+
* Get available speakers/voices
|
|
288
|
+
* @returns Array of available voice IDs and their metadata
|
|
289
|
+
*/
|
|
290
|
+
getListener() {
|
|
291
|
+
this.logger.debug("getListener not implemented by this voice provider");
|
|
292
|
+
return Promise.resolve({ enabled: false });
|
|
293
|
+
}
|
|
266
294
|
};
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
295
|
+
//#endregion
|
|
296
|
+
//#region src/types.ts
|
|
297
|
+
/**
|
|
298
|
+
* Error codes for Nova Sonic specific errors
|
|
299
|
+
*/
|
|
300
|
+
let NovaSonicErrorCode = /* @__PURE__ */ function(NovaSonicErrorCode) {
|
|
301
|
+
NovaSonicErrorCode["CONNECTION_FAILED"] = "connection_failed";
|
|
302
|
+
NovaSonicErrorCode["CONNECTION_NOT_ESTABLISHED"] = "connection_not_established";
|
|
303
|
+
NovaSonicErrorCode["AUTHENTICATION_FAILED"] = "authentication_failed";
|
|
304
|
+
NovaSonicErrorCode["CREDENTIALS_MISSING"] = "credentials_missing";
|
|
305
|
+
NovaSonicErrorCode["REGION_INVALID"] = "region_invalid";
|
|
306
|
+
NovaSonicErrorCode["WEBSOCKET_ERROR"] = "websocket_error";
|
|
307
|
+
NovaSonicErrorCode["AUDIO_PROCESSING_ERROR"] = "audio_processing_error";
|
|
308
|
+
NovaSonicErrorCode["AUDIO_STREAM_ERROR"] = "audio_stream_error";
|
|
309
|
+
NovaSonicErrorCode["SPEAKER_STREAM_ERROR"] = "speaker_stream_error";
|
|
310
|
+
NovaSonicErrorCode["TRANSCRIPTION_TIMEOUT"] = "transcription_timeout";
|
|
311
|
+
NovaSonicErrorCode["TRANSCRIPTION_FAILED"] = "transcription_failed";
|
|
312
|
+
NovaSonicErrorCode["TOOL_EXECUTION_ERROR"] = "tool_execution_error";
|
|
313
|
+
NovaSonicErrorCode["TOOL_NOT_FOUND"] = "tool_not_found";
|
|
314
|
+
NovaSonicErrorCode["SESSION_CONFIG_UPDATE_FAILED"] = "session_config_update_failed";
|
|
315
|
+
NovaSonicErrorCode["INVALID_AUDIO_FORMAT"] = "invalid_audio_format";
|
|
316
|
+
NovaSonicErrorCode["NOT_CONNECTED"] = "not_connected";
|
|
317
|
+
NovaSonicErrorCode["INVALID_STATE"] = "invalid_state";
|
|
318
|
+
NovaSonicErrorCode["VALIDATION_ERROR"] = "validation_error";
|
|
319
|
+
NovaSonicErrorCode["UNKNOWN_ERROR"] = "unknown_error";
|
|
320
|
+
return NovaSonicErrorCode;
|
|
321
|
+
}({});
|
|
322
|
+
//#endregion
|
|
323
|
+
//#region src/utils/errors.ts
|
|
324
|
+
/**
|
|
325
|
+
* Helper class for consistent error handling across Nova Sonic voice integration
|
|
326
|
+
*/
|
|
293
327
|
var NovaSonicError = class extends Error {
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
328
|
+
code;
|
|
329
|
+
details;
|
|
330
|
+
timestamp;
|
|
331
|
+
constructor(code, message, details) {
|
|
332
|
+
super(message);
|
|
333
|
+
this.name = "NovaSonicError";
|
|
334
|
+
this.code = code;
|
|
335
|
+
this.details = details;
|
|
336
|
+
this.timestamp = Date.now();
|
|
337
|
+
}
|
|
338
|
+
toEventData() {
|
|
339
|
+
return {
|
|
340
|
+
message: this.message,
|
|
341
|
+
code: this.code,
|
|
342
|
+
details: this.details,
|
|
343
|
+
timestamp: this.timestamp
|
|
344
|
+
};
|
|
345
|
+
}
|
|
312
346
|
};
|
|
313
|
-
|
|
314
|
-
|
|
347
|
+
//#endregion
|
|
348
|
+
//#region src/utils/auth.ts
|
|
349
|
+
/**
|
|
350
|
+
* Get AWS credentials from various sources
|
|
351
|
+
*/
|
|
315
352
|
async function getAwsCredentials(explicitCredentials, debug) {
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
}
|
|
330
|
-
return credentials;
|
|
331
|
-
} catch (error) {
|
|
332
|
-
if (error instanceof NovaSonicError) {
|
|
333
|
-
throw error;
|
|
334
|
-
}
|
|
335
|
-
throw new NovaSonicError(
|
|
336
|
-
"authentication_failed" /* AUTHENTICATION_FAILED */,
|
|
337
|
-
`Failed to load AWS credentials: ${error instanceof Error ? error.message : "Unknown error"}`,
|
|
338
|
-
error
|
|
339
|
-
);
|
|
340
|
-
}
|
|
353
|
+
if (explicitCredentials) {
|
|
354
|
+
if (debug) console.info("[getAwsCredentials] Using explicit credentials provided in config");
|
|
355
|
+
return explicitCredentials;
|
|
356
|
+
}
|
|
357
|
+
try {
|
|
358
|
+
if (debug) console.info("[getAwsCredentials] Using default credential provider chain");
|
|
359
|
+
const credentials = await defaultProvider()();
|
|
360
|
+
if (debug) console.info("[getAwsCredentials] Credentials retrieved successfully");
|
|
361
|
+
return credentials;
|
|
362
|
+
} catch (error) {
|
|
363
|
+
if (error instanceof NovaSonicError) throw error;
|
|
364
|
+
throw new NovaSonicError("authentication_failed", `Failed to load AWS credentials: ${error instanceof Error ? error.message : "Unknown error"}`, error);
|
|
365
|
+
}
|
|
341
366
|
}
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
367
|
+
//#endregion
|
|
368
|
+
//#region src/index.ts
|
|
369
|
+
/**
|
|
370
|
+
* Default configuration values
|
|
371
|
+
*/
|
|
372
|
+
const DEFAULT_MODEL = "amazon.nova-2-sonic-v1:0";
|
|
373
|
+
const DEFAULT_REGION = "us-east-1";
|
|
374
|
+
/**
|
|
375
|
+
* NovaSonicVoice provides real-time voice interaction capabilities using AWS Nova 2 Sonic's
|
|
376
|
+
* bidirectional streaming API. It supports:
|
|
377
|
+
* - Real-time text-to-speech
|
|
378
|
+
* - Speech-to-text (transcription)
|
|
379
|
+
* - Voice activity detection
|
|
380
|
+
* - Multiple voice options and languages
|
|
381
|
+
* - Polyglot voices
|
|
382
|
+
* - Tool/function calling
|
|
383
|
+
* - Cross-modal input (audio and text)
|
|
384
|
+
*
|
|
385
|
+
* The class manages HTTP/2 bidirectional streaming connections, audio streaming, and event handling
|
|
386
|
+
* for seamless voice interactions.
|
|
387
|
+
*
|
|
388
|
+
* @extends MastraVoice
|
|
389
|
+
*
|
|
390
|
+
* @example
|
|
391
|
+
* ```typescript
|
|
392
|
+
* const voice = new NovaSonicVoice({
|
|
393
|
+
* region: 'us-east-1',
|
|
394
|
+
* model: 'amazon.nova-2-sonic-v1:0',
|
|
395
|
+
* speaker: 'default',
|
|
396
|
+
* });
|
|
397
|
+
*
|
|
398
|
+
* await voice.connect();
|
|
399
|
+
* voice.on('speaking', ({ audio }) => {
|
|
400
|
+
* // Handle audio data
|
|
401
|
+
* });
|
|
402
|
+
*
|
|
403
|
+
* await voice.speak('Hello, how can I help you today?');
|
|
404
|
+
* ```
|
|
405
|
+
*/
|
|
346
406
|
var NovaSonicVoice = class extends MastraVoice {
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
const contentName = randomUUID();
|
|
1630
|
-
await this.sendClientEvent({
|
|
1631
|
-
contentStart: {
|
|
1632
|
-
promptName,
|
|
1633
|
-
contentName,
|
|
1634
|
-
type: "TEXT",
|
|
1635
|
-
interactive: true,
|
|
1636
|
-
role: "USER",
|
|
1637
|
-
textInputConfiguration: {
|
|
1638
|
-
mediaType: "text/plain"
|
|
1639
|
-
}
|
|
1640
|
-
}
|
|
1641
|
-
});
|
|
1642
|
-
await this.sendClientEvent({
|
|
1643
|
-
textInput: {
|
|
1644
|
-
promptName,
|
|
1645
|
-
contentName,
|
|
1646
|
-
content: text
|
|
1647
|
-
}
|
|
1648
|
-
});
|
|
1649
|
-
await this.sendClientEvent({
|
|
1650
|
-
contentEnd: {
|
|
1651
|
-
promptName,
|
|
1652
|
-
contentName
|
|
1653
|
-
}
|
|
1654
|
-
});
|
|
1655
|
-
}
|
|
1656
|
-
/**
|
|
1657
|
-
* Convert speech to text (transcription)
|
|
1658
|
-
* For Nova Sonic, this is the same as send() - both stream audio input
|
|
1659
|
-
*/
|
|
1660
|
-
async listen(audioStream, _options) {
|
|
1661
|
-
if (audioStream && typeof audioStream === "object" && "read" in audioStream) {
|
|
1662
|
-
await this.send(audioStream);
|
|
1663
|
-
} else {
|
|
1664
|
-
throw new NovaSonicError("invalid_audio_format" /* INVALID_AUDIO_FORMAT */, "Unsupported audio stream format for listen()");
|
|
1665
|
-
}
|
|
1666
|
-
}
|
|
1667
|
-
/**
|
|
1668
|
-
* Streams audio data in real-time to the AWS Bedrock service.
|
|
1669
|
-
* Following AWS Nova 2 Sonic event sequence:
|
|
1670
|
-
* 1. contentStart (AUDIO, USER) - if not already sent
|
|
1671
|
-
* 2. audioInput events (one per chunk)
|
|
1672
|
-
* 3. contentEnd - when audio stream ends (handled separately via endAudioInput)
|
|
1673
|
-
*/
|
|
1674
|
-
async send(audioData) {
|
|
1675
|
-
this.log(`[send] Current state: ${this.state}`);
|
|
1676
|
-
if (this.state !== "connected") {
|
|
1677
|
-
this.log(`[send] ERROR: State is '${this.state}', expected 'connected'`);
|
|
1678
|
-
throw new NovaSonicError(
|
|
1679
|
-
"not_connected" /* NOT_CONNECTED */,
|
|
1680
|
-
`Not connected. Current state: ${this.state}. Call connect() first.`
|
|
1681
|
-
);
|
|
1682
|
-
}
|
|
1683
|
-
this.log(`[send] State check passed, proceeding with send`);
|
|
1684
|
-
if (!(audioData instanceof Int16Array) && !(audioData && typeof audioData === "object" && "read" in audioData)) {
|
|
1685
|
-
throw new NovaSonicError("invalid_audio_format" /* INVALID_AUDIO_FORMAT */, "Unsupported audio data format");
|
|
1686
|
-
}
|
|
1687
|
-
if (this.turnCompleted || this.hasSentContentEnd) {
|
|
1688
|
-
this.log(
|
|
1689
|
-
`[send] Starting new turn - resetting flags. turnCompleted=${this.turnCompleted}, hasSentContentEnd=${this.hasSentContentEnd}.`
|
|
1690
|
-
);
|
|
1691
|
-
const needNewContent = this.hasSentContentEnd;
|
|
1692
|
-
this.turnCompleted = false;
|
|
1693
|
-
this.hasSentContentEnd = false;
|
|
1694
|
-
this.streamRestartAttempted = false;
|
|
1695
|
-
if (needNewContent) {
|
|
1696
|
-
this.audioContentStarted = false;
|
|
1697
|
-
this.log(`[send] contentEnd was previously sent - will create new audio content container`);
|
|
1698
|
-
}
|
|
1699
|
-
this.log(
|
|
1700
|
-
`[send] State reset: turnCompleted=false, hasSentContentEnd=false, audioContentStarted=${this.audioContentStarted}`
|
|
1701
|
-
);
|
|
1702
|
-
}
|
|
1703
|
-
if (!this.promptStarted) {
|
|
1704
|
-
this.promptStarted = true;
|
|
1705
|
-
}
|
|
1706
|
-
const promptName = this._promptName;
|
|
1707
|
-
if (!promptName) {
|
|
1708
|
-
throw new NovaSonicError(
|
|
1709
|
-
"not_connected" /* NOT_CONNECTED */,
|
|
1710
|
-
"Prompt name not initialized. Connection may not be fully established."
|
|
1711
|
-
);
|
|
1712
|
-
}
|
|
1713
|
-
if (!this.audioContentStarted) {
|
|
1714
|
-
const audioContentId = randomUUID();
|
|
1715
|
-
this.audioContentName = audioContentId;
|
|
1716
|
-
this.log(`[send] First audio send - sending AUDIO contentStart with contentName: ${audioContentId}`);
|
|
1717
|
-
await this.sendClientEvent({
|
|
1718
|
-
contentStart: {
|
|
1719
|
-
promptName,
|
|
1720
|
-
contentName: audioContentId,
|
|
1721
|
-
type: "AUDIO",
|
|
1722
|
-
interactive: true,
|
|
1723
|
-
role: "USER",
|
|
1724
|
-
audioInputConfiguration: {
|
|
1725
|
-
mediaType: "audio/lpcm",
|
|
1726
|
-
sampleRateHertz: 16e3,
|
|
1727
|
-
sampleSizeBits: 16,
|
|
1728
|
-
channelCount: 1,
|
|
1729
|
-
encoding: "base64",
|
|
1730
|
-
audioType: "SPEECH"
|
|
1731
|
-
}
|
|
1732
|
-
}
|
|
1733
|
-
});
|
|
1734
|
-
this.audioContentStarted = true;
|
|
1735
|
-
this.log(`[send] AUDIO contentStart sent, ready to stream audio`);
|
|
1736
|
-
} else {
|
|
1737
|
-
this.log(`[send] AUDIO contentStart already sent, sending audioInput chunks directly`);
|
|
1738
|
-
}
|
|
1739
|
-
if (!this.audioContentName) {
|
|
1740
|
-
throw new NovaSonicError("invalid_state" /* INVALID_STATE */, "Audio content name not initialized. This should not happen.");
|
|
1741
|
-
}
|
|
1742
|
-
const contentName = this.audioContentName;
|
|
1743
|
-
if (audioData instanceof Int16Array) {
|
|
1744
|
-
const buffer = Buffer.from(audioData.buffer, audioData.byteOffset, audioData.byteLength);
|
|
1745
|
-
const base64Audio = buffer.toString("base64");
|
|
1746
|
-
this.log(
|
|
1747
|
-
`[send] Sending audioInput chunk, size: ${buffer.length} bytes, contentName: ${contentName}, turnCompleted: ${this.turnCompleted}, hasSentContentEnd: ${this.hasSentContentEnd}, audioContentStarted: ${this.audioContentStarted}, state: ${this.state}`
|
|
1748
|
-
);
|
|
1749
|
-
if (this.state !== "connected") {
|
|
1750
|
-
this.log(`[send] ERROR: State changed to '${this.state}' during send!`);
|
|
1751
|
-
throw new NovaSonicError("not_connected" /* NOT_CONNECTED */, `Connection lost during send. State: ${this.state}`);
|
|
1752
|
-
}
|
|
1753
|
-
await this.sendClientEvent({
|
|
1754
|
-
audioInput: {
|
|
1755
|
-
promptName,
|
|
1756
|
-
contentName,
|
|
1757
|
-
content: base64Audio
|
|
1758
|
-
}
|
|
1759
|
-
});
|
|
1760
|
-
this.log(`[send] audioInput chunk sent successfully`);
|
|
1761
|
-
} else if (audioData && typeof audioData === "object" && "read" in audioData) {
|
|
1762
|
-
const stream = audioData;
|
|
1763
|
-
for await (const chunk of stream) {
|
|
1764
|
-
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
1765
|
-
const base64Audio = buffer.toString("base64");
|
|
1766
|
-
this.log(
|
|
1767
|
-
`[send] Sending audioInput chunk from stream, size: ${buffer.length} bytes, contentName: ${contentName}, turnCompleted: ${this.turnCompleted}, hasSentContentEnd: ${this.hasSentContentEnd}`
|
|
1768
|
-
);
|
|
1769
|
-
await this.sendClientEvent({
|
|
1770
|
-
audioInput: {
|
|
1771
|
-
promptName,
|
|
1772
|
-
contentName,
|
|
1773
|
-
content: base64Audio
|
|
1774
|
-
}
|
|
1775
|
-
});
|
|
1776
|
-
}
|
|
1777
|
-
} else {
|
|
1778
|
-
throw new NovaSonicError("invalid_audio_format" /* INVALID_AUDIO_FORMAT */, "Unsupported audio data format");
|
|
1779
|
-
}
|
|
1780
|
-
}
|
|
1781
|
-
/**
|
|
1782
|
-
* End audio input stream (sends contentEnd for audio)
|
|
1783
|
-
* Call this when done sending audio chunks
|
|
1784
|
-
*/
|
|
1785
|
-
async endAudioInput() {
|
|
1786
|
-
if (this.hasSentContentEnd) {
|
|
1787
|
-
this.log("[endAudioInput] contentEnd already sent for this turn, skipping");
|
|
1788
|
-
return;
|
|
1789
|
-
}
|
|
1790
|
-
if (this.turnCompleted) {
|
|
1791
|
-
this.log(
|
|
1792
|
-
"[endAudioInput] Turn already completed by AWS, skipping contentEnd. Resetting turnCompleted flag for next turn."
|
|
1793
|
-
);
|
|
1794
|
-
this.turnCompleted = false;
|
|
1795
|
-
this.hasSentContentEnd = false;
|
|
1796
|
-
return;
|
|
1797
|
-
}
|
|
1798
|
-
if (this.audioContentStarted && this.audioContentName && this._promptName) {
|
|
1799
|
-
const promptName = this._promptName;
|
|
1800
|
-
this.log("[endAudioInput] Sending contentEnd for audio input");
|
|
1801
|
-
await this.sendClientEvent({
|
|
1802
|
-
contentEnd: {
|
|
1803
|
-
promptName,
|
|
1804
|
-
contentName: this.audioContentName
|
|
1805
|
-
}
|
|
1806
|
-
});
|
|
1807
|
-
this.hasSentContentEnd = true;
|
|
1808
|
-
} else {
|
|
1809
|
-
this.log(
|
|
1810
|
-
"[endAudioInput] Cannot send contentEnd: audioContentStarted=" + this.audioContentStarted + ", audioContentName=" + this.audioContentName
|
|
1811
|
-
);
|
|
1812
|
-
}
|
|
1813
|
-
}
|
|
1814
|
-
/**
|
|
1815
|
-
* Register an event listener
|
|
1816
|
-
*/
|
|
1817
|
-
on(event, callback) {
|
|
1818
|
-
if (!this.events[event]) {
|
|
1819
|
-
this.events[event] = [];
|
|
1820
|
-
}
|
|
1821
|
-
this.events[event].push(callback);
|
|
1822
|
-
}
|
|
1823
|
-
/**
|
|
1824
|
-
* Remove an event listener
|
|
1825
|
-
*/
|
|
1826
|
-
off(event, callback) {
|
|
1827
|
-
if (!this.events[event]) {
|
|
1828
|
-
return;
|
|
1829
|
-
}
|
|
1830
|
-
const index = this.events[event].indexOf(callback);
|
|
1831
|
-
if (index !== -1) {
|
|
1832
|
-
this.events[event].splice(index, 1);
|
|
1833
|
-
}
|
|
1834
|
-
}
|
|
1835
|
-
/**
|
|
1836
|
-
* Emit an event with arguments
|
|
1837
|
-
*/
|
|
1838
|
-
emit(event, data) {
|
|
1839
|
-
if (!this.events[event]) {
|
|
1840
|
-
this.log(`[NovaSonic] emit('${event}'): No listeners registered for this event`);
|
|
1841
|
-
return;
|
|
1842
|
-
}
|
|
1843
|
-
const listenerCount = this.events[event].length;
|
|
1844
|
-
this.log(`[NovaSonic] emit('${event}'): Calling ${listenerCount} listener(s)`);
|
|
1845
|
-
for (const callback of this.events[event]) {
|
|
1846
|
-
try {
|
|
1847
|
-
callback(data);
|
|
1848
|
-
this.log(`[NovaSonic] emit('${event}'): Successfully called one listener`);
|
|
1849
|
-
} catch (error) {
|
|
1850
|
-
this.log(`Error in event handler for ${event}:`, error);
|
|
1851
|
-
}
|
|
1852
|
-
}
|
|
1853
|
-
this.log(`[NovaSonic] emit('${event}'): Finished calling all ${listenerCount} listener(s)`);
|
|
1854
|
-
}
|
|
1855
|
-
/**
|
|
1856
|
-
* Get listener status
|
|
1857
|
-
*/
|
|
1858
|
-
async getListener() {
|
|
1859
|
-
return { enabled: this.state === "connected" };
|
|
1860
|
-
}
|
|
1861
|
-
/**
|
|
1862
|
-
* Log helper
|
|
1863
|
-
*/
|
|
1864
|
-
log(...args) {
|
|
1865
|
-
if (this.debug) {
|
|
1866
|
-
console.info("[NovaSonicVoice]", ...args);
|
|
1867
|
-
}
|
|
1868
|
-
}
|
|
407
|
+
client;
|
|
408
|
+
stream;
|
|
409
|
+
inputStream;
|
|
410
|
+
_eventQueue;
|
|
411
|
+
_signalQueue;
|
|
412
|
+
_closeSignal;
|
|
413
|
+
_promptName;
|
|
414
|
+
state = "disconnected";
|
|
415
|
+
events;
|
|
416
|
+
instructions;
|
|
417
|
+
tools;
|
|
418
|
+
requestContext;
|
|
419
|
+
debug;
|
|
420
|
+
region;
|
|
421
|
+
model;
|
|
422
|
+
credentials;
|
|
423
|
+
speakerStreams;
|
|
424
|
+
currentResponseId;
|
|
425
|
+
processingStream = false;
|
|
426
|
+
streamRestartAttempted = false;
|
|
427
|
+
sessionConfig;
|
|
428
|
+
promptStarted = false;
|
|
429
|
+
audioContentName;
|
|
430
|
+
audioContentStarted = false;
|
|
431
|
+
hasSentContentEnd = false;
|
|
432
|
+
turnCompleted = false;
|
|
433
|
+
turnCompleteTimeout;
|
|
434
|
+
isReceivingAssistantAudio = false;
|
|
435
|
+
currentTextGenerationStage;
|
|
436
|
+
/**
|
|
437
|
+
* Creates a new instance of NovaSonicVoice.
|
|
438
|
+
*
|
|
439
|
+
* @param config - Configuration options for the voice instance
|
|
440
|
+
* @param config.region - AWS region (defaults to us-east-1)
|
|
441
|
+
* @param config.model - The model ID to use (defaults to amazon.nova-2-sonic-v1:0)
|
|
442
|
+
* @param config.credentials - AWS credentials (optional, uses default credential chain)
|
|
443
|
+
* @param config.speaker - Voice name/identifier
|
|
444
|
+
* @param config.languageCode - Language code for the voice
|
|
445
|
+
* @param config.debug - Enable debug mode
|
|
446
|
+
*
|
|
447
|
+
* @example
|
|
448
|
+
* ```typescript
|
|
449
|
+
* const voice = new NovaSonicVoice({
|
|
450
|
+
* region: 'us-east-1',
|
|
451
|
+
* model: 'amazon.nova-2-sonic-v1:0',
|
|
452
|
+
* speaker: 'default',
|
|
453
|
+
* });
|
|
454
|
+
* ```
|
|
455
|
+
*/
|
|
456
|
+
constructor(config = {}) {
|
|
457
|
+
let normalizedConfig;
|
|
458
|
+
if ("realtimeConfig" in config || "speechModel" in config || "listeningModel" in config) normalizedConfig = config;
|
|
459
|
+
else {
|
|
460
|
+
const configOptions = config;
|
|
461
|
+
normalizedConfig = {
|
|
462
|
+
realtimeConfig: {
|
|
463
|
+
model: configOptions.model || DEFAULT_MODEL,
|
|
464
|
+
apiKey: void 0,
|
|
465
|
+
options: configOptions
|
|
466
|
+
},
|
|
467
|
+
speaker: typeof configOptions.speaker === "string" ? configOptions.speaker : "matthew"
|
|
468
|
+
};
|
|
469
|
+
}
|
|
470
|
+
super(normalizedConfig);
|
|
471
|
+
const options = normalizedConfig.realtimeConfig?.options || config;
|
|
472
|
+
this.region = options.region || DEFAULT_REGION;
|
|
473
|
+
this.model = options.model || DEFAULT_MODEL;
|
|
474
|
+
this.credentials = options.credentials;
|
|
475
|
+
this.debug = options.debug || false;
|
|
476
|
+
this.sessionConfig = options.sessionConfig;
|
|
477
|
+
this.events = {};
|
|
478
|
+
this.speakerStreams = /* @__PURE__ */ new Map();
|
|
479
|
+
const validRegions = [
|
|
480
|
+
"us-east-1",
|
|
481
|
+
"us-west-2",
|
|
482
|
+
"ap-northeast-1"
|
|
483
|
+
];
|
|
484
|
+
if (!validRegions.includes(this.region)) throw new NovaSonicError("region_invalid", `Invalid region: ${this.region}. Supported regions: ${validRegions.join(", ")}`);
|
|
485
|
+
}
|
|
486
|
+
/**
|
|
487
|
+
* Returns a list of available voice speakers.
|
|
488
|
+
*
|
|
489
|
+
* Nova 2 Sonic provides expressive voices across multiple languages.
|
|
490
|
+
* Tiffany (en-US, feminine) and Matthew (en-US, masculine) are polyglot
|
|
491
|
+
* voices that can speak all supported languages.
|
|
492
|
+
*
|
|
493
|
+
* @returns Promise resolving to an array of voice objects
|
|
494
|
+
*/
|
|
495
|
+
async getSpeakers() {
|
|
496
|
+
return Promise.resolve([
|
|
497
|
+
{
|
|
498
|
+
voiceId: "tiffany",
|
|
499
|
+
name: "Tiffany",
|
|
500
|
+
language: "English",
|
|
501
|
+
locale: "en-US",
|
|
502
|
+
gender: "feminine",
|
|
503
|
+
polyglot: true
|
|
504
|
+
},
|
|
505
|
+
{
|
|
506
|
+
voiceId: "matthew",
|
|
507
|
+
name: "Matthew",
|
|
508
|
+
language: "English",
|
|
509
|
+
locale: "en-US",
|
|
510
|
+
gender: "masculine",
|
|
511
|
+
polyglot: true
|
|
512
|
+
},
|
|
513
|
+
{
|
|
514
|
+
voiceId: "amy",
|
|
515
|
+
name: "Amy",
|
|
516
|
+
language: "English",
|
|
517
|
+
locale: "en-GB",
|
|
518
|
+
gender: "feminine",
|
|
519
|
+
polyglot: false
|
|
520
|
+
},
|
|
521
|
+
{
|
|
522
|
+
voiceId: "olivia",
|
|
523
|
+
name: "Olivia",
|
|
524
|
+
language: "English",
|
|
525
|
+
locale: "en-AU",
|
|
526
|
+
gender: "feminine",
|
|
527
|
+
polyglot: false
|
|
528
|
+
},
|
|
529
|
+
{
|
|
530
|
+
voiceId: "kiara",
|
|
531
|
+
name: "Kiara",
|
|
532
|
+
language: "English",
|
|
533
|
+
locale: "en-IN",
|
|
534
|
+
gender: "feminine",
|
|
535
|
+
polyglot: false
|
|
536
|
+
},
|
|
537
|
+
{
|
|
538
|
+
voiceId: "arjun",
|
|
539
|
+
name: "Arjun",
|
|
540
|
+
language: "English",
|
|
541
|
+
locale: "en-IN",
|
|
542
|
+
gender: "masculine",
|
|
543
|
+
polyglot: false
|
|
544
|
+
},
|
|
545
|
+
{
|
|
546
|
+
voiceId: "ambre",
|
|
547
|
+
name: "Ambre",
|
|
548
|
+
language: "French",
|
|
549
|
+
locale: "fr-FR",
|
|
550
|
+
gender: "feminine",
|
|
551
|
+
polyglot: false
|
|
552
|
+
},
|
|
553
|
+
{
|
|
554
|
+
voiceId: "florian",
|
|
555
|
+
name: "Florian",
|
|
556
|
+
language: "French",
|
|
557
|
+
locale: "fr-FR",
|
|
558
|
+
gender: "masculine",
|
|
559
|
+
polyglot: false
|
|
560
|
+
},
|
|
561
|
+
{
|
|
562
|
+
voiceId: "beatrice",
|
|
563
|
+
name: "Beatrice",
|
|
564
|
+
language: "Italian",
|
|
565
|
+
locale: "it-IT",
|
|
566
|
+
gender: "feminine",
|
|
567
|
+
polyglot: false
|
|
568
|
+
},
|
|
569
|
+
{
|
|
570
|
+
voiceId: "lorenzo",
|
|
571
|
+
name: "Lorenzo",
|
|
572
|
+
language: "Italian",
|
|
573
|
+
locale: "it-IT",
|
|
574
|
+
gender: "masculine",
|
|
575
|
+
polyglot: false
|
|
576
|
+
},
|
|
577
|
+
{
|
|
578
|
+
voiceId: "tina",
|
|
579
|
+
name: "Tina",
|
|
580
|
+
language: "German",
|
|
581
|
+
locale: "de-DE",
|
|
582
|
+
gender: "feminine",
|
|
583
|
+
polyglot: false
|
|
584
|
+
},
|
|
585
|
+
{
|
|
586
|
+
voiceId: "lennart",
|
|
587
|
+
name: "Lennart",
|
|
588
|
+
language: "German",
|
|
589
|
+
locale: "de-DE",
|
|
590
|
+
gender: "masculine",
|
|
591
|
+
polyglot: false
|
|
592
|
+
},
|
|
593
|
+
{
|
|
594
|
+
voiceId: "lupe",
|
|
595
|
+
name: "Lupe",
|
|
596
|
+
language: "Spanish",
|
|
597
|
+
locale: "es-US",
|
|
598
|
+
gender: "feminine",
|
|
599
|
+
polyglot: false
|
|
600
|
+
},
|
|
601
|
+
{
|
|
602
|
+
voiceId: "carlos",
|
|
603
|
+
name: "Carlos",
|
|
604
|
+
language: "Spanish",
|
|
605
|
+
locale: "es-US",
|
|
606
|
+
gender: "masculine",
|
|
607
|
+
polyglot: false
|
|
608
|
+
},
|
|
609
|
+
{
|
|
610
|
+
voiceId: "carolina",
|
|
611
|
+
name: "Carolina",
|
|
612
|
+
language: "Portuguese",
|
|
613
|
+
locale: "pt-BR",
|
|
614
|
+
gender: "feminine",
|
|
615
|
+
polyglot: false
|
|
616
|
+
},
|
|
617
|
+
{
|
|
618
|
+
voiceId: "leo",
|
|
619
|
+
name: "Leo",
|
|
620
|
+
language: "Portuguese",
|
|
621
|
+
locale: "pt-BR",
|
|
622
|
+
gender: "masculine",
|
|
623
|
+
polyglot: false
|
|
624
|
+
},
|
|
625
|
+
{
|
|
626
|
+
voiceId: "kiara",
|
|
627
|
+
name: "Kiara",
|
|
628
|
+
language: "Hindi",
|
|
629
|
+
locale: "hi-IN",
|
|
630
|
+
gender: "feminine",
|
|
631
|
+
polyglot: false
|
|
632
|
+
},
|
|
633
|
+
{
|
|
634
|
+
voiceId: "arjun",
|
|
635
|
+
name: "Arjun",
|
|
636
|
+
language: "Hindi",
|
|
637
|
+
locale: "hi-IN",
|
|
638
|
+
gender: "masculine",
|
|
639
|
+
polyglot: false
|
|
640
|
+
}
|
|
641
|
+
]);
|
|
642
|
+
}
|
|
643
|
+
/**
|
|
644
|
+
* Establishes a connection to the AWS Bedrock bidirectional streaming service.
|
|
645
|
+
* Must be called before using speak, listen, or send functions.
|
|
646
|
+
*
|
|
647
|
+
* @throws {NovaSonicError} If connection fails or credentials are missing
|
|
648
|
+
*
|
|
649
|
+
* @example
|
|
650
|
+
* ```typescript
|
|
651
|
+
* await voice.connect();
|
|
652
|
+
* // Now ready for voice interactions
|
|
653
|
+
* ```
|
|
654
|
+
*/
|
|
655
|
+
async connect({ requestContext } = {}) {
|
|
656
|
+
if (this.state === "connected" || this.state === "connecting") {
|
|
657
|
+
this.log("Already connected or connecting");
|
|
658
|
+
return;
|
|
659
|
+
}
|
|
660
|
+
this.state = "connecting";
|
|
661
|
+
this.requestContext = requestContext;
|
|
662
|
+
this.streamRestartAttempted = false;
|
|
663
|
+
try {
|
|
664
|
+
await this.createBedrockClient();
|
|
665
|
+
const asyncIterable = this.createEventQueue();
|
|
666
|
+
this.enqueueInitialSessionEvents();
|
|
667
|
+
await this.sendInitialConnectCommand(asyncIterable);
|
|
668
|
+
this.processStream().catch((error) => {
|
|
669
|
+
this.log("Error in stream processing:", error);
|
|
670
|
+
this.emit("error", {
|
|
671
|
+
message: error instanceof Error ? error.message : "Stream processing error",
|
|
672
|
+
code: "STREAM_PROCESSING_ERROR",
|
|
673
|
+
details: error
|
|
674
|
+
});
|
|
675
|
+
});
|
|
676
|
+
this.log("Connected to AWS Bedrock Nova 2 Sonic");
|
|
677
|
+
} catch (error) {
|
|
678
|
+
this.state = "disconnected";
|
|
679
|
+
if (this.client) {
|
|
680
|
+
if (typeof this.client.destroy === "function") this.client.destroy();
|
|
681
|
+
this.client = void 0;
|
|
682
|
+
}
|
|
683
|
+
this.log("Connection error:", error);
|
|
684
|
+
throw new NovaSonicError("connection_failed", `Failed to connect to AWS Bedrock: ${error instanceof Error ? error.message : "Unknown error during connection"}`, error);
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
/**
|
|
688
|
+
* Resolve credentials and initialize the Bedrock Runtime client over HTTP/2.
|
|
689
|
+
*/
|
|
690
|
+
async createBedrockClient() {
|
|
691
|
+
this.log("Getting AWS credentials...");
|
|
692
|
+
const credentials = await getAwsCredentials(this.credentials, this.debug);
|
|
693
|
+
if (!credentials) throw new NovaSonicError("credentials_missing", "AWS credentials are required. Please configure AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables or provide credentials in the config.");
|
|
694
|
+
this.log("Credentials retrieved:", {
|
|
695
|
+
hasAccessKeyId: !!credentials.accessKeyId,
|
|
696
|
+
hasSecretAccessKey: !!credentials.secretAccessKey,
|
|
697
|
+
hasSessionToken: !!credentials.sessionToken,
|
|
698
|
+
accessKeyIdPrefix: credentials.accessKeyId ? `${credentials.accessKeyId.substring(0, 6)}...` : "missing",
|
|
699
|
+
expiration: credentials.expiration ? credentials.expiration.toISOString() : "no expiration"
|
|
700
|
+
});
|
|
701
|
+
this.log(`Initializing Bedrock Runtime client for region: ${this.region}, model: ${this.model}`);
|
|
702
|
+
const nodeHttp2Handler = new NodeHttp2Handler({
|
|
703
|
+
requestTimeout: 3e5,
|
|
704
|
+
sessionTimeout: 3e5,
|
|
705
|
+
disableConcurrentStreams: false,
|
|
706
|
+
maxConcurrentStreams: 20
|
|
707
|
+
});
|
|
708
|
+
this.client = new BedrockRuntimeClient({
|
|
709
|
+
region: this.region,
|
|
710
|
+
credentials,
|
|
711
|
+
requestHandler: nodeHttp2Handler
|
|
712
|
+
});
|
|
713
|
+
}
|
|
714
|
+
/**
|
|
715
|
+
* Build the async-iterable event queue used as the request body for the
|
|
716
|
+
* bidirectional stream. Returns the iterable and wires up internal queue
|
|
717
|
+
* helpers (_eventQueue, _signalQueue, _closeSignal) used by sendClientEvent.
|
|
718
|
+
*/
|
|
719
|
+
createEventQueue() {
|
|
720
|
+
this.log("Creating bidirectional stream command...");
|
|
721
|
+
const voiceInstance = this;
|
|
722
|
+
const eventQueue = [];
|
|
723
|
+
const pendingResolvers = [];
|
|
724
|
+
let closeSignal = false;
|
|
725
|
+
const signalQueue = () => {
|
|
726
|
+
if (pendingResolvers.length > 0) {
|
|
727
|
+
voiceInstance.log(`[AsyncIterable] Signaling queue - resolving ${pendingResolvers.length} pending Promise(s)`);
|
|
728
|
+
const resolvers = [...pendingResolvers];
|
|
729
|
+
pendingResolvers.length = 0;
|
|
730
|
+
resolvers.forEach((resolve) => resolve());
|
|
731
|
+
} else voiceInstance.log("[AsyncIterable] signalQueue called but no pending Promise");
|
|
732
|
+
};
|
|
733
|
+
const asyncIterable = { [Symbol.asyncIterator]: () => {
|
|
734
|
+
voiceInstance.log("[AsyncIterable] Iterator requested");
|
|
735
|
+
return {
|
|
736
|
+
next: async () => {
|
|
737
|
+
try {
|
|
738
|
+
if (closeSignal || voiceInstance.state === "disconnected") {
|
|
739
|
+
voiceInstance.log(`[AsyncIterable] Stream closed (state: ${voiceInstance.state}), done = true`);
|
|
740
|
+
return {
|
|
741
|
+
value: void 0,
|
|
742
|
+
done: true
|
|
743
|
+
};
|
|
744
|
+
}
|
|
745
|
+
if (eventQueue.length === 0) try {
|
|
746
|
+
voiceInstance.log("[AsyncIterable] Queue empty, waiting for signal...");
|
|
747
|
+
await new Promise((resolve) => {
|
|
748
|
+
pendingResolvers.push(resolve);
|
|
749
|
+
voiceInstance.log(`[AsyncIterable] Promise created, waiting for signal (${pendingResolvers.length} pending)...`);
|
|
750
|
+
setImmediate(() => {
|
|
751
|
+
if (eventQueue.length > 0) {
|
|
752
|
+
voiceInstance.log("[AsyncIterable] Data arrived before wait, resolving immediately");
|
|
753
|
+
const index = pendingResolvers.indexOf(resolve);
|
|
754
|
+
if (index !== -1) {
|
|
755
|
+
pendingResolvers.splice(index, 1);
|
|
756
|
+
resolve();
|
|
757
|
+
}
|
|
758
|
+
return;
|
|
759
|
+
}
|
|
760
|
+
if (closeSignal || voiceInstance.state === "disconnected") {
|
|
761
|
+
voiceInstance.log("[AsyncIterable] Closed before wait, resolving");
|
|
762
|
+
const index = pendingResolvers.indexOf(resolve);
|
|
763
|
+
if (index !== -1) {
|
|
764
|
+
pendingResolvers.splice(index, 1);
|
|
765
|
+
resolve();
|
|
766
|
+
}
|
|
767
|
+
return;
|
|
768
|
+
}
|
|
769
|
+
});
|
|
770
|
+
});
|
|
771
|
+
voiceInstance.log("[AsyncIterable] Promise resolved, checking queue...");
|
|
772
|
+
} catch (error) {
|
|
773
|
+
if (error instanceof Error && error.message === "Stream closed") {
|
|
774
|
+
voiceInstance.log("[AsyncIterable] Stream closed during wait");
|
|
775
|
+
return {
|
|
776
|
+
value: void 0,
|
|
777
|
+
done: true
|
|
778
|
+
};
|
|
779
|
+
}
|
|
780
|
+
voiceInstance.log("[AsyncIterable] Error during wait:", error);
|
|
781
|
+
}
|
|
782
|
+
if (closeSignal) {
|
|
783
|
+
voiceInstance.log("[AsyncIterable] Stream closed (closeSignal)");
|
|
784
|
+
return {
|
|
785
|
+
value: void 0,
|
|
786
|
+
done: true
|
|
787
|
+
};
|
|
788
|
+
}
|
|
789
|
+
if (voiceInstance.state === "disconnected") {
|
|
790
|
+
voiceInstance.log("[AsyncIterable] Stream closed (disconnected state)");
|
|
791
|
+
return {
|
|
792
|
+
value: void 0,
|
|
793
|
+
done: true
|
|
794
|
+
};
|
|
795
|
+
}
|
|
796
|
+
while (eventQueue.length === 0 && !closeSignal) {
|
|
797
|
+
if (voiceInstance.state === "disconnected") {
|
|
798
|
+
voiceInstance.log("[AsyncIterable] Stream closed before wait loop");
|
|
799
|
+
return {
|
|
800
|
+
value: void 0,
|
|
801
|
+
done: true
|
|
802
|
+
};
|
|
803
|
+
}
|
|
804
|
+
voiceInstance.log("[AsyncIterable] Queue still empty, waiting again...");
|
|
805
|
+
await new Promise((resolve) => {
|
|
806
|
+
pendingResolvers.push(resolve);
|
|
807
|
+
setImmediate(() => {
|
|
808
|
+
if (eventQueue.length > 0 || closeSignal || voiceInstance.state === "disconnected") {
|
|
809
|
+
const index = pendingResolvers.indexOf(resolve);
|
|
810
|
+
if (index !== -1) {
|
|
811
|
+
pendingResolvers.splice(index, 1);
|
|
812
|
+
resolve();
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
});
|
|
816
|
+
});
|
|
817
|
+
if (closeSignal || voiceInstance.state === "disconnected") {
|
|
818
|
+
voiceInstance.log("[AsyncIterable] Stream closed during wait loop");
|
|
819
|
+
return {
|
|
820
|
+
value: void 0,
|
|
821
|
+
done: true
|
|
822
|
+
};
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
const nextEvent = eventQueue.shift();
|
|
826
|
+
const eventJson = JSON.stringify(nextEvent);
|
|
827
|
+
const eventBytes = Buffer.from(eventJson, "utf-8");
|
|
828
|
+
voiceInstance.log(`[AsyncIterable] Yielding event of size: ${eventBytes.length}`);
|
|
829
|
+
return {
|
|
830
|
+
value: { chunk: { bytes: eventBytes } },
|
|
831
|
+
done: false
|
|
832
|
+
};
|
|
833
|
+
} catch (error) {
|
|
834
|
+
voiceInstance.log("[AsyncIterable] Error in iterator:", error);
|
|
835
|
+
closeSignal = true;
|
|
836
|
+
return {
|
|
837
|
+
value: void 0,
|
|
838
|
+
done: true
|
|
839
|
+
};
|
|
840
|
+
}
|
|
841
|
+
},
|
|
842
|
+
return: async () => {
|
|
843
|
+
voiceInstance.log("[AsyncIterable] Iterator return() called");
|
|
844
|
+
closeSignal = true;
|
|
845
|
+
signalQueue();
|
|
846
|
+
return {
|
|
847
|
+
value: void 0,
|
|
848
|
+
done: true
|
|
849
|
+
};
|
|
850
|
+
},
|
|
851
|
+
throw: async (error) => {
|
|
852
|
+
voiceInstance.log("[AsyncIterable] Iterator throw() called:", error);
|
|
853
|
+
closeSignal = true;
|
|
854
|
+
error instanceof Error || new Error(String(error));
|
|
855
|
+
signalQueue();
|
|
856
|
+
throw error;
|
|
857
|
+
}
|
|
858
|
+
};
|
|
859
|
+
} };
|
|
860
|
+
this._eventQueue = eventQueue;
|
|
861
|
+
this._signalQueue = signalQueue;
|
|
862
|
+
this._closeSignal = () => {
|
|
863
|
+
closeSignal = true;
|
|
864
|
+
signalQueue();
|
|
865
|
+
};
|
|
866
|
+
return asyncIterable;
|
|
867
|
+
}
|
|
868
|
+
/**
|
|
869
|
+
* Pre-populate the event queue with the AWS Nova Sonic connection
|
|
870
|
+
* handshake events: sessionStart, promptStart, then a SYSTEM text content
|
|
871
|
+
* block carrying the configured instructions. AUDIO contentStart is NOT
|
|
872
|
+
* sent here; it is deferred to the first send() call.
|
|
873
|
+
*/
|
|
874
|
+
enqueueInitialSessionEvents() {
|
|
875
|
+
const eventQueue = this._eventQueue;
|
|
876
|
+
if (!eventQueue) throw new NovaSonicError("connection_failed", "Event queue must be initialized before enqueueing session events");
|
|
877
|
+
this.log("Pre-populating queue with sessionStart and promptStart events...");
|
|
878
|
+
const promptName = randomUUID();
|
|
879
|
+
this._promptName = promptName;
|
|
880
|
+
const sessionStartEvent = {};
|
|
881
|
+
if (this.sessionConfig) {
|
|
882
|
+
if (this.sessionConfig.inferenceConfiguration) sessionStartEvent.inferenceConfiguration = {
|
|
883
|
+
maxTokens: this.sessionConfig.inferenceConfiguration.maxTokens || 4096,
|
|
884
|
+
topP: this.sessionConfig.inferenceConfiguration.topP || .9,
|
|
885
|
+
temperature: this.sessionConfig.inferenceConfiguration.temperature || .7,
|
|
886
|
+
...this.sessionConfig.inferenceConfiguration.topK !== void 0 && { topK: this.sessionConfig.inferenceConfiguration.topK },
|
|
887
|
+
...this.sessionConfig.inferenceConfiguration.stopSequences && { stopSequences: this.sessionConfig.inferenceConfiguration.stopSequences }
|
|
888
|
+
};
|
|
889
|
+
else sessionStartEvent.inferenceConfiguration = {
|
|
890
|
+
maxTokens: 4096,
|
|
891
|
+
topP: .9,
|
|
892
|
+
temperature: .7
|
|
893
|
+
};
|
|
894
|
+
if (this.sessionConfig.turnDetectionConfiguration) sessionStartEvent.turnDetectionConfiguration = { ...this.sessionConfig.turnDetectionConfiguration.endpointingSensitivity && { endpointingSensitivity: this.sessionConfig.turnDetectionConfiguration.endpointingSensitivity } };
|
|
895
|
+
} else sessionStartEvent.inferenceConfiguration = {
|
|
896
|
+
maxTokens: 4096,
|
|
897
|
+
topP: .9,
|
|
898
|
+
temperature: .7
|
|
899
|
+
};
|
|
900
|
+
eventQueue.push({ event: { sessionStart: sessionStartEvent } });
|
|
901
|
+
let voiceId = "matthew";
|
|
902
|
+
if (this.sessionConfig?.voice) {
|
|
903
|
+
if (typeof this.sessionConfig.voice === "string") voiceId = this.sessionConfig.voice;
|
|
904
|
+
else if (this.sessionConfig.voice.name) voiceId = this.sessionConfig.voice.name;
|
|
905
|
+
} else if (this.speaker && this.speaker !== "default") if (typeof this.speaker === "string") voiceId = this.speaker;
|
|
906
|
+
else {
|
|
907
|
+
const speakerObj = this.speaker;
|
|
908
|
+
if (speakerObj && typeof speakerObj === "object" && speakerObj.name) voiceId = speakerObj.name;
|
|
909
|
+
}
|
|
910
|
+
const promptStartEvent = {
|
|
911
|
+
promptName,
|
|
912
|
+
textOutputConfiguration: { mediaType: "text/plain" },
|
|
913
|
+
audioOutputConfiguration: {
|
|
914
|
+
mediaType: "audio/lpcm",
|
|
915
|
+
sampleRateHertz: 24e3,
|
|
916
|
+
sampleSizeBits: 16,
|
|
917
|
+
channelCount: 1,
|
|
918
|
+
voiceId,
|
|
919
|
+
encoding: "base64",
|
|
920
|
+
audioType: "SPEECH"
|
|
921
|
+
}
|
|
922
|
+
};
|
|
923
|
+
if (this.sessionConfig?.tools && this.sessionConfig.tools.length > 0) promptStartEvent.toolConfiguration = {
|
|
924
|
+
tools: this.sessionConfig.tools.map((tool) => {
|
|
925
|
+
let inputSchemaJson;
|
|
926
|
+
if (typeof tool.inputSchema === "string") inputSchemaJson = tool.inputSchema;
|
|
927
|
+
else inputSchemaJson = JSON.stringify(tool.inputSchema);
|
|
928
|
+
return { toolSpec: {
|
|
929
|
+
name: tool.name,
|
|
930
|
+
description: tool.description,
|
|
931
|
+
inputSchema: { json: inputSchemaJson }
|
|
932
|
+
} };
|
|
933
|
+
}),
|
|
934
|
+
...this.sessionConfig?.toolChoice && { toolChoice: this.sessionConfig.toolChoice }
|
|
935
|
+
};
|
|
936
|
+
else if (this.sessionConfig?.toolChoice) promptStartEvent.toolConfiguration = { toolChoice: this.sessionConfig.toolChoice };
|
|
937
|
+
eventQueue.push({ event: { promptStart: promptStartEvent } });
|
|
938
|
+
this.promptStarted = true;
|
|
939
|
+
const systemContentName = randomUUID();
|
|
940
|
+
eventQueue.push({ event: { contentStart: {
|
|
941
|
+
promptName,
|
|
942
|
+
contentName: systemContentName,
|
|
943
|
+
type: "TEXT",
|
|
944
|
+
interactive: false,
|
|
945
|
+
role: "SYSTEM",
|
|
946
|
+
textInputConfiguration: { mediaType: "text/plain" }
|
|
947
|
+
} } });
|
|
948
|
+
eventQueue.push({ event: { textInput: {
|
|
949
|
+
promptName,
|
|
950
|
+
contentName: systemContentName,
|
|
951
|
+
content: this.instructions || ""
|
|
952
|
+
} } });
|
|
953
|
+
eventQueue.push({ event: { contentEnd: {
|
|
954
|
+
promptName,
|
|
955
|
+
contentName: systemContentName
|
|
956
|
+
} } });
|
|
957
|
+
this.audioContentStarted = false;
|
|
958
|
+
this.log(`Queue pre-populated with ${eventQueue.length} event(s)`);
|
|
959
|
+
}
|
|
960
|
+
/**
|
|
961
|
+
* Issue the InvokeModelWithBidirectionalStreamCommand to AWS Bedrock with
|
|
962
|
+
* a 5-second abort timeout that tears down the client on hang to avoid
|
|
963
|
+
* leaked HTTP/2 sessions. On success the response stream is stored and the
|
|
964
|
+
* voice transitions to 'connected'.
|
|
965
|
+
*/
|
|
966
|
+
async sendInitialConnectCommand(asyncIterable) {
|
|
967
|
+
if (!this.client) throw new NovaSonicError("connection_failed", "Bedrock client must be created before sending the initial command");
|
|
968
|
+
const command = new InvokeModelWithBidirectionalStreamCommand({
|
|
969
|
+
modelId: this.model,
|
|
970
|
+
body: asyncIterable
|
|
971
|
+
});
|
|
972
|
+
const sendStartTime = Date.now();
|
|
973
|
+
const abortController = new AbortController();
|
|
974
|
+
const timeoutId = setTimeout(() => {
|
|
975
|
+
this.log("[DEBUG] client.send() timeout after 5 seconds - aborting request");
|
|
976
|
+
abortController.abort();
|
|
977
|
+
}, 5e3);
|
|
978
|
+
let response;
|
|
979
|
+
try {
|
|
980
|
+
response = await this.client.send(command, { abortSignal: abortController.signal });
|
|
981
|
+
} catch (error) {
|
|
982
|
+
const sendDuration = Date.now() - sendStartTime;
|
|
983
|
+
if (abortController.signal.aborted) {
|
|
984
|
+
this.log(`[DEBUG] client.send() aborted after ${sendDuration}ms`);
|
|
985
|
+
this._closeSignal?.();
|
|
986
|
+
this.client.destroy();
|
|
987
|
+
throw new Error("client.send() timeout");
|
|
988
|
+
}
|
|
989
|
+
this.log(`[DEBUG] client.send() error after ${sendDuration}ms:`, error);
|
|
990
|
+
throw error;
|
|
991
|
+
} finally {
|
|
992
|
+
clearTimeout(timeoutId);
|
|
993
|
+
}
|
|
994
|
+
const sendDuration = Date.now() - sendStartTime;
|
|
995
|
+
this.log(`[DEBUG] client.send() completed in ${sendDuration}ms`);
|
|
996
|
+
this.log("Received response from AWS Bedrock");
|
|
997
|
+
this.stream = response.body;
|
|
998
|
+
this.log(`[DEBUG] Response stream is async iterable: ${this.stream && typeof this.stream[Symbol.asyncIterator] === "function"}`);
|
|
999
|
+
this.state = "connected";
|
|
1000
|
+
this.log(`[STATE] State set to 'connected'`);
|
|
1001
|
+
}
|
|
1002
|
+
/**
|
|
1003
|
+
* Process the bidirectional stream from AWS Bedrock
|
|
1004
|
+
*/
|
|
1005
|
+
async processStream() {
|
|
1006
|
+
if (!this.stream) {
|
|
1007
|
+
this.log("[Stream] No stream available, cannot process");
|
|
1008
|
+
return;
|
|
1009
|
+
}
|
|
1010
|
+
if (this.processingStream) {
|
|
1011
|
+
this.log("[Stream] Already processing stream, skipping");
|
|
1012
|
+
return;
|
|
1013
|
+
}
|
|
1014
|
+
this.processingStream = true;
|
|
1015
|
+
this.log("[Stream] Starting stream processing");
|
|
1016
|
+
let eventCount = 0;
|
|
1017
|
+
let lastEventTime = Date.now();
|
|
1018
|
+
try {
|
|
1019
|
+
for await (const chunk of this.stream) if (chunk.chunk) {
|
|
1020
|
+
const textResponse = Buffer.from(chunk.chunk.bytes || []).toString("utf-8");
|
|
1021
|
+
eventCount++;
|
|
1022
|
+
const now = Date.now();
|
|
1023
|
+
const timeSinceLastEvent = now - lastEventTime;
|
|
1024
|
+
lastEventTime = now;
|
|
1025
|
+
this.log(`[Stream] Received chunk #${eventCount}, length: ${textResponse.length}, time since last: ${timeSinceLastEvent}ms`);
|
|
1026
|
+
try {
|
|
1027
|
+
const jsonResponse = JSON.parse(textResponse);
|
|
1028
|
+
this.log(`[Stream] ========================================`);
|
|
1029
|
+
this.log(`[Stream] Parsed JSON response, keys: ${Object.keys(jsonResponse).join(", ")}`);
|
|
1030
|
+
if (jsonResponse.event) {
|
|
1031
|
+
const eventKeys = Object.keys(jsonResponse.event);
|
|
1032
|
+
this.log(`[Stream] Event keys: ${eventKeys.join(", ")}`);
|
|
1033
|
+
if (jsonResponse.event.contentStart) {
|
|
1034
|
+
this.log(`[Stream] → Handling contentStart`);
|
|
1035
|
+
this.handleServerEvent({ contentStart: jsonResponse.event.contentStart });
|
|
1036
|
+
} else if (jsonResponse.event.textOutput) {
|
|
1037
|
+
this.log(`[Stream] → Handling textOutput, content length: ${jsonResponse.event.textOutput?.content?.length ?? 0}`);
|
|
1038
|
+
this.handleServerEvent({ textOutput: jsonResponse.event.textOutput });
|
|
1039
|
+
} else if (jsonResponse.event.audioOutput) this.handleServerEvent({ audioOutput: jsonResponse.event.audioOutput });
|
|
1040
|
+
else if (jsonResponse.event.toolUse) this.handleServerEvent({ toolUse: jsonResponse.event.toolUse });
|
|
1041
|
+
else if (jsonResponse.event.contentEnd && jsonResponse.event.contentEnd.type === "TOOL") this.handleServerEvent({ contentEnd: jsonResponse.event.contentEnd });
|
|
1042
|
+
else if (jsonResponse.event.contentEnd) {
|
|
1043
|
+
this.log(`[Stream] Found contentEnd, type: ${jsonResponse.event.contentEnd.type}, stopReason: ${jsonResponse.event.contentEnd.stopReason}`);
|
|
1044
|
+
this.handleServerEvent({ contentEnd: jsonResponse.event.contentEnd });
|
|
1045
|
+
} else if (jsonResponse.event.completionStart) {
|
|
1046
|
+
this.log("[Stream] Found completionStart inside event object:", JSON.stringify(jsonResponse.event.completionStart, null, 2));
|
|
1047
|
+
this.emit("completionStart", jsonResponse.event.completionStart);
|
|
1048
|
+
} else if (jsonResponse.event.completionEnd) {
|
|
1049
|
+
this.log("[Stream] Found completionEnd inside event object:", JSON.stringify(jsonResponse.event.completionEnd, null, 2));
|
|
1050
|
+
this.handleServerEvent({ completionEnd: jsonResponse.event.completionEnd });
|
|
1051
|
+
} else {
|
|
1052
|
+
const eventKeys = Object.keys(jsonResponse.event || {});
|
|
1053
|
+
this.log(`[Stream] Event keys for other events: ${eventKeys.join(", ")}`);
|
|
1054
|
+
if (eventKeys.length > 0) if (eventKeys.includes("completionEnd")) {
|
|
1055
|
+
this.log("[Stream] Found completionEnd in other events, handling explicitly");
|
|
1056
|
+
this.handleServerEvent({ completionEnd: jsonResponse.event.completionEnd });
|
|
1057
|
+
} else {
|
|
1058
|
+
const eventKey = eventKeys[0];
|
|
1059
|
+
this.log(`[Stream] Dispatching other event: ${eventKey}`);
|
|
1060
|
+
const eventValue = jsonResponse.event[eventKey];
|
|
1061
|
+
if (eventValue !== void 0) if (eventKey === "completionEnd") this.handleServerEvent({ completionEnd: eventValue });
|
|
1062
|
+
else this.handleServerEvent({ [eventKey]: eventValue });
|
|
1063
|
+
}
|
|
1064
|
+
else if (Object.keys(jsonResponse).length > 0) this.log(`[Stream] Unknown event structure, keys:`, Object.keys(jsonResponse).join(", "));
|
|
1065
|
+
}
|
|
1066
|
+
} else {
|
|
1067
|
+
if (this.debug) this.log("[Stream] Received event without \"event\" wrapper, keys:", Object.keys(jsonResponse).join(", "));
|
|
1068
|
+
if (jsonResponse.usageEvent) this.emit("usage", {
|
|
1069
|
+
inputTokens: jsonResponse.usageEvent.totalInputTokens || 0,
|
|
1070
|
+
outputTokens: jsonResponse.usageEvent.totalOutputTokens || 0,
|
|
1071
|
+
totalTokens: jsonResponse.usageEvent.totalTokens || 0
|
|
1072
|
+
});
|
|
1073
|
+
if (jsonResponse.completionEnd) {
|
|
1074
|
+
this.log("[Stream] Found completionEnd at top level:", JSON.stringify(jsonResponse.completionEnd, null, 2));
|
|
1075
|
+
this.handleServerEvent({ completionEnd: jsonResponse.completionEnd });
|
|
1076
|
+
}
|
|
1077
|
+
if (!jsonResponse.event && !jsonResponse.completionEnd && !jsonResponse.usageEvent) this.log("[Stream] Received response without event wrapper, keys:", Object.keys(jsonResponse).join(", "));
|
|
1078
|
+
if (jsonResponse.completionStart || jsonResponse.event?.completionStart) {
|
|
1079
|
+
const completionStart = jsonResponse.completionStart || jsonResponse.event.completionStart;
|
|
1080
|
+
this.log("[Stream] Found completionStart:", JSON.stringify(completionStart, null, 2));
|
|
1081
|
+
this.emit("completionStart", completionStart);
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
1084
|
+
} catch (parseError) {
|
|
1085
|
+
this.log("[Stream] Failed to parse JSON response:", textResponse.substring(0, 200));
|
|
1086
|
+
this.emit("error", {
|
|
1087
|
+
message: "Failed to parse stream response",
|
|
1088
|
+
code: "PARSE_ERROR",
|
|
1089
|
+
details: parseError
|
|
1090
|
+
});
|
|
1091
|
+
}
|
|
1092
|
+
} else if (chunk.internalServerException) this.emit("error", {
|
|
1093
|
+
message: "Internal server error",
|
|
1094
|
+
code: "INTERNAL_SERVER_ERROR",
|
|
1095
|
+
details: chunk.internalServerException
|
|
1096
|
+
});
|
|
1097
|
+
else if (chunk.modelStreamErrorException) this.emit("error", {
|
|
1098
|
+
message: "Model stream error",
|
|
1099
|
+
code: "MODEL_STREAM_ERROR",
|
|
1100
|
+
details: chunk.modelStreamErrorException
|
|
1101
|
+
});
|
|
1102
|
+
else if (chunk.modelTimeoutException) this.emit("error", {
|
|
1103
|
+
message: "Model timeout",
|
|
1104
|
+
code: "MODEL_TIMEOUT",
|
|
1105
|
+
details: chunk.modelTimeoutException
|
|
1106
|
+
});
|
|
1107
|
+
else if (chunk.serviceUnavailableException) this.emit("error", {
|
|
1108
|
+
message: "Service unavailable",
|
|
1109
|
+
code: "SERVICE_UNAVAILABLE",
|
|
1110
|
+
details: chunk.serviceUnavailableException
|
|
1111
|
+
});
|
|
1112
|
+
else if (chunk.throttlingException) this.emit("error", {
|
|
1113
|
+
message: "Request throttled",
|
|
1114
|
+
code: "THROTTLING",
|
|
1115
|
+
details: chunk.throttlingException
|
|
1116
|
+
});
|
|
1117
|
+
else if (chunk.validationException) this.emit("error", {
|
|
1118
|
+
message: "Validation error",
|
|
1119
|
+
code: "VALIDATION_ERROR",
|
|
1120
|
+
details: chunk.validationException
|
|
1121
|
+
});
|
|
1122
|
+
} catch (streamError) {
|
|
1123
|
+
this.log("[Stream] Error in processStream:", streamError);
|
|
1124
|
+
this.emit("error", {
|
|
1125
|
+
message: "Stream processing error",
|
|
1126
|
+
code: "STREAM_ERROR",
|
|
1127
|
+
details: streamError instanceof Error ? streamError.message : String(streamError)
|
|
1128
|
+
});
|
|
1129
|
+
} finally {
|
|
1130
|
+
this.processingStream = false;
|
|
1131
|
+
this.log(`[Stream] processStream finished, processingStream set to false. Total events received: ${eventCount || 0}`);
|
|
1132
|
+
this.log(`[Stream] Stream state: state=${this.state}, stream exists=${!!this.stream}`);
|
|
1133
|
+
if (!this.turnCompleted && this.audioContentStarted) {
|
|
1134
|
+
this.log("[Stream] Stream ended but turn not completed - signaling turn completion as fallback");
|
|
1135
|
+
this.log(`[Stream] State: turnCompleted=${this.turnCompleted}, audioContentStarted=${this.audioContentStarted}, hasSentContentEnd=${this.hasSentContentEnd}`);
|
|
1136
|
+
this.turnCompleted = true;
|
|
1137
|
+
this.emit("turnComplete", { timestamp: Date.now() });
|
|
1138
|
+
if (this.currentResponseId) {
|
|
1139
|
+
const stream = this.speakerStreams.get(this.currentResponseId);
|
|
1140
|
+
if (stream) stream.end();
|
|
1141
|
+
this.speakerStreams.delete(this.currentResponseId);
|
|
1142
|
+
this.currentResponseId = void 0;
|
|
1143
|
+
}
|
|
1144
|
+
this.hasSentContentEnd = false;
|
|
1145
|
+
this.log("[Stream] Turn completion signaled, ready for next turn");
|
|
1146
|
+
} else if (this.turnCompleted) this.log("[Stream] Stream ended and turn was already completed");
|
|
1147
|
+
else this.log(`[Stream] Stream ended but turn not completed - audioContentStarted=${this.audioContentStarted}, turnCompleted=${this.turnCompleted}`);
|
|
1148
|
+
if (this.stream && this.state === "connected" && !this.processingStream && !this.streamRestartAttempted) {
|
|
1149
|
+
this.log("[Stream] Stream still open but processing stopped - will restart stream processing");
|
|
1150
|
+
this.streamRestartAttempted = true;
|
|
1151
|
+
setImmediate(() => {
|
|
1152
|
+
if (this.stream && this.state === "connected" && !this.processingStream) {
|
|
1153
|
+
this.log("[Stream] Restarting stream processing for subsequent turns");
|
|
1154
|
+
this.processStream().catch((error) => {
|
|
1155
|
+
this.log("[Stream] Error restarting stream processing:", error);
|
|
1156
|
+
this.streamRestartAttempted = false;
|
|
1157
|
+
});
|
|
1158
|
+
} else this.streamRestartAttempted = false;
|
|
1159
|
+
});
|
|
1160
|
+
} else if (this.streamRestartAttempted) this.log("[Stream] Stream restart already attempted, skipping");
|
|
1161
|
+
}
|
|
1162
|
+
}
|
|
1163
|
+
/**
|
|
1164
|
+
* Handle server events from AWS Bedrock
|
|
1165
|
+
*/
|
|
1166
|
+
handleServerEvent(event) {
|
|
1167
|
+
if (this.debug) this.log("Received event, keys:", Object.keys(event).join(", "));
|
|
1168
|
+
if (event.contentStart) this.handleContentStart(event.contentStart);
|
|
1169
|
+
if (event.textOutput) this.handleTextOutput(event.textOutput);
|
|
1170
|
+
if (event.audioOutput?.content) this.handleAudioOutput(event.audioOutput);
|
|
1171
|
+
if (event.toolUse) this.handleToolUse(event.toolUse);
|
|
1172
|
+
if (event.contentEnd) this.handleContentEnd(event.contentEnd);
|
|
1173
|
+
if (event.completionEnd) this.handleCompletionEnd(event.completionEnd);
|
|
1174
|
+
if (event.error) this.emit("error", {
|
|
1175
|
+
message: event.error.message || "Unknown error",
|
|
1176
|
+
code: event.error.code || "UNKNOWN_ERROR",
|
|
1177
|
+
details: event.error
|
|
1178
|
+
});
|
|
1179
|
+
}
|
|
1180
|
+
/**
|
|
1181
|
+
* Handle a contentStart event. Tracks generationStage for text content
|
|
1182
|
+
* blocks so the corresponding 'writing' events can be tagged
|
|
1183
|
+
* SPECULATIVE/FINAL for the client.
|
|
1184
|
+
*/
|
|
1185
|
+
handleContentStart(contentStart) {
|
|
1186
|
+
const role = contentStart.role?.toLowerCase();
|
|
1187
|
+
const contentType = contentStart.type;
|
|
1188
|
+
this.log(`[Event] contentStart: type=${contentType || "unknown"}, role=${role}`);
|
|
1189
|
+
this.emit("contentStart", contentStart);
|
|
1190
|
+
if (contentType === "TEXT" && contentStart.additionalModelFields) try {
|
|
1191
|
+
const additionalFields = JSON.parse(contentStart.additionalModelFields);
|
|
1192
|
+
this.currentTextGenerationStage = additionalFields.generationStage;
|
|
1193
|
+
this.log(`[Event] Text content generationStage: ${this.currentTextGenerationStage}`);
|
|
1194
|
+
} catch {
|
|
1195
|
+
this.currentTextGenerationStage = void 0;
|
|
1196
|
+
}
|
|
1197
|
+
else if (contentType === "TEXT") this.currentTextGenerationStage = void 0;
|
|
1198
|
+
}
|
|
1199
|
+
/**
|
|
1200
|
+
* Handle a textOutput event. Detects interruption (barge-in) markers in
|
|
1201
|
+
* the payload, otherwise emits a 'writing' event with the text and
|
|
1202
|
+
* current generationStage.
|
|
1203
|
+
*/
|
|
1204
|
+
handleTextOutput(textOutput) {
|
|
1205
|
+
const text = textOutput.content || "";
|
|
1206
|
+
const role = textOutput.role?.toLowerCase() || "assistant";
|
|
1207
|
+
this.log(`[Event] textOutput received: role=${role}, text length=${text.length}`);
|
|
1208
|
+
let isInterrupted = false;
|
|
1209
|
+
try {
|
|
1210
|
+
const parsed = JSON.parse(text);
|
|
1211
|
+
if (parsed && parsed.interrupted === true) isInterrupted = true;
|
|
1212
|
+
} catch {
|
|
1213
|
+
if (/interrupted/i.test(text)) isInterrupted = true;
|
|
1214
|
+
}
|
|
1215
|
+
if (isInterrupted) {
|
|
1216
|
+
this.log(`[Event] Interrupt detected, emitting interrupt event`);
|
|
1217
|
+
this.emit("interrupt", {
|
|
1218
|
+
type: "user",
|
|
1219
|
+
timestamp: Date.now()
|
|
1220
|
+
});
|
|
1221
|
+
return;
|
|
1222
|
+
}
|
|
1223
|
+
const generationStage = this.currentTextGenerationStage;
|
|
1224
|
+
this.log(`[Event] Emitting 'writing': role=${role}, generationStage=${generationStage}, length=${text.length}`);
|
|
1225
|
+
this.emit("writing", {
|
|
1226
|
+
text,
|
|
1227
|
+
role,
|
|
1228
|
+
generationStage
|
|
1229
|
+
});
|
|
1230
|
+
}
|
|
1231
|
+
/**
|
|
1232
|
+
* Handle an audioOutput event. Decodes the base64 LPCM payload, emits
|
|
1233
|
+
* 'speaking' with both the base64 string and an Int16Array view, and
|
|
1234
|
+
* forwards bytes to any active speaker stream.
|
|
1235
|
+
*/
|
|
1236
|
+
handleAudioOutput(audioOutput) {
|
|
1237
|
+
try {
|
|
1238
|
+
const content = audioOutput.content;
|
|
1239
|
+
const audioBytes = Buffer.from(content, "base64");
|
|
1240
|
+
this.log(`[Event] Audio output: ${audioBytes.length} bytes`);
|
|
1241
|
+
this.isReceivingAssistantAudio = true;
|
|
1242
|
+
const audioData = new Int16Array(audioBytes.buffer, audioBytes.byteOffset, audioBytes.byteLength / 2);
|
|
1243
|
+
this.emit("speaking", {
|
|
1244
|
+
audio: content,
|
|
1245
|
+
audioData,
|
|
1246
|
+
response_id: this.currentResponseId
|
|
1247
|
+
});
|
|
1248
|
+
if (this.currentResponseId) {
|
|
1249
|
+
const stream = this.speakerStreams.get(this.currentResponseId);
|
|
1250
|
+
if (stream) stream.write(audioBytes);
|
|
1251
|
+
}
|
|
1252
|
+
} catch (error) {
|
|
1253
|
+
this.log("[Event] Error decoding audio:", error);
|
|
1254
|
+
this.emit("error", {
|
|
1255
|
+
message: "Failed to decode audio",
|
|
1256
|
+
code: "AUDIO_DECODE_ERROR",
|
|
1257
|
+
details: error
|
|
1258
|
+
});
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
/**
|
|
1262
|
+
* Handle a toolUse event. Emits 'toolCall' and dispatches to the
|
|
1263
|
+
* configured tool's execute() function via handleToolCall().
|
|
1264
|
+
*/
|
|
1265
|
+
handleToolUse(toolUse) {
|
|
1266
|
+
const toolUseId = toolUse.toolUseId || "";
|
|
1267
|
+
const toolName = toolUse.toolName || "";
|
|
1268
|
+
const toolInput = toolUse.input || {};
|
|
1269
|
+
this.emit("toolCall", {
|
|
1270
|
+
name: toolName,
|
|
1271
|
+
args: toolInput,
|
|
1272
|
+
id: toolUseId
|
|
1273
|
+
});
|
|
1274
|
+
if (this.tools && toolName in this.tools) this.handleToolCall(toolName, toolInput, toolUseId);
|
|
1275
|
+
}
|
|
1276
|
+
/**
|
|
1277
|
+
* Handle a contentEnd event. Forwards it to clients, then routes by
|
|
1278
|
+
* stopReason / type:
|
|
1279
|
+
* - INTERRUPTED: emit 'interrupt' and tear down the active speaker stream
|
|
1280
|
+
* - TOOL: end the active speaker stream
|
|
1281
|
+
* - AUDIO with END_TURN: signal turnComplete (assistant audio finished)
|
|
1282
|
+
* - AUDIO with PARTIAL_TURN while receiving assistant audio: schedule
|
|
1283
|
+
* fallback turnComplete in case completionEnd never arrives
|
|
1284
|
+
* - AUDIO otherwise: user input ended, reset turn flags
|
|
1285
|
+
*/
|
|
1286
|
+
handleContentEnd(contentEnd) {
|
|
1287
|
+
this.log(`[Event] contentEnd received: type=${contentEnd.type}, stopReason=${contentEnd.stopReason}`);
|
|
1288
|
+
this.emit("contentEnd", contentEnd);
|
|
1289
|
+
if (contentEnd.stopReason === "INTERRUPTED") {
|
|
1290
|
+
this.log("[Event] Content interrupted by user (barge-in)");
|
|
1291
|
+
this.emit("interrupt", {
|
|
1292
|
+
type: "user",
|
|
1293
|
+
timestamp: Date.now()
|
|
1294
|
+
});
|
|
1295
|
+
if (this.currentResponseId) {
|
|
1296
|
+
const stream = this.speakerStreams.get(this.currentResponseId);
|
|
1297
|
+
if (stream) stream.destroy();
|
|
1298
|
+
this.speakerStreams.delete(this.currentResponseId);
|
|
1299
|
+
}
|
|
1300
|
+
this.currentResponseId = void 0;
|
|
1301
|
+
this.log("[Event] After interruption, keeping audioContentStarted=true for continued streaming");
|
|
1302
|
+
} else if (contentEnd.type === "TOOL" && this.currentResponseId) {
|
|
1303
|
+
const stream = this.speakerStreams.get(this.currentResponseId);
|
|
1304
|
+
if (stream) stream.end();
|
|
1305
|
+
} else if (contentEnd.type === "AUDIO") if (contentEnd.stopReason === "END_TURN") {
|
|
1306
|
+
this.log(`[Event] contentEnd (AUDIO) with stopReason END_TURN - signaling turn complete`);
|
|
1307
|
+
if (this.currentResponseId) {
|
|
1308
|
+
const stream = this.speakerStreams.get(this.currentResponseId);
|
|
1309
|
+
if (stream) stream.end();
|
|
1310
|
+
this.speakerStreams.delete(this.currentResponseId);
|
|
1311
|
+
this.currentResponseId = void 0;
|
|
1312
|
+
}
|
|
1313
|
+
if (!this.turnCompleted) {
|
|
1314
|
+
this.turnCompleted = true;
|
|
1315
|
+
this.emit("turnComplete", { timestamp: Date.now() });
|
|
1316
|
+
this.hasSentContentEnd = false;
|
|
1317
|
+
this.log(`[Event] Turn complete (from contentEnd AUDIO with END_TURN), ready for next turn. audioContentStarted: ${this.audioContentStarted}, audioContentName: ${this.audioContentName}`);
|
|
1318
|
+
} else this.log(`[Event] contentEnd (AUDIO) with END_TURN received but turn already completed - skipping duplicate turnComplete emission`);
|
|
1319
|
+
if (!this.turnCompleteTimeout) this.turnCompleteTimeout = setTimeout(() => {
|
|
1320
|
+
this.log(`[Event] Timeout: completionEnd not received, but turn already completed from contentEnd`);
|
|
1321
|
+
this.turnCompleteTimeout = void 0;
|
|
1322
|
+
}, 1e3);
|
|
1323
|
+
} else if (this.isReceivingAssistantAudio && contentEnd.stopReason === "PARTIAL_TURN") {
|
|
1324
|
+
this.isReceivingAssistantAudio = false;
|
|
1325
|
+
if (!this.turnCompleteTimeout && !this.turnCompleted) {
|
|
1326
|
+
this.log(`[Event] contentEnd (AUDIO) with PARTIAL_TURN for assistant output - waiting for completionEnd, setting fallback timeout`);
|
|
1327
|
+
this.turnCompleteTimeout = setTimeout(() => {
|
|
1328
|
+
if (!this.turnCompleted) {
|
|
1329
|
+
this.log(`[Event] Fallback: completionEnd not received after contentEnd (AUDIO) with PARTIAL_TURN, signaling turn complete`);
|
|
1330
|
+
this.turnCompleted = true;
|
|
1331
|
+
this.emit("turnComplete", { timestamp: Date.now() });
|
|
1332
|
+
if (this.currentResponseId) {
|
|
1333
|
+
const stream = this.speakerStreams.get(this.currentResponseId);
|
|
1334
|
+
if (stream) stream.end();
|
|
1335
|
+
this.speakerStreams.delete(this.currentResponseId);
|
|
1336
|
+
this.currentResponseId = void 0;
|
|
1337
|
+
}
|
|
1338
|
+
this.hasSentContentEnd = false;
|
|
1339
|
+
this.turnCompleteTimeout = void 0;
|
|
1340
|
+
}
|
|
1341
|
+
}, 2e3);
|
|
1342
|
+
}
|
|
1343
|
+
} else {
|
|
1344
|
+
this.hasSentContentEnd = false;
|
|
1345
|
+
this.turnCompleted = false;
|
|
1346
|
+
this.log(`[Event] contentEnd (AUDIO) - user input ended, stopReason: ${contentEnd.stopReason}. Keeping audioContentStarted=true for next turn. Reset hasSentContentEnd=false, turnCompleted=false.`);
|
|
1347
|
+
}
|
|
1348
|
+
else if (contentEnd.type === "TEXT") {
|
|
1349
|
+
this.currentTextGenerationStage = void 0;
|
|
1350
|
+
this.log(`[Event] contentEnd (TEXT) received, stopReason: ${contentEnd.stopReason}. Turn completion handled by completionEnd/contentEnd(AUDIO).`);
|
|
1351
|
+
if (contentEnd.stopReason === "END_TURN") this.hasSentContentEnd = false;
|
|
1352
|
+
}
|
|
1353
|
+
}
|
|
1354
|
+
/**
|
|
1355
|
+
* Handle a completionEnd event. AWS uses this as the definitive signal
|
|
1356
|
+
* that a turn (and all audio output) has finished. Tears down the active
|
|
1357
|
+
* speaker stream, clears any fallback timer, emits 'turnComplete' once,
|
|
1358
|
+
* and forwards token usage if reported.
|
|
1359
|
+
*/
|
|
1360
|
+
handleCompletionEnd(completionEnd) {
|
|
1361
|
+
this.log(`[Event] completionEnd received, stopReason: ${completionEnd.stopReason}`);
|
|
1362
|
+
if (this.turnCompleteTimeout) {
|
|
1363
|
+
clearTimeout(this.turnCompleteTimeout);
|
|
1364
|
+
this.turnCompleteTimeout = void 0;
|
|
1365
|
+
}
|
|
1366
|
+
if (this.currentResponseId) {
|
|
1367
|
+
const stream = this.speakerStreams.get(this.currentResponseId);
|
|
1368
|
+
if (stream) stream.end();
|
|
1369
|
+
this.speakerStreams.delete(this.currentResponseId);
|
|
1370
|
+
this.currentResponseId = void 0;
|
|
1371
|
+
}
|
|
1372
|
+
this.isReceivingAssistantAudio = false;
|
|
1373
|
+
if (!this.turnCompleted) {
|
|
1374
|
+
this.log(`[Event] completionEnd - signaling turn complete (stopReason: ${completionEnd.stopReason || "undefined"})`);
|
|
1375
|
+
this.turnCompleted = true;
|
|
1376
|
+
this.emit("turnComplete", { timestamp: Date.now() });
|
|
1377
|
+
this.hasSentContentEnd = false;
|
|
1378
|
+
} else this.log(`[Event] completionEnd received but turn already completed - skipping duplicate turnComplete emission`);
|
|
1379
|
+
if (completionEnd.usage) this.emit("usage", {
|
|
1380
|
+
inputTokens: completionEnd.usage.inputTokens || 0,
|
|
1381
|
+
outputTokens: completionEnd.usage.outputTokens || 0,
|
|
1382
|
+
totalTokens: (completionEnd.usage.inputTokens || 0) + (completionEnd.usage.outputTokens || 0)
|
|
1383
|
+
});
|
|
1384
|
+
}
|
|
1385
|
+
/**
|
|
1386
|
+
* Handle tool execution
|
|
1387
|
+
*/
|
|
1388
|
+
async handleToolCall(toolName, args, toolUseId) {
|
|
1389
|
+
const tool = this.tools?.[toolName];
|
|
1390
|
+
if (!tool || !tool.execute) {
|
|
1391
|
+
this.emit("error", {
|
|
1392
|
+
message: `Tool ${toolName} not found or has no execute function`,
|
|
1393
|
+
code: "TOOL_NOT_FOUND"
|
|
1394
|
+
});
|
|
1395
|
+
return;
|
|
1396
|
+
}
|
|
1397
|
+
try {
|
|
1398
|
+
const result = await tool.execute({
|
|
1399
|
+
context: args,
|
|
1400
|
+
requestContext: this.requestContext
|
|
1401
|
+
}, {
|
|
1402
|
+
toolCallId: toolUseId,
|
|
1403
|
+
messages: []
|
|
1404
|
+
});
|
|
1405
|
+
await this.sendClientEvent({ toolResult: {
|
|
1406
|
+
toolUseId,
|
|
1407
|
+
content: [{ json: typeof result === "object" ? result : { result } }]
|
|
1408
|
+
} });
|
|
1409
|
+
} catch (error) {
|
|
1410
|
+
this.emit("error", {
|
|
1411
|
+
message: `Error executing tool ${toolName}: ${error instanceof Error ? error.message : "Unknown error"}`,
|
|
1412
|
+
code: "TOOL_EXECUTION_ERROR",
|
|
1413
|
+
details: error
|
|
1414
|
+
});
|
|
1415
|
+
await this.sendClientEvent({ toolResult: {
|
|
1416
|
+
toolUseId,
|
|
1417
|
+
content: [{ text: `Error: ${error instanceof Error ? error.message : "Unknown error"}` }]
|
|
1418
|
+
} });
|
|
1419
|
+
}
|
|
1420
|
+
}
|
|
1421
|
+
/**
|
|
1422
|
+
* Send a client event to AWS Bedrock
|
|
1423
|
+
* Events are sent through the input stream that was passed to the bidirectional stream command
|
|
1424
|
+
*/
|
|
1425
|
+
async sendClientEvent(event) {
|
|
1426
|
+
if (this.state !== "connected") throw new NovaSonicError("not_connected", "Not connected to AWS Bedrock. Call connect() first.");
|
|
1427
|
+
try {
|
|
1428
|
+
const eventQueue = this._eventQueue;
|
|
1429
|
+
const signalQueue = this._signalQueue;
|
|
1430
|
+
if (!eventQueue || !signalQueue) throw new NovaSonicError("not_connected", "Event queue not initialized. Connection may not be fully established.");
|
|
1431
|
+
this.log(`[sendClientEvent] Adding event to queue (queue size: ${eventQueue.length})`);
|
|
1432
|
+
eventQueue.push({ event });
|
|
1433
|
+
this.log(`[sendClientEvent] Event added, queue size now: ${eventQueue.length}, signaling...`);
|
|
1434
|
+
signalQueue();
|
|
1435
|
+
this.log(`[sendClientEvent] Signal sent`);
|
|
1436
|
+
if (this.debug) this.log("Sent client event, keys:", Object.keys(event).join(", "));
|
|
1437
|
+
} catch (error) {
|
|
1438
|
+
throw new NovaSonicError("websocket_error", `Failed to send client event: ${error instanceof Error ? error.message : "Unknown error"}`, error);
|
|
1439
|
+
}
|
|
1440
|
+
}
|
|
1441
|
+
/**
|
|
1442
|
+
* Disconnects from the AWS Bedrock session and cleans up resources.
|
|
1443
|
+
*
|
|
1444
|
+
* Pushes a `sessionEnd` event to the queue before signalling close,
|
|
1445
|
+
* then schedules client destruction on the next tick so the async
|
|
1446
|
+
* iterator has a chance to yield the event to the SDK.
|
|
1447
|
+
*/
|
|
1448
|
+
close() {
|
|
1449
|
+
if (this.state === "disconnected") return;
|
|
1450
|
+
this.state = "disconnected";
|
|
1451
|
+
this.processingStream = false;
|
|
1452
|
+
if (this.turnCompleteTimeout) {
|
|
1453
|
+
clearTimeout(this.turnCompleteTimeout);
|
|
1454
|
+
this.turnCompleteTimeout = void 0;
|
|
1455
|
+
}
|
|
1456
|
+
const eventQueue = this._eventQueue;
|
|
1457
|
+
const signalQueue = this._signalQueue;
|
|
1458
|
+
if (eventQueue && signalQueue) {
|
|
1459
|
+
eventQueue.push({ event: { sessionEnd: {} } });
|
|
1460
|
+
signalQueue();
|
|
1461
|
+
}
|
|
1462
|
+
const closeSignal = this._closeSignal;
|
|
1463
|
+
if (closeSignal) closeSignal();
|
|
1464
|
+
if (this.inputStream) {
|
|
1465
|
+
this.inputStream.end();
|
|
1466
|
+
this.inputStream = void 0;
|
|
1467
|
+
}
|
|
1468
|
+
for (const stream of this.speakerStreams.values()) stream.end();
|
|
1469
|
+
this.speakerStreams.clear();
|
|
1470
|
+
const client = this.client;
|
|
1471
|
+
this.client = void 0;
|
|
1472
|
+
this.stream = void 0;
|
|
1473
|
+
if (client) setImmediate(() => {
|
|
1474
|
+
if (typeof client.destroy === "function") client.destroy();
|
|
1475
|
+
});
|
|
1476
|
+
this.log("Disconnected from AWS Bedrock Nova 2 Sonic");
|
|
1477
|
+
}
|
|
1478
|
+
/**
|
|
1479
|
+
* Equips the voice instance with a set of instructions.
|
|
1480
|
+
*/
|
|
1481
|
+
addInstructions(instructions) {
|
|
1482
|
+
this.instructions = instructions;
|
|
1483
|
+
}
|
|
1484
|
+
/**
|
|
1485
|
+
* Equips the voice instance with a set of tools.
|
|
1486
|
+
*/
|
|
1487
|
+
addTools(tools) {
|
|
1488
|
+
this.tools = tools || {};
|
|
1489
|
+
}
|
|
1490
|
+
/**
|
|
1491
|
+
* Convert text to speech
|
|
1492
|
+
*/
|
|
1493
|
+
async speak(input, _options) {
|
|
1494
|
+
if (this.state !== "connected") throw new NovaSonicError("not_connected", "Not connected. Call connect() first.");
|
|
1495
|
+
let text = "";
|
|
1496
|
+
if (typeof input !== "string") {
|
|
1497
|
+
const chunks = [];
|
|
1498
|
+
for await (const chunk of input) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)));
|
|
1499
|
+
text = Buffer.concat(chunks).toString("utf-8");
|
|
1500
|
+
} else text = input;
|
|
1501
|
+
if (text.trim().length === 0) throw new NovaSonicError("validation_error", "Input text is empty");
|
|
1502
|
+
this.currentResponseId = `response-${Date.now()}`;
|
|
1503
|
+
const speakerStream = new PassThrough();
|
|
1504
|
+
speakerStream.id = this.currentResponseId;
|
|
1505
|
+
this.speakerStreams.set(this.currentResponseId, speakerStream);
|
|
1506
|
+
this.emit("speaker", speakerStream);
|
|
1507
|
+
const promptName = this._promptName;
|
|
1508
|
+
if (!promptName) throw new NovaSonicError("not_connected", "Prompt name not initialized. Connection may not be fully established.");
|
|
1509
|
+
if (!this.promptStarted) throw new NovaSonicError("invalid_state", "Prompt not started. This should not happen - prompt should be started during connection.");
|
|
1510
|
+
const contentName = randomUUID();
|
|
1511
|
+
await this.sendClientEvent({ contentStart: {
|
|
1512
|
+
promptName,
|
|
1513
|
+
contentName,
|
|
1514
|
+
type: "TEXT",
|
|
1515
|
+
interactive: true,
|
|
1516
|
+
role: "USER",
|
|
1517
|
+
textInputConfiguration: { mediaType: "text/plain" }
|
|
1518
|
+
} });
|
|
1519
|
+
await this.sendClientEvent({ textInput: {
|
|
1520
|
+
promptName,
|
|
1521
|
+
contentName,
|
|
1522
|
+
content: text
|
|
1523
|
+
} });
|
|
1524
|
+
await this.sendClientEvent({ contentEnd: {
|
|
1525
|
+
promptName,
|
|
1526
|
+
contentName
|
|
1527
|
+
} });
|
|
1528
|
+
}
|
|
1529
|
+
/**
|
|
1530
|
+
* Convert speech to text (transcription)
|
|
1531
|
+
* For Nova Sonic, this is the same as send() - both stream audio input
|
|
1532
|
+
*/
|
|
1533
|
+
async listen(audioStream, _options) {
|
|
1534
|
+
if (audioStream && typeof audioStream === "object" && "read" in audioStream) await this.send(audioStream);
|
|
1535
|
+
else throw new NovaSonicError("invalid_audio_format", "Unsupported audio stream format for listen()");
|
|
1536
|
+
}
|
|
1537
|
+
/**
|
|
1538
|
+
* Streams audio data in real-time to the AWS Bedrock service.
|
|
1539
|
+
* Following AWS Nova 2 Sonic event sequence:
|
|
1540
|
+
* 1. contentStart (AUDIO, USER) - if not already sent
|
|
1541
|
+
* 2. audioInput events (one per chunk)
|
|
1542
|
+
* 3. contentEnd - when audio stream ends (handled separately via endAudioInput)
|
|
1543
|
+
*/
|
|
1544
|
+
async send(audioData) {
|
|
1545
|
+
this.log(`[send] Current state: ${this.state}`);
|
|
1546
|
+
if (this.state !== "connected") {
|
|
1547
|
+
this.log(`[send] ERROR: State is '${this.state}', expected 'connected'`);
|
|
1548
|
+
throw new NovaSonicError("not_connected", `Not connected. Current state: ${this.state}. Call connect() first.`);
|
|
1549
|
+
}
|
|
1550
|
+
this.log(`[send] State check passed, proceeding with send`);
|
|
1551
|
+
if (!(audioData instanceof Int16Array) && !(audioData && typeof audioData === "object" && "read" in audioData)) throw new NovaSonicError("invalid_audio_format", "Unsupported audio data format");
|
|
1552
|
+
if (this.turnCompleted || this.hasSentContentEnd) {
|
|
1553
|
+
this.log(`[send] Starting new turn - resetting flags. turnCompleted=${this.turnCompleted}, hasSentContentEnd=${this.hasSentContentEnd}.`);
|
|
1554
|
+
const needNewContent = this.hasSentContentEnd;
|
|
1555
|
+
this.turnCompleted = false;
|
|
1556
|
+
this.hasSentContentEnd = false;
|
|
1557
|
+
this.streamRestartAttempted = false;
|
|
1558
|
+
if (needNewContent) {
|
|
1559
|
+
this.audioContentStarted = false;
|
|
1560
|
+
this.log(`[send] contentEnd was previously sent - will create new audio content container`);
|
|
1561
|
+
}
|
|
1562
|
+
this.log(`[send] State reset: turnCompleted=false, hasSentContentEnd=false, audioContentStarted=${this.audioContentStarted}`);
|
|
1563
|
+
}
|
|
1564
|
+
if (!this.promptStarted) this.promptStarted = true;
|
|
1565
|
+
const promptName = this._promptName;
|
|
1566
|
+
if (!promptName) throw new NovaSonicError("not_connected", "Prompt name not initialized. Connection may not be fully established.");
|
|
1567
|
+
if (!this.audioContentStarted) {
|
|
1568
|
+
const audioContentId = randomUUID();
|
|
1569
|
+
this.audioContentName = audioContentId;
|
|
1570
|
+
this.log(`[send] First audio send - sending AUDIO contentStart with contentName: ${audioContentId}`);
|
|
1571
|
+
await this.sendClientEvent({ contentStart: {
|
|
1572
|
+
promptName,
|
|
1573
|
+
contentName: audioContentId,
|
|
1574
|
+
type: "AUDIO",
|
|
1575
|
+
interactive: true,
|
|
1576
|
+
role: "USER",
|
|
1577
|
+
audioInputConfiguration: {
|
|
1578
|
+
mediaType: "audio/lpcm",
|
|
1579
|
+
sampleRateHertz: 16e3,
|
|
1580
|
+
sampleSizeBits: 16,
|
|
1581
|
+
channelCount: 1,
|
|
1582
|
+
encoding: "base64",
|
|
1583
|
+
audioType: "SPEECH"
|
|
1584
|
+
}
|
|
1585
|
+
} });
|
|
1586
|
+
this.audioContentStarted = true;
|
|
1587
|
+
this.log(`[send] AUDIO contentStart sent, ready to stream audio`);
|
|
1588
|
+
} else this.log(`[send] AUDIO contentStart already sent, sending audioInput chunks directly`);
|
|
1589
|
+
if (!this.audioContentName) throw new NovaSonicError("invalid_state", "Audio content name not initialized. This should not happen.");
|
|
1590
|
+
const contentName = this.audioContentName;
|
|
1591
|
+
if (audioData instanceof Int16Array) {
|
|
1592
|
+
const buffer = Buffer.from(audioData.buffer, audioData.byteOffset, audioData.byteLength);
|
|
1593
|
+
const base64Audio = buffer.toString("base64");
|
|
1594
|
+
this.log(`[send] Sending audioInput chunk, size: ${buffer.length} bytes, contentName: ${contentName}, turnCompleted: ${this.turnCompleted}, hasSentContentEnd: ${this.hasSentContentEnd}, audioContentStarted: ${this.audioContentStarted}, state: ${this.state}`);
|
|
1595
|
+
if (this.state !== "connected") {
|
|
1596
|
+
this.log(`[send] ERROR: State changed to '${this.state}' during send!`);
|
|
1597
|
+
throw new NovaSonicError("not_connected", `Connection lost during send. State: ${this.state}`);
|
|
1598
|
+
}
|
|
1599
|
+
await this.sendClientEvent({ audioInput: {
|
|
1600
|
+
promptName,
|
|
1601
|
+
contentName,
|
|
1602
|
+
content: base64Audio
|
|
1603
|
+
} });
|
|
1604
|
+
this.log(`[send] audioInput chunk sent successfully`);
|
|
1605
|
+
} else if (audioData && typeof audioData === "object" && "read" in audioData) {
|
|
1606
|
+
const stream = audioData;
|
|
1607
|
+
for await (const chunk of stream) {
|
|
1608
|
+
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
1609
|
+
const base64Audio = buffer.toString("base64");
|
|
1610
|
+
this.log(`[send] Sending audioInput chunk from stream, size: ${buffer.length} bytes, contentName: ${contentName}, turnCompleted: ${this.turnCompleted}, hasSentContentEnd: ${this.hasSentContentEnd}`);
|
|
1611
|
+
await this.sendClientEvent({ audioInput: {
|
|
1612
|
+
promptName,
|
|
1613
|
+
contentName,
|
|
1614
|
+
content: base64Audio
|
|
1615
|
+
} });
|
|
1616
|
+
}
|
|
1617
|
+
} else throw new NovaSonicError("invalid_audio_format", "Unsupported audio data format");
|
|
1618
|
+
}
|
|
1619
|
+
/**
|
|
1620
|
+
* End audio input stream (sends contentEnd for audio)
|
|
1621
|
+
* Call this when done sending audio chunks
|
|
1622
|
+
*/
|
|
1623
|
+
async endAudioInput() {
|
|
1624
|
+
if (this.hasSentContentEnd) {
|
|
1625
|
+
this.log("[endAudioInput] contentEnd already sent for this turn, skipping");
|
|
1626
|
+
return;
|
|
1627
|
+
}
|
|
1628
|
+
if (this.turnCompleted) {
|
|
1629
|
+
this.log("[endAudioInput] Turn already completed by AWS, skipping contentEnd. Resetting turnCompleted flag for next turn.");
|
|
1630
|
+
this.turnCompleted = false;
|
|
1631
|
+
this.hasSentContentEnd = false;
|
|
1632
|
+
return;
|
|
1633
|
+
}
|
|
1634
|
+
if (this.audioContentStarted && this.audioContentName && this._promptName) {
|
|
1635
|
+
const promptName = this._promptName;
|
|
1636
|
+
this.log("[endAudioInput] Sending contentEnd for audio input");
|
|
1637
|
+
await this.sendClientEvent({ contentEnd: {
|
|
1638
|
+
promptName,
|
|
1639
|
+
contentName: this.audioContentName
|
|
1640
|
+
} });
|
|
1641
|
+
this.hasSentContentEnd = true;
|
|
1642
|
+
} else this.log("[endAudioInput] Cannot send contentEnd: audioContentStarted=" + this.audioContentStarted + ", audioContentName=" + this.audioContentName);
|
|
1643
|
+
}
|
|
1644
|
+
/**
|
|
1645
|
+
* Register an event listener
|
|
1646
|
+
*/
|
|
1647
|
+
on(event, callback) {
|
|
1648
|
+
if (!this.events[event]) this.events[event] = [];
|
|
1649
|
+
this.events[event].push(callback);
|
|
1650
|
+
}
|
|
1651
|
+
/**
|
|
1652
|
+
* Remove an event listener
|
|
1653
|
+
*/
|
|
1654
|
+
off(event, callback) {
|
|
1655
|
+
if (!this.events[event]) return;
|
|
1656
|
+
const index = this.events[event].indexOf(callback);
|
|
1657
|
+
if (index !== -1) this.events[event].splice(index, 1);
|
|
1658
|
+
}
|
|
1659
|
+
/**
|
|
1660
|
+
* Emit an event with arguments
|
|
1661
|
+
*/
|
|
1662
|
+
emit(event, data) {
|
|
1663
|
+
if (!this.events[event]) {
|
|
1664
|
+
this.log(`[NovaSonic] emit('${event}'): No listeners registered for this event`);
|
|
1665
|
+
return;
|
|
1666
|
+
}
|
|
1667
|
+
const listenerCount = this.events[event].length;
|
|
1668
|
+
this.log(`[NovaSonic] emit('${event}'): Calling ${listenerCount} listener(s)`);
|
|
1669
|
+
for (const callback of this.events[event]) try {
|
|
1670
|
+
callback(data);
|
|
1671
|
+
this.log(`[NovaSonic] emit('${event}'): Successfully called one listener`);
|
|
1672
|
+
} catch (error) {
|
|
1673
|
+
this.log(`Error in event handler for ${event}:`, error);
|
|
1674
|
+
}
|
|
1675
|
+
this.log(`[NovaSonic] emit('${event}'): Finished calling all ${listenerCount} listener(s)`);
|
|
1676
|
+
}
|
|
1677
|
+
/**
|
|
1678
|
+
* Get listener status
|
|
1679
|
+
*/
|
|
1680
|
+
async getListener() {
|
|
1681
|
+
return { enabled: this.state === "connected" };
|
|
1682
|
+
}
|
|
1683
|
+
/**
|
|
1684
|
+
* Log helper
|
|
1685
|
+
*/
|
|
1686
|
+
log(...args) {
|
|
1687
|
+
if (this.debug) console.info("[NovaSonicVoice]", ...args);
|
|
1688
|
+
}
|
|
1869
1689
|
};
|
|
1870
|
-
|
|
1690
|
+
//#endregion
|
|
1871
1691
|
export { NovaSonicError, NovaSonicErrorCode, NovaSonicVoice };
|
|
1872
|
-
|
|
1692
|
+
|
|
1873
1693
|
//# sourceMappingURL=index.js.map
|