@micdrop/server 2.2.6 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +9 -0
- package/dist/index.d.mts +102 -8
- package/dist/index.d.ts +102 -8
- package/dist/index.js +216 -27
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +211 -27
- package/dist/index.mjs.map +1 -1
- package/package.json +8 -9
package/LICENSE
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2022 Rolebase
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
6
|
+
|
|
7
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
8
|
+
|
|
9
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
package/dist/index.d.mts
CHANGED
|
@@ -170,6 +170,44 @@ declare class MockAgent extends Agent {
|
|
|
170
170
|
cancel(): void;
|
|
171
171
|
}
|
|
172
172
|
|
|
173
|
+
/**
|
|
174
|
+
* Streaming linear-interpolation resampler for PCM16 mono audio.
|
|
175
|
+
*
|
|
176
|
+
* Works in both directions (up or downsampling). It is stateful: it handles
|
|
177
|
+
* arbitrary byte boundaries (a network chunk can split a 16-bit sample) and
|
|
178
|
+
* keeps the fractional sample position continuous across chunks, so feeding a
|
|
179
|
+
* stream chunk by chunk yields the same result as resampling it in one go.
|
|
180
|
+
*
|
|
181
|
+
* Providers use it to bridge their own rate with the 16kHz PCM16 the Micdrop
|
|
182
|
+
* client records and plays: OpenaiSTT (16kHz -> 24kHz, the GA Realtime API
|
|
183
|
+
* requires >= 24kHz), OpenaiTTS and KokoroTTS (24kHz output -> 16kHz).
|
|
184
|
+
*/
|
|
185
|
+
declare class Pcm16Resampler {
|
|
186
|
+
private readonly step;
|
|
187
|
+
private leftover;
|
|
188
|
+
private pos;
|
|
189
|
+
constructor(inRate: number, outRate: number);
|
|
190
|
+
reset(): void;
|
|
191
|
+
process(chunk: Buffer): Buffer;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Conversions between the PCM16 buffers exchanged with the Micdrop client and
|
|
196
|
+
* the float samples that local speech models read and write.
|
|
197
|
+
*
|
|
198
|
+
* Both formats are mono. PCM16 is signed 16-bit little-endian, floats are in
|
|
199
|
+
* the [-1, 1] range. Only the scale changes, the sample rate is left alone.
|
|
200
|
+
*/
|
|
201
|
+
/** Turns float samples into a PCM16 buffer, clamping anything out of range. */
|
|
202
|
+
declare function float32ToPcm16(samples: Float32Array): Buffer;
|
|
203
|
+
/**
|
|
204
|
+
* Turns a PCM16 buffer into float samples.
|
|
205
|
+
*
|
|
206
|
+
* A trailing odd byte is dropped: it is half of a sample whose other half has
|
|
207
|
+
* not arrived, and a caller feeding whole utterances never produces one.
|
|
208
|
+
*/
|
|
209
|
+
declare function pcm16ToFloat32(buffer: Buffer): Float32Array;
|
|
210
|
+
|
|
173
211
|
declare enum MicdropErrorCode {
|
|
174
212
|
BadRequest = 4400,
|
|
175
213
|
Unauthorized = 4401,
|
|
@@ -224,13 +262,6 @@ declare abstract class TTS extends EventEmitter<TTSEvents> {
|
|
|
224
262
|
destroy(): void;
|
|
225
263
|
}
|
|
226
264
|
|
|
227
|
-
declare class MockTTS extends TTS {
|
|
228
|
-
private audioFilePaths;
|
|
229
|
-
constructor(audioFilePaths: string[]);
|
|
230
|
-
speak(textStream: Readable): PassThrough;
|
|
231
|
-
cancel(): void;
|
|
232
|
-
}
|
|
233
|
-
|
|
234
265
|
interface FallbackTTSOptions {
|
|
235
266
|
factories: Array<() => TTS>;
|
|
236
267
|
}
|
|
@@ -247,6 +278,69 @@ declare class FallbackTTS extends TTS {
|
|
|
247
278
|
private onFailed;
|
|
248
279
|
}
|
|
249
280
|
|
|
281
|
+
declare class MockTTS extends TTS {
|
|
282
|
+
private audioFilePaths;
|
|
283
|
+
constructor(audioFilePaths: string[]);
|
|
284
|
+
speak(textStream: Readable): PassThrough;
|
|
285
|
+
cancel(): void;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/**
|
|
289
|
+
* Cuts a stream of text into sentences as it arrives.
|
|
290
|
+
*
|
|
291
|
+
* Providers that synthesize a whole input at once need complete sentences, and
|
|
292
|
+
* an agent writes its answer token by token. Feeding every fragment as it comes
|
|
293
|
+
* would either cut words in half or wait for the end of the answer, so the text
|
|
294
|
+
* is buffered until a sentence closes and released the moment it does.
|
|
295
|
+
*
|
|
296
|
+
* The splitter is stateful: `push` returns the sentences that are complete,
|
|
297
|
+
* `flush` returns whatever is left when the stream ends.
|
|
298
|
+
*/
|
|
299
|
+
declare class SentenceSplitter {
|
|
300
|
+
private buffer;
|
|
301
|
+
/** Adds text and returns the sentences it completes. */
|
|
302
|
+
push(text: string): string[];
|
|
303
|
+
/** Returns the sentences left in the buffer and empties it. */
|
|
304
|
+
flush(): string[];
|
|
305
|
+
/** Drops the buffered text, used when an utterance is cancelled. */
|
|
306
|
+
reset(): void;
|
|
307
|
+
private extract;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/**
|
|
311
|
+
* Base class for text to speech engines that read a whole input at once.
|
|
312
|
+
*
|
|
313
|
+
* A local model, and a remote endpoint without a streaming interface, cannot
|
|
314
|
+
* be fed the agent's answer token by token. This class buffers the answer into
|
|
315
|
+
* sentences, hands them over one at a time, and emits the audio in the order
|
|
316
|
+
* they were written. Subclasses only have to turn one sentence into PCM16 at
|
|
317
|
+
* the rate the Micdrop client expects.
|
|
318
|
+
*
|
|
319
|
+
* Sentences are synthesized one after the other rather than at once: a local
|
|
320
|
+
* model is single threaded, so racing two sentences through it slows both down
|
|
321
|
+
* without bringing the first word any closer.
|
|
322
|
+
*/
|
|
323
|
+
declare abstract class SentenceTTS extends TTS {
|
|
324
|
+
private splitter;
|
|
325
|
+
private queue;
|
|
326
|
+
private draining;
|
|
327
|
+
private controller?;
|
|
328
|
+
private generation;
|
|
329
|
+
private counter;
|
|
330
|
+
/**
|
|
331
|
+
* Turns one sentence into PCM16 audio at the client's sample rate.
|
|
332
|
+
*
|
|
333
|
+
* The signal is aborted when the utterance is cancelled, which is the moment
|
|
334
|
+
* to stop a subprocess or an inference that is no longer needed. Returning
|
|
335
|
+
* nothing emits nothing, which is how a cancelled synthesis reports back.
|
|
336
|
+
*/
|
|
337
|
+
protected abstract synthesize(text: string, signal: AbortSignal): Promise<Buffer | undefined>;
|
|
338
|
+
speak(textStream: Readable): void;
|
|
339
|
+
cancel(): void;
|
|
340
|
+
private enqueue;
|
|
341
|
+
private drain;
|
|
342
|
+
}
|
|
343
|
+
|
|
250
344
|
interface MicdropServerEvents {
|
|
251
345
|
End: [MicdropCallSummary];
|
|
252
346
|
UserAudio: [Buffer];
|
|
@@ -322,4 +416,4 @@ declare class MicdropRecorder extends EventEmitter<MicdropRecorderEvents> {
|
|
|
322
416
|
|
|
323
417
|
declare function waitForParams<CallParams>(socket: WebSocket$1, validate: (params: any) => CallParams): Promise<CallParams>;
|
|
324
418
|
|
|
325
|
-
export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, STT, type STTEvents, TTS, type TTSEvents, type Tool, handleError, waitForParams };
|
|
419
|
+
export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, STT, type STTEvents, SentenceSplitter, SentenceTTS, TTS, type TTSEvents, type Tool, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };
|
package/dist/index.d.ts
CHANGED
|
@@ -170,6 +170,44 @@ declare class MockAgent extends Agent {
|
|
|
170
170
|
cancel(): void;
|
|
171
171
|
}
|
|
172
172
|
|
|
173
|
+
/**
|
|
174
|
+
* Streaming linear-interpolation resampler for PCM16 mono audio.
|
|
175
|
+
*
|
|
176
|
+
* Works in both directions (up or downsampling). It is stateful: it handles
|
|
177
|
+
* arbitrary byte boundaries (a network chunk can split a 16-bit sample) and
|
|
178
|
+
* keeps the fractional sample position continuous across chunks, so feeding a
|
|
179
|
+
* stream chunk by chunk yields the same result as resampling it in one go.
|
|
180
|
+
*
|
|
181
|
+
* Providers use it to bridge their own rate with the 16kHz PCM16 the Micdrop
|
|
182
|
+
* client records and plays: OpenaiSTT (16kHz -> 24kHz, the GA Realtime API
|
|
183
|
+
* requires >= 24kHz), OpenaiTTS and KokoroTTS (24kHz output -> 16kHz).
|
|
184
|
+
*/
|
|
185
|
+
declare class Pcm16Resampler {
|
|
186
|
+
private readonly step;
|
|
187
|
+
private leftover;
|
|
188
|
+
private pos;
|
|
189
|
+
constructor(inRate: number, outRate: number);
|
|
190
|
+
reset(): void;
|
|
191
|
+
process(chunk: Buffer): Buffer;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Conversions between the PCM16 buffers exchanged with the Micdrop client and
|
|
196
|
+
* the float samples that local speech models read and write.
|
|
197
|
+
*
|
|
198
|
+
* Both formats are mono. PCM16 is signed 16-bit little-endian, floats are in
|
|
199
|
+
* the [-1, 1] range. Only the scale changes, the sample rate is left alone.
|
|
200
|
+
*/
|
|
201
|
+
/** Turns float samples into a PCM16 buffer, clamping anything out of range. */
|
|
202
|
+
declare function float32ToPcm16(samples: Float32Array): Buffer;
|
|
203
|
+
/**
|
|
204
|
+
* Turns a PCM16 buffer into float samples.
|
|
205
|
+
*
|
|
206
|
+
* A trailing odd byte is dropped: it is half of a sample whose other half has
|
|
207
|
+
* not arrived, and a caller feeding whole utterances never produces one.
|
|
208
|
+
*/
|
|
209
|
+
declare function pcm16ToFloat32(buffer: Buffer): Float32Array;
|
|
210
|
+
|
|
173
211
|
declare enum MicdropErrorCode {
|
|
174
212
|
BadRequest = 4400,
|
|
175
213
|
Unauthorized = 4401,
|
|
@@ -224,13 +262,6 @@ declare abstract class TTS extends EventEmitter<TTSEvents> {
|
|
|
224
262
|
destroy(): void;
|
|
225
263
|
}
|
|
226
264
|
|
|
227
|
-
declare class MockTTS extends TTS {
|
|
228
|
-
private audioFilePaths;
|
|
229
|
-
constructor(audioFilePaths: string[]);
|
|
230
|
-
speak(textStream: Readable): PassThrough;
|
|
231
|
-
cancel(): void;
|
|
232
|
-
}
|
|
233
|
-
|
|
234
265
|
interface FallbackTTSOptions {
|
|
235
266
|
factories: Array<() => TTS>;
|
|
236
267
|
}
|
|
@@ -247,6 +278,69 @@ declare class FallbackTTS extends TTS {
|
|
|
247
278
|
private onFailed;
|
|
248
279
|
}
|
|
249
280
|
|
|
281
|
+
declare class MockTTS extends TTS {
|
|
282
|
+
private audioFilePaths;
|
|
283
|
+
constructor(audioFilePaths: string[]);
|
|
284
|
+
speak(textStream: Readable): PassThrough;
|
|
285
|
+
cancel(): void;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/**
|
|
289
|
+
* Cuts a stream of text into sentences as it arrives.
|
|
290
|
+
*
|
|
291
|
+
* Providers that synthesize a whole input at once need complete sentences, and
|
|
292
|
+
* an agent writes its answer token by token. Feeding every fragment as it comes
|
|
293
|
+
* would either cut words in half or wait for the end of the answer, so the text
|
|
294
|
+
* is buffered until a sentence closes and released the moment it does.
|
|
295
|
+
*
|
|
296
|
+
* The splitter is stateful: `push` returns the sentences that are complete,
|
|
297
|
+
* `flush` returns whatever is left when the stream ends.
|
|
298
|
+
*/
|
|
299
|
+
declare class SentenceSplitter {
|
|
300
|
+
private buffer;
|
|
301
|
+
/** Adds text and returns the sentences it completes. */
|
|
302
|
+
push(text: string): string[];
|
|
303
|
+
/** Returns the sentences left in the buffer and empties it. */
|
|
304
|
+
flush(): string[];
|
|
305
|
+
/** Drops the buffered text, used when an utterance is cancelled. */
|
|
306
|
+
reset(): void;
|
|
307
|
+
private extract;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/**
|
|
311
|
+
* Base class for text to speech engines that read a whole input at once.
|
|
312
|
+
*
|
|
313
|
+
* A local model, and a remote endpoint without a streaming interface, cannot
|
|
314
|
+
* be fed the agent's answer token by token. This class buffers the answer into
|
|
315
|
+
* sentences, hands them over one at a time, and emits the audio in the order
|
|
316
|
+
* they were written. Subclasses only have to turn one sentence into PCM16 at
|
|
317
|
+
* the rate the Micdrop client expects.
|
|
318
|
+
*
|
|
319
|
+
* Sentences are synthesized one after the other rather than at once: a local
|
|
320
|
+
* model is single threaded, so racing two sentences through it slows both down
|
|
321
|
+
* without bringing the first word any closer.
|
|
322
|
+
*/
|
|
323
|
+
declare abstract class SentenceTTS extends TTS {
|
|
324
|
+
private splitter;
|
|
325
|
+
private queue;
|
|
326
|
+
private draining;
|
|
327
|
+
private controller?;
|
|
328
|
+
private generation;
|
|
329
|
+
private counter;
|
|
330
|
+
/**
|
|
331
|
+
* Turns one sentence into PCM16 audio at the client's sample rate.
|
|
332
|
+
*
|
|
333
|
+
* The signal is aborted when the utterance is cancelled, which is the moment
|
|
334
|
+
* to stop a subprocess or an inference that is no longer needed. Returning
|
|
335
|
+
* nothing emits nothing, which is how a cancelled synthesis reports back.
|
|
336
|
+
*/
|
|
337
|
+
protected abstract synthesize(text: string, signal: AbortSignal): Promise<Buffer | undefined>;
|
|
338
|
+
speak(textStream: Readable): void;
|
|
339
|
+
cancel(): void;
|
|
340
|
+
private enqueue;
|
|
341
|
+
private drain;
|
|
342
|
+
}
|
|
343
|
+
|
|
250
344
|
interface MicdropServerEvents {
|
|
251
345
|
End: [MicdropCallSummary];
|
|
252
346
|
UserAudio: [Buffer];
|
|
@@ -322,4 +416,4 @@ declare class MicdropRecorder extends EventEmitter<MicdropRecorderEvents> {
|
|
|
322
416
|
|
|
323
417
|
declare function waitForParams<CallParams>(socket: WebSocket$1, validate: (params: any) => CallParams): Promise<CallParams>;
|
|
324
418
|
|
|
325
|
-
export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, STT, type STTEvents, TTS, type TTSEvents, type Tool, handleError, waitForParams };
|
|
419
|
+
export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, STT, type STTEvents, SentenceSplitter, SentenceTTS, TTS, type TTSEvents, type Tool, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };
|
package/dist/index.js
CHANGED
|
@@ -50,9 +50,14 @@ __export(index_exports, {
|
|
|
50
50
|
MockAgent: () => MockAgent,
|
|
51
51
|
MockSTT: () => MockSTT,
|
|
52
52
|
MockTTS: () => MockTTS,
|
|
53
|
+
Pcm16Resampler: () => Pcm16Resampler,
|
|
53
54
|
STT: () => STT,
|
|
55
|
+
SentenceSplitter: () => SentenceSplitter,
|
|
56
|
+
SentenceTTS: () => SentenceTTS,
|
|
54
57
|
TTS: () => TTS,
|
|
58
|
+
float32ToPcm16: () => float32ToPcm16,
|
|
55
59
|
handleError: () => handleError,
|
|
60
|
+
pcm16ToFloat32: () => pcm16ToFloat32,
|
|
56
61
|
waitForParams: () => waitForParams
|
|
57
62
|
});
|
|
58
63
|
module.exports = __toCommonJS(index_exports);
|
|
@@ -399,6 +404,69 @@ var MockAgent = class extends Agent {
|
|
|
399
404
|
}
|
|
400
405
|
};
|
|
401
406
|
|
|
407
|
+
// src/audio/Pcm16Resampler.ts
|
|
408
|
+
var Pcm16Resampler = class {
|
|
409
|
+
// Fractional position into the first sample of the buffer
|
|
410
|
+
constructor(inRate, outRate) {
|
|
411
|
+
this.leftover = Buffer.alloc(0);
|
|
412
|
+
this.pos = 0;
|
|
413
|
+
this.step = inRate / outRate;
|
|
414
|
+
}
|
|
415
|
+
// Reset to the initial state, to resample a new independent stream
|
|
416
|
+
// (e.g. resending buffered audio after a reconnection).
|
|
417
|
+
reset() {
|
|
418
|
+
this.leftover = Buffer.alloc(0);
|
|
419
|
+
this.pos = 0;
|
|
420
|
+
}
|
|
421
|
+
process(chunk) {
|
|
422
|
+
const buf = this.leftover.length ? Buffer.concat([this.leftover, chunk]) : chunk;
|
|
423
|
+
const samples = Math.floor(buf.length / 2);
|
|
424
|
+
if (samples < 2) {
|
|
425
|
+
this.leftover = buf;
|
|
426
|
+
return Buffer.alloc(0);
|
|
427
|
+
}
|
|
428
|
+
const out = [];
|
|
429
|
+
let p = this.pos;
|
|
430
|
+
while (Math.floor(p) + 1 < samples) {
|
|
431
|
+
const i = Math.floor(p);
|
|
432
|
+
const frac = p - i;
|
|
433
|
+
const s0 = buf.readInt16LE(i * 2);
|
|
434
|
+
const s1 = buf.readInt16LE((i + 1) * 2);
|
|
435
|
+
out.push(Math.round(s0 + (s1 - s0) * frac));
|
|
436
|
+
p += this.step;
|
|
437
|
+
}
|
|
438
|
+
const consumed = Math.floor(p);
|
|
439
|
+
this.pos = p - consumed;
|
|
440
|
+
this.leftover = buf.subarray(consumed * 2);
|
|
441
|
+
const result = Buffer.alloc(out.length * 2);
|
|
442
|
+
for (let k = 0; k < out.length; k++) {
|
|
443
|
+
result.writeInt16LE(out[k], k * 2);
|
|
444
|
+
}
|
|
445
|
+
return result;
|
|
446
|
+
}
|
|
447
|
+
};
|
|
448
|
+
|
|
449
|
+
// src/audio/pcm16.ts
|
|
450
|
+
var PCM16_MAX = 32767;
|
|
451
|
+
var PCM16_MIN = -32768;
|
|
452
|
+
function float32ToPcm16(samples) {
|
|
453
|
+
const buffer = Buffer.alloc(samples.length * 2);
|
|
454
|
+
for (let i = 0; i < samples.length; i++) {
|
|
455
|
+
const scaled = Math.round(samples[i] * PCM16_MAX);
|
|
456
|
+
const clamped = Math.max(PCM16_MIN, Math.min(PCM16_MAX, scaled));
|
|
457
|
+
buffer.writeInt16LE(clamped, i * 2);
|
|
458
|
+
}
|
|
459
|
+
return buffer;
|
|
460
|
+
}
|
|
461
|
+
function pcm16ToFloat32(buffer) {
|
|
462
|
+
const length = Math.floor(buffer.length / 2);
|
|
463
|
+
const samples = new Float32Array(length);
|
|
464
|
+
for (let i = 0; i < length; i++) {
|
|
465
|
+
samples[i] = buffer.readInt16LE(i * 2) / PCM16_MAX;
|
|
466
|
+
}
|
|
467
|
+
return samples;
|
|
468
|
+
}
|
|
469
|
+
|
|
402
470
|
// src/errors.ts
|
|
403
471
|
var MicdropErrorCode = /* @__PURE__ */ ((MicdropErrorCode2) => {
|
|
404
472
|
MicdropErrorCode2[MicdropErrorCode2["BadRequest"] = 4400] = "BadRequest";
|
|
@@ -888,8 +956,7 @@ var FallbackSTT = class extends STT {
|
|
|
888
956
|
}
|
|
889
957
|
};
|
|
890
958
|
|
|
891
|
-
// src/tts/
|
|
892
|
-
var fs = __toESM(require("fs"));
|
|
959
|
+
// src/tts/FallbackTTS.ts
|
|
893
960
|
var import_stream4 = require("stream");
|
|
894
961
|
|
|
895
962
|
// src/tts/TTS.ts
|
|
@@ -904,31 +971,7 @@ var TTS = class extends import_eventemitter35.EventEmitter {
|
|
|
904
971
|
}
|
|
905
972
|
};
|
|
906
973
|
|
|
907
|
-
// src/tts/MockTTS.ts
|
|
908
|
-
var MockTTS = class extends TTS {
|
|
909
|
-
constructor(audioFilePaths) {
|
|
910
|
-
super();
|
|
911
|
-
this.audioFilePaths = audioFilePaths;
|
|
912
|
-
}
|
|
913
|
-
speak(textStream) {
|
|
914
|
-
const audioStream = new import_stream4.PassThrough();
|
|
915
|
-
textStream.once("data", async () => {
|
|
916
|
-
for (const filePath of this.audioFilePaths) {
|
|
917
|
-
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
918
|
-
const audioBuffer = fs.readFileSync(filePath);
|
|
919
|
-
this.log(`Loaded chunk (${audioBuffer.length} bytes)`);
|
|
920
|
-
audioStream.write(audioBuffer);
|
|
921
|
-
}
|
|
922
|
-
audioStream.end();
|
|
923
|
-
});
|
|
924
|
-
return audioStream;
|
|
925
|
-
}
|
|
926
|
-
cancel() {
|
|
927
|
-
}
|
|
928
|
-
};
|
|
929
|
-
|
|
930
974
|
// src/tts/FallbackTTS.ts
|
|
931
|
-
var import_stream5 = require("stream");
|
|
932
975
|
var FallbackTTS = class extends TTS {
|
|
933
976
|
// Start at -1 because we need to increment it before using it
|
|
934
977
|
constructor(options) {
|
|
@@ -944,7 +987,7 @@ var FallbackTTS = class extends TTS {
|
|
|
944
987
|
this.startNextTTS();
|
|
945
988
|
if (chunks.length > 0) {
|
|
946
989
|
this.log("Sending text chunks again");
|
|
947
|
-
const stream = new
|
|
990
|
+
const stream = new import_stream4.PassThrough();
|
|
948
991
|
this.tts?.speak(stream);
|
|
949
992
|
chunks.forEach((chunk) => stream.write(chunk));
|
|
950
993
|
stream.end();
|
|
@@ -984,6 +1027,147 @@ var FallbackTTS = class extends TTS {
|
|
|
984
1027
|
}
|
|
985
1028
|
};
|
|
986
1029
|
|
|
1030
|
+
// src/tts/MockTTS.ts
|
|
1031
|
+
var fs = __toESM(require("fs"));
|
|
1032
|
+
var import_stream5 = require("stream");
|
|
1033
|
+
var MockTTS = class extends TTS {
|
|
1034
|
+
constructor(audioFilePaths) {
|
|
1035
|
+
super();
|
|
1036
|
+
this.audioFilePaths = audioFilePaths;
|
|
1037
|
+
}
|
|
1038
|
+
speak(textStream) {
|
|
1039
|
+
const audioStream = new import_stream5.PassThrough();
|
|
1040
|
+
textStream.once("data", async () => {
|
|
1041
|
+
for (const filePath of this.audioFilePaths) {
|
|
1042
|
+
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
1043
|
+
const audioBuffer = fs.readFileSync(filePath);
|
|
1044
|
+
this.log(`Loaded chunk (${audioBuffer.length} bytes)`);
|
|
1045
|
+
audioStream.write(audioBuffer);
|
|
1046
|
+
}
|
|
1047
|
+
audioStream.end();
|
|
1048
|
+
});
|
|
1049
|
+
return audioStream;
|
|
1050
|
+
}
|
|
1051
|
+
cancel() {
|
|
1052
|
+
}
|
|
1053
|
+
};
|
|
1054
|
+
|
|
1055
|
+
// src/tts/SentenceSplitter.ts
|
|
1056
|
+
var SentenceSplitter = class {
|
|
1057
|
+
constructor() {
|
|
1058
|
+
this.buffer = "";
|
|
1059
|
+
}
|
|
1060
|
+
/** Adds text and returns the sentences it completes. */
|
|
1061
|
+
push(text) {
|
|
1062
|
+
this.buffer += text;
|
|
1063
|
+
return this.extract(false);
|
|
1064
|
+
}
|
|
1065
|
+
/** Returns the sentences left in the buffer and empties it. */
|
|
1066
|
+
flush() {
|
|
1067
|
+
const sentences = this.extract(true);
|
|
1068
|
+
const rest = this.buffer.trim();
|
|
1069
|
+
this.buffer = "";
|
|
1070
|
+
if (rest) sentences.push(rest);
|
|
1071
|
+
return sentences;
|
|
1072
|
+
}
|
|
1073
|
+
/** Drops the buffered text, used when an utterance is cancelled. */
|
|
1074
|
+
reset() {
|
|
1075
|
+
this.buffer = "";
|
|
1076
|
+
}
|
|
1077
|
+
extract(end) {
|
|
1078
|
+
const sentences = [];
|
|
1079
|
+
const regex = /[\s\S]*?[.!?…\n]+(?=\s|$)/g;
|
|
1080
|
+
let match;
|
|
1081
|
+
let lastIndex = 0;
|
|
1082
|
+
while ((match = regex.exec(this.buffer)) !== null) {
|
|
1083
|
+
if (!end && regex.lastIndex === this.buffer.length) break;
|
|
1084
|
+
const sentence = match[0].trim();
|
|
1085
|
+
if (sentence) sentences.push(sentence);
|
|
1086
|
+
lastIndex = regex.lastIndex;
|
|
1087
|
+
}
|
|
1088
|
+
this.buffer = this.buffer.slice(lastIndex);
|
|
1089
|
+
return sentences;
|
|
1090
|
+
}
|
|
1091
|
+
};
|
|
1092
|
+
|
|
1093
|
+
// src/tts/SentenceTTS.ts
|
|
1094
|
+
var SentenceTTS = class extends TTS {
|
|
1095
|
+
constructor() {
|
|
1096
|
+
super(...arguments);
|
|
1097
|
+
this.splitter = new SentenceSplitter();
|
|
1098
|
+
this.queue = [];
|
|
1099
|
+
this.draining = false;
|
|
1100
|
+
// Bumped by every speak() and every cancel(), so a call claimed late can tell
|
|
1101
|
+
// whether it is still the one that should be heard.
|
|
1102
|
+
this.generation = 0;
|
|
1103
|
+
this.counter = 0;
|
|
1104
|
+
}
|
|
1105
|
+
speak(textStream) {
|
|
1106
|
+
const generation = ++this.generation;
|
|
1107
|
+
let counter = 0;
|
|
1108
|
+
const claimCall = () => {
|
|
1109
|
+
if (counter) return true;
|
|
1110
|
+
if (this.generation !== generation) return false;
|
|
1111
|
+
this.counter++;
|
|
1112
|
+
counter = this.counter;
|
|
1113
|
+
this.splitter.reset();
|
|
1114
|
+
return true;
|
|
1115
|
+
};
|
|
1116
|
+
textStream.on("data", (chunk) => {
|
|
1117
|
+
if (!claimCall()) return;
|
|
1118
|
+
if (counter !== this.counter) return;
|
|
1119
|
+
this.enqueue(counter, this.splitter.push(chunk.toString("utf-8")));
|
|
1120
|
+
});
|
|
1121
|
+
textStream.on("error", (error) => {
|
|
1122
|
+
this.log("Error in text stream", error);
|
|
1123
|
+
});
|
|
1124
|
+
textStream.on("end", () => {
|
|
1125
|
+
if (!counter || counter !== this.counter) return;
|
|
1126
|
+
this.enqueue(counter, this.splitter.flush());
|
|
1127
|
+
});
|
|
1128
|
+
}
|
|
1129
|
+
cancel() {
|
|
1130
|
+
this.log("Cancel");
|
|
1131
|
+
this.generation++;
|
|
1132
|
+
this.counter++;
|
|
1133
|
+
this.splitter.reset();
|
|
1134
|
+
this.queue = [];
|
|
1135
|
+
this.controller?.abort();
|
|
1136
|
+
this.controller = void 0;
|
|
1137
|
+
}
|
|
1138
|
+
enqueue(counter, sentences) {
|
|
1139
|
+
if (sentences.length === 0) return;
|
|
1140
|
+
if (counter !== this.counter) return;
|
|
1141
|
+
this.queue.push(...sentences);
|
|
1142
|
+
this.drain();
|
|
1143
|
+
}
|
|
1144
|
+
async drain() {
|
|
1145
|
+
if (this.draining) return;
|
|
1146
|
+
this.draining = true;
|
|
1147
|
+
while (this.queue.length > 0) {
|
|
1148
|
+
const counter = this.counter;
|
|
1149
|
+
const text = this.queue.shift();
|
|
1150
|
+
const controller = new AbortController();
|
|
1151
|
+
this.controller = controller;
|
|
1152
|
+
try {
|
|
1153
|
+
this.log(`Synthesizing: "${text}"`);
|
|
1154
|
+
const audio = await this.synthesize(text, controller.signal);
|
|
1155
|
+
if (counter !== this.counter) continue;
|
|
1156
|
+
if (audio?.length) this.emit("Audio", audio);
|
|
1157
|
+
} catch (error) {
|
|
1158
|
+
if (counter !== this.counter) continue;
|
|
1159
|
+
this.log("Error synthesizing speech", error);
|
|
1160
|
+
this.emit("Failed", [text, ...this.queue]);
|
|
1161
|
+
this.queue = [];
|
|
1162
|
+
} finally {
|
|
1163
|
+
if (this.controller === controller) this.controller = void 0;
|
|
1164
|
+
}
|
|
1165
|
+
}
|
|
1166
|
+
this.draining = false;
|
|
1167
|
+
if (this.queue.length > 0) this.drain();
|
|
1168
|
+
}
|
|
1169
|
+
};
|
|
1170
|
+
|
|
987
1171
|
// src/waitForParams.ts
|
|
988
1172
|
async function waitForParams(socket, validate) {
|
|
989
1173
|
return new Promise((resolve, reject) => {
|
|
@@ -1025,9 +1209,14 @@ async function waitForParams(socket, validate) {
|
|
|
1025
1209
|
MockAgent,
|
|
1026
1210
|
MockSTT,
|
|
1027
1211
|
MockTTS,
|
|
1212
|
+
Pcm16Resampler,
|
|
1028
1213
|
STT,
|
|
1214
|
+
SentenceSplitter,
|
|
1215
|
+
SentenceTTS,
|
|
1029
1216
|
TTS,
|
|
1217
|
+
float32ToPcm16,
|
|
1030
1218
|
handleError,
|
|
1219
|
+
pcm16ToFloat32,
|
|
1031
1220
|
waitForParams
|
|
1032
1221
|
});
|
|
1033
1222
|
//# sourceMappingURL=index.js.map
|