@micdrop/server 2.2.6 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/index.d.mts +113 -8
- package/dist/index.d.ts +113 -8
- package/dist/index.js +233 -27
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +228 -27
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# 🖐️🎤 Micdrop: Real-Time Voice Conversations with AI
|
|
2
2
|
|
|
3
|
-
[Micdrop website](https://micdrop.dev) | [Documentation](https://micdrop.dev/docs/server) | [Demo](../../examples/
|
|
3
|
+
[Micdrop website](https://micdrop.dev) | [Documentation](https://micdrop.dev/docs/server) | [Basic example](../../examples/basic) | [Demo](../../examples/advanced)
|
|
4
4
|
|
|
5
5
|
Micdrop is a set of open source Typescript packages to build real-time voice conversations with AI agents. It handles all the complexities on the browser and server side (microphone, speaker, VAD, network communication, etc) and provides ready-to-use implementations for various AI providers.
|
|
6
6
|
|
package/dist/index.d.mts
CHANGED
|
@@ -170,6 +170,44 @@ declare class MockAgent extends Agent {
|
|
|
170
170
|
cancel(): void;
|
|
171
171
|
}
|
|
172
172
|
|
|
173
|
+
/**
|
|
174
|
+
* Streaming linear-interpolation resampler for PCM16 mono audio.
|
|
175
|
+
*
|
|
176
|
+
* Works in both directions (up or downsampling). It is stateful: it handles
|
|
177
|
+
* arbitrary byte boundaries (a network chunk can split a 16-bit sample) and
|
|
178
|
+
* keeps the fractional sample position continuous across chunks, so feeding a
|
|
179
|
+
* stream chunk by chunk yields the same result as resampling it in one go.
|
|
180
|
+
*
|
|
181
|
+
* Providers use it to bridge their own rate with the 16kHz PCM16 the Micdrop
|
|
182
|
+
* client records and plays: OpenaiSTT (16kHz -> 24kHz, the GA Realtime API
|
|
183
|
+
* requires >= 24kHz), OpenaiTTS and KokoroTTS (24kHz output -> 16kHz).
|
|
184
|
+
*/
|
|
185
|
+
declare class Pcm16Resampler {
|
|
186
|
+
private readonly step;
|
|
187
|
+
private leftover;
|
|
188
|
+
private pos;
|
|
189
|
+
constructor(inRate: number, outRate: number);
|
|
190
|
+
reset(): void;
|
|
191
|
+
process(chunk: Buffer): Buffer;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Conversions between the PCM16 buffers exchanged with the Micdrop client and
|
|
196
|
+
* the float samples that local speech models read and write.
|
|
197
|
+
*
|
|
198
|
+
* Both formats are mono. PCM16 is signed 16-bit little-endian, floats are in
|
|
199
|
+
* the [-1, 1] range. Only the scale changes, the sample rate is left alone.
|
|
200
|
+
*/
|
|
201
|
+
/** Turns float samples into a PCM16 buffer, clamping anything out of range. */
|
|
202
|
+
declare function float32ToPcm16(samples: Float32Array): Buffer;
|
|
203
|
+
/**
|
|
204
|
+
* Turns a PCM16 buffer into float samples.
|
|
205
|
+
*
|
|
206
|
+
* A trailing odd byte is dropped: it is half of a sample whose other half has
|
|
207
|
+
* not arrived, and a caller feeding whole utterances never produces one.
|
|
208
|
+
*/
|
|
209
|
+
declare function pcm16ToFloat32(buffer: Buffer): Float32Array;
|
|
210
|
+
|
|
173
211
|
declare enum MicdropErrorCode {
|
|
174
212
|
BadRequest = 4400,
|
|
175
213
|
Unauthorized = 4401,
|
|
@@ -224,13 +262,6 @@ declare abstract class TTS extends EventEmitter<TTSEvents> {
|
|
|
224
262
|
destroy(): void;
|
|
225
263
|
}
|
|
226
264
|
|
|
227
|
-
declare class MockTTS extends TTS {
|
|
228
|
-
private audioFilePaths;
|
|
229
|
-
constructor(audioFilePaths: string[]);
|
|
230
|
-
speak(textStream: Readable): PassThrough;
|
|
231
|
-
cancel(): void;
|
|
232
|
-
}
|
|
233
|
-
|
|
234
265
|
interface FallbackTTSOptions {
|
|
235
266
|
factories: Array<() => TTS>;
|
|
236
267
|
}
|
|
@@ -247,6 +278,80 @@ declare class FallbackTTS extends TTS {
|
|
|
247
278
|
private onFailed;
|
|
248
279
|
}
|
|
249
280
|
|
|
281
|
+
declare class MockTTS extends TTS {
|
|
282
|
+
private audioFilePaths;
|
|
283
|
+
constructor(audioFilePaths: string[]);
|
|
284
|
+
speak(textStream: Readable): PassThrough;
|
|
285
|
+
cancel(): void;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/**
|
|
289
|
+
* Cuts a stream of text into sentences as it arrives.
|
|
290
|
+
*
|
|
291
|
+
* Providers that synthesize a whole input at once need complete sentences, and
|
|
292
|
+
* an agent writes its answer token by token. Feeding every fragment as it comes
|
|
293
|
+
* would either cut words in half or wait for the end of the answer, so the text
|
|
294
|
+
* is buffered until a sentence closes and released the moment it does.
|
|
295
|
+
*
|
|
296
|
+
* The splitter is stateful: `push` returns the sentences that are complete,
|
|
297
|
+
* `flush` returns whatever is left when the stream ends.
|
|
298
|
+
*/
|
|
299
|
+
declare class SentenceSplitter {
|
|
300
|
+
private buffer;
|
|
301
|
+
/** Adds text and returns the sentences it completes. */
|
|
302
|
+
push(text: string): string[];
|
|
303
|
+
/** Returns the sentences left in the buffer and empties it. */
|
|
304
|
+
flush(): string[];
|
|
305
|
+
/** Drops the buffered text, used when an utterance is cancelled. */
|
|
306
|
+
reset(): void;
|
|
307
|
+
private extract;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/**
|
|
311
|
+
* Base class for text to speech engines that read a whole input at once.
|
|
312
|
+
*
|
|
313
|
+
* A local model, and a remote endpoint without a streaming interface, cannot
|
|
314
|
+
* be fed the agent's answer token by token. This class buffers the answer into
|
|
315
|
+
* sentences, hands them over one at a time, and emits the audio in the order
|
|
316
|
+
* they were written. Subclasses only have to turn one sentence into PCM16 at
|
|
317
|
+
* the rate the Micdrop client expects.
|
|
318
|
+
*
|
|
319
|
+
* Sentences are synthesized one after the other rather than at once: a local
|
|
320
|
+
* model is single threaded, so racing two sentences through it slows both down
|
|
321
|
+
* without bringing the first word any closer.
|
|
322
|
+
*/
|
|
323
|
+
declare abstract class SentenceTTS extends TTS {
|
|
324
|
+
private splitter;
|
|
325
|
+
private queue;
|
|
326
|
+
private draining;
|
|
327
|
+
private controller?;
|
|
328
|
+
private generation;
|
|
329
|
+
private counter;
|
|
330
|
+
private synthesizing;
|
|
331
|
+
/**
|
|
332
|
+
* Turns one sentence into PCM16 audio at the client's sample rate.
|
|
333
|
+
*
|
|
334
|
+
* The signal is aborted when the utterance is cancelled, which is the moment
|
|
335
|
+
* to stop a subprocess or an inference that is no longer needed. Returning
|
|
336
|
+
* nothing emits nothing, which is how a cancelled synthesis reports back.
|
|
337
|
+
*/
|
|
338
|
+
protected abstract synthesize(text: string, signal: AbortSignal): Promise<Buffer | undefined>;
|
|
339
|
+
/**
|
|
340
|
+
* Emits a piece of the sentence being synthesized.
|
|
341
|
+
*
|
|
342
|
+
* A model that generates progressively can hand its chunks over as they
|
|
343
|
+
* come rather than waiting for the sentence to be finished, which brings
|
|
344
|
+
* the first word forward by the duration of that sentence. The false it
|
|
345
|
+
* returns says the utterance was cancelled or replaced, so the generation
|
|
346
|
+
* it comes from can be stopped there.
|
|
347
|
+
*/
|
|
348
|
+
protected emitAudio(audio: Buffer): boolean;
|
|
349
|
+
speak(textStream: Readable): void;
|
|
350
|
+
cancel(): void;
|
|
351
|
+
private enqueue;
|
|
352
|
+
private drain;
|
|
353
|
+
}
|
|
354
|
+
|
|
250
355
|
interface MicdropServerEvents {
|
|
251
356
|
End: [MicdropCallSummary];
|
|
252
357
|
UserAudio: [Buffer];
|
|
@@ -322,4 +427,4 @@ declare class MicdropRecorder extends EventEmitter<MicdropRecorderEvents> {
|
|
|
322
427
|
|
|
323
428
|
declare function waitForParams<CallParams>(socket: WebSocket$1, validate: (params: any) => CallParams): Promise<CallParams>;
|
|
324
429
|
|
|
325
|
-
export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, STT, type STTEvents, TTS, type TTSEvents, type Tool, handleError, waitForParams };
|
|
430
|
+
export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, STT, type STTEvents, SentenceSplitter, SentenceTTS, TTS, type TTSEvents, type Tool, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };
|
package/dist/index.d.ts
CHANGED
|
@@ -170,6 +170,44 @@ declare class MockAgent extends Agent {
|
|
|
170
170
|
cancel(): void;
|
|
171
171
|
}
|
|
172
172
|
|
|
173
|
+
/**
|
|
174
|
+
* Streaming linear-interpolation resampler for PCM16 mono audio.
|
|
175
|
+
*
|
|
176
|
+
* Works in both directions (up or downsampling). It is stateful: it handles
|
|
177
|
+
* arbitrary byte boundaries (a network chunk can split a 16-bit sample) and
|
|
178
|
+
* keeps the fractional sample position continuous across chunks, so feeding a
|
|
179
|
+
* stream chunk by chunk yields the same result as resampling it in one go.
|
|
180
|
+
*
|
|
181
|
+
* Providers use it to bridge their own rate with the 16kHz PCM16 the Micdrop
|
|
182
|
+
* client records and plays: OpenaiSTT (16kHz -> 24kHz, the GA Realtime API
|
|
183
|
+
* requires >= 24kHz), OpenaiTTS and KokoroTTS (24kHz output -> 16kHz).
|
|
184
|
+
*/
|
|
185
|
+
declare class Pcm16Resampler {
|
|
186
|
+
private readonly step;
|
|
187
|
+
private leftover;
|
|
188
|
+
private pos;
|
|
189
|
+
constructor(inRate: number, outRate: number);
|
|
190
|
+
reset(): void;
|
|
191
|
+
process(chunk: Buffer): Buffer;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Conversions between the PCM16 buffers exchanged with the Micdrop client and
|
|
196
|
+
* the float samples that local speech models read and write.
|
|
197
|
+
*
|
|
198
|
+
* Both formats are mono. PCM16 is signed 16-bit little-endian, floats are in
|
|
199
|
+
* the [-1, 1] range. Only the scale changes, the sample rate is left alone.
|
|
200
|
+
*/
|
|
201
|
+
/** Turns float samples into a PCM16 buffer, clamping anything out of range. */
|
|
202
|
+
declare function float32ToPcm16(samples: Float32Array): Buffer;
|
|
203
|
+
/**
|
|
204
|
+
* Turns a PCM16 buffer into float samples.
|
|
205
|
+
*
|
|
206
|
+
* A trailing odd byte is dropped: it is half of a sample whose other half has
|
|
207
|
+
* not arrived, and a caller feeding whole utterances never produces one.
|
|
208
|
+
*/
|
|
209
|
+
declare function pcm16ToFloat32(buffer: Buffer): Float32Array;
|
|
210
|
+
|
|
173
211
|
declare enum MicdropErrorCode {
|
|
174
212
|
BadRequest = 4400,
|
|
175
213
|
Unauthorized = 4401,
|
|
@@ -224,13 +262,6 @@ declare abstract class TTS extends EventEmitter<TTSEvents> {
|
|
|
224
262
|
destroy(): void;
|
|
225
263
|
}
|
|
226
264
|
|
|
227
|
-
declare class MockTTS extends TTS {
|
|
228
|
-
private audioFilePaths;
|
|
229
|
-
constructor(audioFilePaths: string[]);
|
|
230
|
-
speak(textStream: Readable): PassThrough;
|
|
231
|
-
cancel(): void;
|
|
232
|
-
}
|
|
233
|
-
|
|
234
265
|
interface FallbackTTSOptions {
|
|
235
266
|
factories: Array<() => TTS>;
|
|
236
267
|
}
|
|
@@ -247,6 +278,80 @@ declare class FallbackTTS extends TTS {
|
|
|
247
278
|
private onFailed;
|
|
248
279
|
}
|
|
249
280
|
|
|
281
|
+
declare class MockTTS extends TTS {
|
|
282
|
+
private audioFilePaths;
|
|
283
|
+
constructor(audioFilePaths: string[]);
|
|
284
|
+
speak(textStream: Readable): PassThrough;
|
|
285
|
+
cancel(): void;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/**
|
|
289
|
+
* Cuts a stream of text into sentences as it arrives.
|
|
290
|
+
*
|
|
291
|
+
* Providers that synthesize a whole input at once need complete sentences, and
|
|
292
|
+
* an agent writes its answer token by token. Feeding every fragment as it comes
|
|
293
|
+
* would either cut words in half or wait for the end of the answer, so the text
|
|
294
|
+
* is buffered until a sentence closes and released the moment it does.
|
|
295
|
+
*
|
|
296
|
+
* The splitter is stateful: `push` returns the sentences that are complete,
|
|
297
|
+
* `flush` returns whatever is left when the stream ends.
|
|
298
|
+
*/
|
|
299
|
+
declare class SentenceSplitter {
|
|
300
|
+
private buffer;
|
|
301
|
+
/** Adds text and returns the sentences it completes. */
|
|
302
|
+
push(text: string): string[];
|
|
303
|
+
/** Returns the sentences left in the buffer and empties it. */
|
|
304
|
+
flush(): string[];
|
|
305
|
+
/** Drops the buffered text, used when an utterance is cancelled. */
|
|
306
|
+
reset(): void;
|
|
307
|
+
private extract;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/**
|
|
311
|
+
* Base class for text to speech engines that read a whole input at once.
|
|
312
|
+
*
|
|
313
|
+
* A local model, and a remote endpoint without a streaming interface, cannot
|
|
314
|
+
* be fed the agent's answer token by token. This class buffers the answer into
|
|
315
|
+
* sentences, hands them over one at a time, and emits the audio in the order
|
|
316
|
+
* they were written. Subclasses only have to turn one sentence into PCM16 at
|
|
317
|
+
* the rate the Micdrop client expects.
|
|
318
|
+
*
|
|
319
|
+
* Sentences are synthesized one after the other rather than at once: a local
|
|
320
|
+
* model is single threaded, so racing two sentences through it slows both down
|
|
321
|
+
* without bringing the first word any closer.
|
|
322
|
+
*/
|
|
323
|
+
declare abstract class SentenceTTS extends TTS {
|
|
324
|
+
private splitter;
|
|
325
|
+
private queue;
|
|
326
|
+
private draining;
|
|
327
|
+
private controller?;
|
|
328
|
+
private generation;
|
|
329
|
+
private counter;
|
|
330
|
+
private synthesizing;
|
|
331
|
+
/**
|
|
332
|
+
* Turns one sentence into PCM16 audio at the client's sample rate.
|
|
333
|
+
*
|
|
334
|
+
* The signal is aborted when the utterance is cancelled, which is the moment
|
|
335
|
+
* to stop a subprocess or an inference that is no longer needed. Returning
|
|
336
|
+
* nothing emits nothing, which is how a cancelled synthesis reports back.
|
|
337
|
+
*/
|
|
338
|
+
protected abstract synthesize(text: string, signal: AbortSignal): Promise<Buffer | undefined>;
|
|
339
|
+
/**
|
|
340
|
+
* Emits a piece of the sentence being synthesized.
|
|
341
|
+
*
|
|
342
|
+
* A model that generates progressively can hand its chunks over as they
|
|
343
|
+
* come rather than waiting for the sentence to be finished, which brings
|
|
344
|
+
* the first word forward by the duration of that sentence. The false it
|
|
345
|
+
* returns says the utterance was cancelled or replaced, so the generation
|
|
346
|
+
* it comes from can be stopped there.
|
|
347
|
+
*/
|
|
348
|
+
protected emitAudio(audio: Buffer): boolean;
|
|
349
|
+
speak(textStream: Readable): void;
|
|
350
|
+
cancel(): void;
|
|
351
|
+
private enqueue;
|
|
352
|
+
private drain;
|
|
353
|
+
}
|
|
354
|
+
|
|
250
355
|
interface MicdropServerEvents {
|
|
251
356
|
End: [MicdropCallSummary];
|
|
252
357
|
UserAudio: [Buffer];
|
|
@@ -322,4 +427,4 @@ declare class MicdropRecorder extends EventEmitter<MicdropRecorderEvents> {
|
|
|
322
427
|
|
|
323
428
|
declare function waitForParams<CallParams>(socket: WebSocket$1, validate: (params: any) => CallParams): Promise<CallParams>;
|
|
324
429
|
|
|
325
|
-
export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, STT, type STTEvents, TTS, type TTSEvents, type Tool, handleError, waitForParams };
|
|
430
|
+
export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, STT, type STTEvents, SentenceSplitter, SentenceTTS, TTS, type TTSEvents, type Tool, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };
|
package/dist/index.js
CHANGED
|
@@ -50,9 +50,14 @@ __export(index_exports, {
|
|
|
50
50
|
MockAgent: () => MockAgent,
|
|
51
51
|
MockSTT: () => MockSTT,
|
|
52
52
|
MockTTS: () => MockTTS,
|
|
53
|
+
Pcm16Resampler: () => Pcm16Resampler,
|
|
53
54
|
STT: () => STT,
|
|
55
|
+
SentenceSplitter: () => SentenceSplitter,
|
|
56
|
+
SentenceTTS: () => SentenceTTS,
|
|
54
57
|
TTS: () => TTS,
|
|
58
|
+
float32ToPcm16: () => float32ToPcm16,
|
|
55
59
|
handleError: () => handleError,
|
|
60
|
+
pcm16ToFloat32: () => pcm16ToFloat32,
|
|
56
61
|
waitForParams: () => waitForParams
|
|
57
62
|
});
|
|
58
63
|
module.exports = __toCommonJS(index_exports);
|
|
@@ -399,6 +404,69 @@ var MockAgent = class extends Agent {
|
|
|
399
404
|
}
|
|
400
405
|
};
|
|
401
406
|
|
|
407
|
+
// src/audio/Pcm16Resampler.ts
|
|
408
|
+
var Pcm16Resampler = class {
|
|
409
|
+
// Fractional position into the first sample of the buffer
|
|
410
|
+
constructor(inRate, outRate) {
|
|
411
|
+
this.leftover = Buffer.alloc(0);
|
|
412
|
+
this.pos = 0;
|
|
413
|
+
this.step = inRate / outRate;
|
|
414
|
+
}
|
|
415
|
+
// Reset to the initial state, to resample a new independent stream
|
|
416
|
+
// (e.g. resending buffered audio after a reconnection).
|
|
417
|
+
reset() {
|
|
418
|
+
this.leftover = Buffer.alloc(0);
|
|
419
|
+
this.pos = 0;
|
|
420
|
+
}
|
|
421
|
+
process(chunk) {
|
|
422
|
+
const buf = this.leftover.length ? Buffer.concat([this.leftover, chunk]) : chunk;
|
|
423
|
+
const samples = Math.floor(buf.length / 2);
|
|
424
|
+
if (samples < 2) {
|
|
425
|
+
this.leftover = buf;
|
|
426
|
+
return Buffer.alloc(0);
|
|
427
|
+
}
|
|
428
|
+
const out = [];
|
|
429
|
+
let p = this.pos;
|
|
430
|
+
while (Math.floor(p) + 1 < samples) {
|
|
431
|
+
const i = Math.floor(p);
|
|
432
|
+
const frac = p - i;
|
|
433
|
+
const s0 = buf.readInt16LE(i * 2);
|
|
434
|
+
const s1 = buf.readInt16LE((i + 1) * 2);
|
|
435
|
+
out.push(Math.round(s0 + (s1 - s0) * frac));
|
|
436
|
+
p += this.step;
|
|
437
|
+
}
|
|
438
|
+
const consumed = Math.floor(p);
|
|
439
|
+
this.pos = p - consumed;
|
|
440
|
+
this.leftover = buf.subarray(consumed * 2);
|
|
441
|
+
const result = Buffer.alloc(out.length * 2);
|
|
442
|
+
for (let k = 0; k < out.length; k++) {
|
|
443
|
+
result.writeInt16LE(out[k], k * 2);
|
|
444
|
+
}
|
|
445
|
+
return result;
|
|
446
|
+
}
|
|
447
|
+
};
|
|
448
|
+
|
|
449
|
+
// src/audio/pcm16.ts
|
|
450
|
+
var PCM16_MAX = 32767;
|
|
451
|
+
var PCM16_MIN = -32768;
|
|
452
|
+
function float32ToPcm16(samples) {
|
|
453
|
+
const buffer = Buffer.alloc(samples.length * 2);
|
|
454
|
+
for (let i = 0; i < samples.length; i++) {
|
|
455
|
+
const scaled = Math.round(samples[i] * PCM16_MAX);
|
|
456
|
+
const clamped = Math.max(PCM16_MIN, Math.min(PCM16_MAX, scaled));
|
|
457
|
+
buffer.writeInt16LE(clamped, i * 2);
|
|
458
|
+
}
|
|
459
|
+
return buffer;
|
|
460
|
+
}
|
|
461
|
+
function pcm16ToFloat32(buffer) {
|
|
462
|
+
const length = Math.floor(buffer.length / 2);
|
|
463
|
+
const samples = new Float32Array(length);
|
|
464
|
+
for (let i = 0; i < length; i++) {
|
|
465
|
+
samples[i] = buffer.readInt16LE(i * 2) / PCM16_MAX;
|
|
466
|
+
}
|
|
467
|
+
return samples;
|
|
468
|
+
}
|
|
469
|
+
|
|
402
470
|
// src/errors.ts
|
|
403
471
|
var MicdropErrorCode = /* @__PURE__ */ ((MicdropErrorCode2) => {
|
|
404
472
|
MicdropErrorCode2[MicdropErrorCode2["BadRequest"] = 4400] = "BadRequest";
|
|
@@ -888,8 +956,7 @@ var FallbackSTT = class extends STT {
|
|
|
888
956
|
}
|
|
889
957
|
};
|
|
890
958
|
|
|
891
|
-
// src/tts/
|
|
892
|
-
var fs = __toESM(require("fs"));
|
|
959
|
+
// src/tts/FallbackTTS.ts
|
|
893
960
|
var import_stream4 = require("stream");
|
|
894
961
|
|
|
895
962
|
// src/tts/TTS.ts
|
|
@@ -904,31 +971,7 @@ var TTS = class extends import_eventemitter35.EventEmitter {
|
|
|
904
971
|
}
|
|
905
972
|
};
|
|
906
973
|
|
|
907
|
-
// src/tts/MockTTS.ts
|
|
908
|
-
var MockTTS = class extends TTS {
|
|
909
|
-
constructor(audioFilePaths) {
|
|
910
|
-
super();
|
|
911
|
-
this.audioFilePaths = audioFilePaths;
|
|
912
|
-
}
|
|
913
|
-
speak(textStream) {
|
|
914
|
-
const audioStream = new import_stream4.PassThrough();
|
|
915
|
-
textStream.once("data", async () => {
|
|
916
|
-
for (const filePath of this.audioFilePaths) {
|
|
917
|
-
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
918
|
-
const audioBuffer = fs.readFileSync(filePath);
|
|
919
|
-
this.log(`Loaded chunk (${audioBuffer.length} bytes)`);
|
|
920
|
-
audioStream.write(audioBuffer);
|
|
921
|
-
}
|
|
922
|
-
audioStream.end();
|
|
923
|
-
});
|
|
924
|
-
return audioStream;
|
|
925
|
-
}
|
|
926
|
-
cancel() {
|
|
927
|
-
}
|
|
928
|
-
};
|
|
929
|
-
|
|
930
974
|
// src/tts/FallbackTTS.ts
|
|
931
|
-
var import_stream5 = require("stream");
|
|
932
975
|
var FallbackTTS = class extends TTS {
|
|
933
976
|
// Start at -1 because we need to increment it before using it
|
|
934
977
|
constructor(options) {
|
|
@@ -944,7 +987,7 @@ var FallbackTTS = class extends TTS {
|
|
|
944
987
|
this.startNextTTS();
|
|
945
988
|
if (chunks.length > 0) {
|
|
946
989
|
this.log("Sending text chunks again");
|
|
947
|
-
const stream = new
|
|
990
|
+
const stream = new import_stream4.PassThrough();
|
|
948
991
|
this.tts?.speak(stream);
|
|
949
992
|
chunks.forEach((chunk) => stream.write(chunk));
|
|
950
993
|
stream.end();
|
|
@@ -984,6 +1027,164 @@ var FallbackTTS = class extends TTS {
|
|
|
984
1027
|
}
|
|
985
1028
|
};
|
|
986
1029
|
|
|
1030
|
+
// src/tts/MockTTS.ts
|
|
1031
|
+
var fs = __toESM(require("fs"));
|
|
1032
|
+
var import_stream5 = require("stream");
|
|
1033
|
+
var MockTTS = class extends TTS {
|
|
1034
|
+
constructor(audioFilePaths) {
|
|
1035
|
+
super();
|
|
1036
|
+
this.audioFilePaths = audioFilePaths;
|
|
1037
|
+
}
|
|
1038
|
+
speak(textStream) {
|
|
1039
|
+
const audioStream = new import_stream5.PassThrough();
|
|
1040
|
+
textStream.once("data", async () => {
|
|
1041
|
+
for (const filePath of this.audioFilePaths) {
|
|
1042
|
+
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
1043
|
+
const audioBuffer = fs.readFileSync(filePath);
|
|
1044
|
+
this.log(`Loaded chunk (${audioBuffer.length} bytes)`);
|
|
1045
|
+
audioStream.write(audioBuffer);
|
|
1046
|
+
}
|
|
1047
|
+
audioStream.end();
|
|
1048
|
+
});
|
|
1049
|
+
return audioStream;
|
|
1050
|
+
}
|
|
1051
|
+
cancel() {
|
|
1052
|
+
}
|
|
1053
|
+
};
|
|
1054
|
+
|
|
1055
|
+
// src/tts/SentenceSplitter.ts
|
|
1056
|
+
var SentenceSplitter = class {
|
|
1057
|
+
constructor() {
|
|
1058
|
+
this.buffer = "";
|
|
1059
|
+
}
|
|
1060
|
+
/** Adds text and returns the sentences it completes. */
|
|
1061
|
+
push(text) {
|
|
1062
|
+
this.buffer += text;
|
|
1063
|
+
return this.extract(false);
|
|
1064
|
+
}
|
|
1065
|
+
/** Returns the sentences left in the buffer and empties it. */
|
|
1066
|
+
flush() {
|
|
1067
|
+
const sentences = this.extract(true);
|
|
1068
|
+
const rest = this.buffer.trim();
|
|
1069
|
+
this.buffer = "";
|
|
1070
|
+
if (rest) sentences.push(rest);
|
|
1071
|
+
return sentences;
|
|
1072
|
+
}
|
|
1073
|
+
/** Drops the buffered text, used when an utterance is cancelled. */
|
|
1074
|
+
reset() {
|
|
1075
|
+
this.buffer = "";
|
|
1076
|
+
}
|
|
1077
|
+
extract(end) {
|
|
1078
|
+
const sentences = [];
|
|
1079
|
+
const regex = /[\s\S]*?[.!?…\n]+(?=\s|$)/g;
|
|
1080
|
+
let match;
|
|
1081
|
+
let lastIndex = 0;
|
|
1082
|
+
while ((match = regex.exec(this.buffer)) !== null) {
|
|
1083
|
+
if (!end && regex.lastIndex === this.buffer.length) break;
|
|
1084
|
+
const sentence = match[0].trim();
|
|
1085
|
+
if (sentence) sentences.push(sentence);
|
|
1086
|
+
lastIndex = regex.lastIndex;
|
|
1087
|
+
}
|
|
1088
|
+
this.buffer = this.buffer.slice(lastIndex);
|
|
1089
|
+
return sentences;
|
|
1090
|
+
}
|
|
1091
|
+
};
|
|
1092
|
+
|
|
1093
|
+
// src/tts/SentenceTTS.ts
|
|
1094
|
+
var SentenceTTS = class extends TTS {
|
|
1095
|
+
constructor() {
|
|
1096
|
+
super(...arguments);
|
|
1097
|
+
this.splitter = new SentenceSplitter();
|
|
1098
|
+
this.queue = [];
|
|
1099
|
+
this.draining = false;
|
|
1100
|
+
// Bumped by every speak() and every cancel(), so a call claimed late can tell
|
|
1101
|
+
// whether it is still the one that should be heard.
|
|
1102
|
+
this.generation = 0;
|
|
1103
|
+
this.counter = 0;
|
|
1104
|
+
// Identifies the current speak() call
|
|
1105
|
+
this.synthesizing = 0;
|
|
1106
|
+
}
|
|
1107
|
+
/**
|
|
1108
|
+
* Emits a piece of the sentence being synthesized.
|
|
1109
|
+
*
|
|
1110
|
+
* A model that generates progressively can hand its chunks over as they
|
|
1111
|
+
* come rather than waiting for the sentence to be finished, which brings
|
|
1112
|
+
* the first word forward by the duration of that sentence. The false it
|
|
1113
|
+
* returns says the utterance was cancelled or replaced, so the generation
|
|
1114
|
+
* it comes from can be stopped there.
|
|
1115
|
+
*/
|
|
1116
|
+
emitAudio(audio) {
|
|
1117
|
+
if (this.synthesizing !== this.counter) return false;
|
|
1118
|
+
if (audio.length) this.emit("Audio", audio);
|
|
1119
|
+
return true;
|
|
1120
|
+
}
|
|
1121
|
+
speak(textStream) {
|
|
1122
|
+
const generation = ++this.generation;
|
|
1123
|
+
let counter = 0;
|
|
1124
|
+
const claimCall = () => {
|
|
1125
|
+
if (counter) return true;
|
|
1126
|
+
if (this.generation !== generation) return false;
|
|
1127
|
+
this.counter++;
|
|
1128
|
+
counter = this.counter;
|
|
1129
|
+
this.splitter.reset();
|
|
1130
|
+
return true;
|
|
1131
|
+
};
|
|
1132
|
+
textStream.on("data", (chunk) => {
|
|
1133
|
+
if (!claimCall()) return;
|
|
1134
|
+
if (counter !== this.counter) return;
|
|
1135
|
+
this.enqueue(counter, this.splitter.push(chunk.toString("utf-8")));
|
|
1136
|
+
});
|
|
1137
|
+
textStream.on("error", (error) => {
|
|
1138
|
+
this.log("Error in text stream", error);
|
|
1139
|
+
});
|
|
1140
|
+
textStream.on("end", () => {
|
|
1141
|
+
if (!counter || counter !== this.counter) return;
|
|
1142
|
+
this.enqueue(counter, this.splitter.flush());
|
|
1143
|
+
});
|
|
1144
|
+
}
|
|
1145
|
+
cancel() {
|
|
1146
|
+
this.log("Cancel");
|
|
1147
|
+
this.generation++;
|
|
1148
|
+
this.counter++;
|
|
1149
|
+
this.splitter.reset();
|
|
1150
|
+
this.queue = [];
|
|
1151
|
+
this.controller?.abort();
|
|
1152
|
+
this.controller = void 0;
|
|
1153
|
+
}
|
|
1154
|
+
enqueue(counter, sentences) {
|
|
1155
|
+
if (sentences.length === 0) return;
|
|
1156
|
+
if (counter !== this.counter) return;
|
|
1157
|
+
this.queue.push(...sentences);
|
|
1158
|
+
this.drain();
|
|
1159
|
+
}
|
|
1160
|
+
async drain() {
|
|
1161
|
+
if (this.draining) return;
|
|
1162
|
+
this.draining = true;
|
|
1163
|
+
while (this.queue.length > 0) {
|
|
1164
|
+
const counter = this.counter;
|
|
1165
|
+
this.synthesizing = counter;
|
|
1166
|
+
const text = this.queue.shift();
|
|
1167
|
+
const controller = new AbortController();
|
|
1168
|
+
this.controller = controller;
|
|
1169
|
+
try {
|
|
1170
|
+
this.log(`Synthesizing: "${text}"`);
|
|
1171
|
+
const audio = await this.synthesize(text, controller.signal);
|
|
1172
|
+
if (counter !== this.counter) continue;
|
|
1173
|
+
if (audio?.length) this.emit("Audio", audio);
|
|
1174
|
+
} catch (error) {
|
|
1175
|
+
if (counter !== this.counter) continue;
|
|
1176
|
+
this.log("Error synthesizing speech", error);
|
|
1177
|
+
this.emit("Failed", [text, ...this.queue]);
|
|
1178
|
+
this.queue = [];
|
|
1179
|
+
} finally {
|
|
1180
|
+
if (this.controller === controller) this.controller = void 0;
|
|
1181
|
+
}
|
|
1182
|
+
}
|
|
1183
|
+
this.draining = false;
|
|
1184
|
+
if (this.queue.length > 0) this.drain();
|
|
1185
|
+
}
|
|
1186
|
+
};
|
|
1187
|
+
|
|
987
1188
|
// src/waitForParams.ts
|
|
988
1189
|
async function waitForParams(socket, validate) {
|
|
989
1190
|
return new Promise((resolve, reject) => {
|
|
@@ -1025,9 +1226,14 @@ async function waitForParams(socket, validate) {
|
|
|
1025
1226
|
MockAgent,
|
|
1026
1227
|
MockSTT,
|
|
1027
1228
|
MockTTS,
|
|
1229
|
+
Pcm16Resampler,
|
|
1028
1230
|
STT,
|
|
1231
|
+
SentenceSplitter,
|
|
1232
|
+
SentenceTTS,
|
|
1029
1233
|
TTS,
|
|
1234
|
+
float32ToPcm16,
|
|
1030
1235
|
handleError,
|
|
1236
|
+
pcm16ToFloat32,
|
|
1031
1237
|
waitForParams
|
|
1032
1238
|
});
|
|
1033
1239
|
//# sourceMappingURL=index.js.map
|