@micdrop/server 2.9.0 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -12
- package/dist/index.d.mts +372 -31
- package/dist/index.d.ts +372 -31
- package/dist/index.js +647 -115
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +639 -115
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { EventEmitter } from 'eventemitter3';
|
|
2
2
|
import { Writable, PassThrough, Readable } from 'stream';
|
|
3
3
|
import { z } from 'zod';
|
|
4
|
-
import WebSocket, { WebSocket as WebSocket$1 } from 'ws';
|
|
4
|
+
import WebSocket, { WebSocket as WebSocket$1, RawData, ClientOptions } from 'ws';
|
|
5
5
|
|
|
6
6
|
declare class Logger {
|
|
7
7
|
name: string;
|
|
@@ -21,8 +21,14 @@ declare enum MicdropServerCommands {
|
|
|
21
21
|
SkipAnswer = "SkipAnswer",
|
|
22
22
|
EndCall = "EndCall",
|
|
23
23
|
ToolCall = "ToolCall",
|
|
24
|
-
Classification = "Classification"
|
|
24
|
+
Classification = "Classification",
|
|
25
|
+
SampleRate = "SampleRate"
|
|
25
26
|
}
|
|
27
|
+
/**
|
|
28
|
+
* Rate of the audio the client records, and of the assistant voice until the
|
|
29
|
+
* server announces another one with a `SampleRate` command.
|
|
30
|
+
*/
|
|
31
|
+
declare const DEFAULT_SAMPLE_RATE = 16000;
|
|
26
32
|
/**
|
|
27
33
|
* Hears whether a sentence has landed, where voice activity detection only
|
|
28
34
|
* hears whether someone is speaking.
|
|
@@ -125,7 +131,7 @@ interface AgentOptions {
|
|
|
125
131
|
autoEndCall?: boolean | string;
|
|
126
132
|
autoSemanticTurn?: boolean | string;
|
|
127
133
|
autoIgnoreUserNoise?: boolean | string;
|
|
128
|
-
extract?:
|
|
134
|
+
extract?: ExtractOptions;
|
|
129
135
|
onBeforeAnswer?: (this: Agent, stream: Writable) => void | boolean | string | Promise<void | boolean | string>;
|
|
130
136
|
}
|
|
131
137
|
interface AgentEvents {
|
|
@@ -137,27 +143,48 @@ interface AgentEvents {
|
|
|
137
143
|
Failed: [];
|
|
138
144
|
}
|
|
139
145
|
interface ExtractOptions {
|
|
140
|
-
|
|
141
|
-
saveInMetadata?: boolean;
|
|
142
|
-
}
|
|
143
|
-
interface ExtractJsonOptions extends ExtractOptions {
|
|
144
|
-
json: true;
|
|
146
|
+
/** Called with the parsed value */
|
|
145
147
|
callback?: (value: any) => void;
|
|
148
|
+
/** Keeps the parsed value in the metadata of the message, as `extracted` */
|
|
149
|
+
saveInMetadata?: boolean;
|
|
150
|
+
/** @deprecated JSON is the only format, this flag changes nothing */
|
|
151
|
+
json?: true;
|
|
146
152
|
}
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
endTag: string;
|
|
150
|
-
}
|
|
153
|
+
/** @deprecated Use `ExtractOptions` */
|
|
154
|
+
type ExtractJsonOptions = ExtractOptions;
|
|
151
155
|
declare abstract class Agent<Options extends AgentOptions = AgentOptions, Events extends AgentEvents = AgentEvents> extends EventEmitter<Events> {
|
|
152
156
|
protected options: Options;
|
|
153
157
|
logger?: Logger;
|
|
154
158
|
conversation: MicdropConversation;
|
|
155
159
|
tools: Tool[];
|
|
160
|
+
private answerController?;
|
|
161
|
+
/**
|
|
162
|
+
* @deprecated Read the signal given to `generateAnswer` instead. Still
|
|
163
|
+
* bumped by every answer and every cancel, for agents written before it.
|
|
164
|
+
*/
|
|
156
165
|
protected answerCount: number;
|
|
157
|
-
protected answering: boolean;
|
|
158
166
|
constructor(options: Options);
|
|
159
|
-
|
|
160
|
-
|
|
167
|
+
/**
|
|
168
|
+
* Writes the answer to the stream, and records it in the conversation.
|
|
169
|
+
*
|
|
170
|
+
* The signal is aborted when the answer is cancelled or replaced by the next
|
|
171
|
+
* one. From then on nothing may be written nor recorded: hand it to the
|
|
172
|
+
* request, and check it before a retry and before adding a message.
|
|
173
|
+
*/
|
|
174
|
+
protected abstract generateAnswer(stream: PassThrough, signal: AbortSignal): Promise<void>;
|
|
175
|
+
/** Whether an answer is being generated */
|
|
176
|
+
get isAnswering(): boolean;
|
|
177
|
+
/** @deprecated Read `isAnswering` instead */
|
|
178
|
+
protected get answering(): boolean;
|
|
179
|
+
/** @deprecated The answer is over once its stream ends or it is cancelled */
|
|
180
|
+
protected set answering(_value: boolean);
|
|
181
|
+
/**
|
|
182
|
+
* Stops the answer being generated, by aborting its signal.
|
|
183
|
+
*
|
|
184
|
+
* An agent overriding it to stop more than the signal does calls
|
|
185
|
+
* `super.cancel()`.
|
|
186
|
+
*/
|
|
187
|
+
cancel(): void;
|
|
161
188
|
answer(): Readable;
|
|
162
189
|
addUserMessage(text: string, metadata?: MicdropAnswerMetadata): void;
|
|
163
190
|
addAssistantMessage(text: string, metadata?: MicdropAnswerMetadata): void;
|
|
@@ -179,7 +206,15 @@ declare abstract class Agent<Options extends AgentOptions = AgentOptions, Events
|
|
|
179
206
|
};
|
|
180
207
|
skipAnswer?: undefined;
|
|
181
208
|
}>;
|
|
182
|
-
|
|
209
|
+
/**
|
|
210
|
+
* Writes the answer to the stream as it is generated, leaving out the JSON
|
|
211
|
+
* value to extract: the voice stops at its first `{`.
|
|
212
|
+
*
|
|
213
|
+
* The value is cut from the message once the answer is complete, by
|
|
214
|
+
* `extract()`. Only the speech has to be cut while streaming.
|
|
215
|
+
*/
|
|
216
|
+
protected answerWriter(stream: Writable): (text: string) => void;
|
|
217
|
+
/** Cuts the JSON value to extract from a complete answer, and parses it */
|
|
183
218
|
extract(message: string): {
|
|
184
219
|
message: string;
|
|
185
220
|
metadata: MicdropAnswerMetadata | undefined;
|
|
@@ -207,7 +242,7 @@ declare class FallbackAgent extends Agent {
|
|
|
207
242
|
message: string;
|
|
208
243
|
metadata: MicdropAnswerMetadata | undefined;
|
|
209
244
|
};
|
|
210
|
-
protected generateAnswer(stream: PassThrough): Promise<void>;
|
|
245
|
+
protected generateAnswer(stream: PassThrough, signal: AbortSignal): Promise<void>;
|
|
211
246
|
cancel(): void;
|
|
212
247
|
destroy(): void;
|
|
213
248
|
private pipeAnswer;
|
|
@@ -223,7 +258,6 @@ declare class MockAgent extends Agent {
|
|
|
223
258
|
private i;
|
|
224
259
|
constructor();
|
|
225
260
|
protected generateAnswer(stream: PassThrough): Promise<void>;
|
|
226
|
-
cancel(): void;
|
|
227
261
|
}
|
|
228
262
|
|
|
229
263
|
/** What a classifier reads: a text, or JSON with named fields */
|
|
@@ -296,6 +330,20 @@ declare function turnInput(conversation: MicdropConversation, history?: number):
|
|
|
296
330
|
message: MicdropConversationMessage<MicdropAnswerMetadata>;
|
|
297
331
|
};
|
|
298
332
|
|
|
333
|
+
/**
|
|
334
|
+
* Cuts a stream of PCM16 bytes on sample boundaries.
|
|
335
|
+
*
|
|
336
|
+
* A network or process chunk can end in the middle of a 16-bit sample, and the
|
|
337
|
+
* client reads each chunk as whole samples. The odd byte is held back and put
|
|
338
|
+
* in front of the next chunk, so every chunk returned holds whole samples.
|
|
339
|
+
*/
|
|
340
|
+
declare class Pcm16Aligner {
|
|
341
|
+
private leftover?;
|
|
342
|
+
process(chunk: Buffer): Buffer;
|
|
343
|
+
/** Forgets the held byte, to start an independent stream */
|
|
344
|
+
reset(): void;
|
|
345
|
+
}
|
|
346
|
+
|
|
299
347
|
/**
|
|
300
348
|
* Streaming linear-interpolation resampler for PCM16 mono audio.
|
|
301
349
|
*
|
|
@@ -304,9 +352,10 @@ declare function turnInput(conversation: MicdropConversation, history?: number):
|
|
|
304
352
|
* keeps the fractional sample position continuous across chunks, so feeding a
|
|
305
353
|
* stream chunk by chunk yields the same result as resampling it in one go.
|
|
306
354
|
*
|
|
307
|
-
* Providers use it to bridge
|
|
308
|
-
*
|
|
309
|
-
* requires >= 24kHz)
|
|
355
|
+
* Providers use it to bridge the 16kHz PCM16 the Micdrop client records with
|
|
356
|
+
* the rate their API reads, like OpenaiSTT (16kHz -> 24kHz, the GA Realtime API
|
|
357
|
+
* requires >= 24kHz). Speech goes out at the rate it was produced, so text to
|
|
358
|
+
* speech has no use for it.
|
|
310
359
|
*/
|
|
311
360
|
declare class Pcm16Resampler {
|
|
312
361
|
private readonly step;
|
|
@@ -345,9 +394,50 @@ declare class MicdropError extends Error {
|
|
|
345
394
|
}
|
|
346
395
|
declare function handleError(socket: WebSocket, error: unknown): void;
|
|
347
396
|
|
|
397
|
+
/**
|
|
398
|
+
* Fetches a list from a provider once per key and account, then shares it with
|
|
399
|
+
* every caller of the process for a day. Calls asking at the same time share
|
|
400
|
+
* one request, and a failed one is not kept.
|
|
401
|
+
*
|
|
402
|
+
* @param key - What the list is, the provider first, then what changes it
|
|
403
|
+
* @param apiKey - The account, kept only as a hash
|
|
404
|
+
* @param load - Fetches the list
|
|
405
|
+
* @param refresh - Fetches it again, for an item added a moment ago
|
|
406
|
+
*/
|
|
407
|
+
declare function cachedList<T>(key: unknown[], apiKey: string, load: () => Promise<T[]>, refresh?: boolean): Promise<T[]>;
|
|
408
|
+
|
|
409
|
+
/**
|
|
410
|
+
* A model an agent, a speech to text or a text to speech can run, as its
|
|
411
|
+
* `listModels()` describes it.
|
|
412
|
+
*
|
|
413
|
+
* Providers describe their models unevenly, so only `id` is always there. The
|
|
414
|
+
* rest is filled when the provider says it.
|
|
415
|
+
*/
|
|
416
|
+
interface MicdropModel {
|
|
417
|
+
/** What the `model` option takes */
|
|
418
|
+
id: string;
|
|
419
|
+
name?: string;
|
|
420
|
+
description?: string;
|
|
421
|
+
/** Other names the provider answers to for the same model */
|
|
422
|
+
aliases?: string[];
|
|
423
|
+
/** Languages it speaks or transcribes, as codes such as `en` or `fr-FR` */
|
|
424
|
+
languages?: string[];
|
|
425
|
+
/** Can reason before answering, which delays its first word */
|
|
426
|
+
reasoning?: boolean;
|
|
427
|
+
/** The provider announced its end, `shutdownDate` says when if known */
|
|
428
|
+
deprecated?: boolean;
|
|
429
|
+
/** Day the provider stops serving it, as `YYYY-MM-DD` */
|
|
430
|
+
shutdownDate?: string;
|
|
431
|
+
}
|
|
432
|
+
/**
|
|
433
|
+
* Lists models once per provider, account and options, then shares the list
|
|
434
|
+
* with every caller of the process for a day. See `cachedList()`.
|
|
435
|
+
*/
|
|
436
|
+
declare function cachedModels(key: unknown[], apiKey: string, load: () => Promise<MicdropModel[]>, refresh?: boolean): Promise<MicdropModel[]>;
|
|
437
|
+
|
|
348
438
|
type RealtimeOptions = Pick<AgentOptions, 'systemPrompt' | 'autoEndCall' | 'autoSemanticTurn' | 'autoIgnoreUserNoise'>;
|
|
349
439
|
interface RealtimeEvents extends AgentEvents {
|
|
350
|
-
/** Voice of the assistant, in PCM16 at
|
|
440
|
+
/** Voice of the assistant, in PCM16 mono at `sampleRate` */
|
|
351
441
|
Audio: [Buffer];
|
|
352
442
|
/** What the assistant has said so far in the answer it is giving */
|
|
353
443
|
PartialMessage: [string];
|
|
@@ -433,7 +523,12 @@ declare abstract class Realtime<Options extends RealtimeOptions = RealtimeOption
|
|
|
433
523
|
protected abstract injectMessage(role: 'user' | 'assistant' | 'system', text: string): void;
|
|
434
524
|
/** Nobody is speaking, nor about to, so the connection can be renewed */
|
|
435
525
|
protected get isIdle(): boolean;
|
|
436
|
-
/**
|
|
526
|
+
/**
|
|
527
|
+
* Rate of the audio emitted, in Hz, the one the model speaks at. The server
|
|
528
|
+
* passes it on to the client, which plays the audio as it comes.
|
|
529
|
+
*/
|
|
530
|
+
get sampleRate(): number;
|
|
531
|
+
/** Voice of the answer, PCM16 at `sampleRate` */
|
|
437
532
|
protected emitAudio(chunk: Buffer): void;
|
|
438
533
|
/** Transcript of the answer, as it is spoken */
|
|
439
534
|
protected addAnswerTranscript(text: string): void;
|
|
@@ -496,20 +591,34 @@ declare class FallbackSTT extends STT {
|
|
|
496
591
|
private readonly options;
|
|
497
592
|
private stt;
|
|
498
593
|
private sttIndex;
|
|
594
|
+
private pending;
|
|
499
595
|
constructor(options: FallbackSTTOptions);
|
|
500
596
|
transcribe(audioStream: Readable): void;
|
|
501
597
|
destroy(): void;
|
|
502
598
|
private startNextSTT;
|
|
503
599
|
private onTranscript;
|
|
600
|
+
/**
|
|
601
|
+
* Hands every utterance still waiting to the next STT, one stream each so
|
|
602
|
+
* each gets its own transcript. One still being spoken keeps going there.
|
|
603
|
+
*/
|
|
504
604
|
private onFailed;
|
|
505
605
|
}
|
|
506
606
|
|
|
507
607
|
interface TTSEvents {
|
|
608
|
+
/** Voice of the answer, in PCM16 mono at `sampleRate` */
|
|
508
609
|
Audio: [Buffer];
|
|
509
610
|
Failed: [string[]];
|
|
510
611
|
}
|
|
511
612
|
declare abstract class TTS extends EventEmitter<TTSEvents> {
|
|
512
613
|
logger?: Logger;
|
|
614
|
+
/**
|
|
615
|
+
* Rate of the audio emitted, in Hz.
|
|
616
|
+
*
|
|
617
|
+
* The server reads it with each chunk and tells the client when it changes,
|
|
618
|
+
* so the audio reaches the client at the rate the provider produced it. A
|
|
619
|
+
* provider that offers several rates takes the one chosen in its options.
|
|
620
|
+
*/
|
|
621
|
+
get sampleRate(): number;
|
|
513
622
|
abstract speak(textStream: Readable): void;
|
|
514
623
|
abstract cancel(): void;
|
|
515
624
|
protected log(...message: any[]): void;
|
|
@@ -523,19 +632,34 @@ declare class FallbackTTS extends TTS {
|
|
|
523
632
|
private readonly options;
|
|
524
633
|
private tts;
|
|
525
634
|
private ttsIndex;
|
|
635
|
+
private current?;
|
|
526
636
|
constructor(options: FallbackTTSOptions);
|
|
637
|
+
/** Rate of the TTS in use, which changes when it falls back to the next */
|
|
638
|
+
get sampleRate(): number;
|
|
527
639
|
speak(textStream: Readable): void;
|
|
528
640
|
cancel(): void;
|
|
529
641
|
destroy(): void;
|
|
530
642
|
private startNextTTS;
|
|
531
643
|
private onAudio;
|
|
644
|
+
/**
|
|
645
|
+
* Hands the unspoken text to the next TTS. An utterance whose text is still
|
|
646
|
+
* being written goes on there, so its end is spoken too.
|
|
647
|
+
*/
|
|
532
648
|
private onFailed;
|
|
533
649
|
}
|
|
534
650
|
|
|
651
|
+
/**
|
|
652
|
+
* Plays the same recorded chunks for any answer, to run a call without a
|
|
653
|
+
* voice provider. The files hold raw PCM 16 bits, at 16 kHz unless another
|
|
654
|
+
* rate is given.
|
|
655
|
+
*/
|
|
535
656
|
declare class MockTTS extends TTS {
|
|
536
657
|
private audioFilePaths;
|
|
537
|
-
|
|
538
|
-
|
|
658
|
+
private readonly rate;
|
|
659
|
+
private generation;
|
|
660
|
+
constructor(audioFilePaths: string[], rate?: number);
|
|
661
|
+
get sampleRate(): number;
|
|
662
|
+
speak(textStream: Readable): void;
|
|
539
663
|
cancel(): void;
|
|
540
664
|
}
|
|
541
665
|
|
|
@@ -568,7 +692,7 @@ declare class SentenceSplitter {
|
|
|
568
692
|
* be fed the agent's answer token by token. This class buffers the answer into
|
|
569
693
|
* sentences, hands them over one at a time, and emits the audio in the order
|
|
570
694
|
* they were written. Subclasses only have to turn one sentence into PCM16 at
|
|
571
|
-
* the rate
|
|
695
|
+
* the rate they announce in `sampleRate`.
|
|
572
696
|
*
|
|
573
697
|
* Sentences are synthesized one after the other rather than at once: a local
|
|
574
698
|
* model is single threaded, so racing two sentences through it slows both down
|
|
@@ -583,7 +707,7 @@ declare abstract class SentenceTTS extends TTS {
|
|
|
583
707
|
private counter;
|
|
584
708
|
private synthesizing;
|
|
585
709
|
/**
|
|
586
|
-
* Turns one sentence into PCM16 audio at
|
|
710
|
+
* Turns one sentence into PCM16 audio at `sampleRate`.
|
|
587
711
|
*
|
|
588
712
|
* The signal is aborted when the utterance is cancelled, which is the moment
|
|
589
713
|
* to stop a subprocess or an inference that is no longer needed. Returning
|
|
@@ -606,12 +730,51 @@ declare abstract class SentenceTTS extends TTS {
|
|
|
606
730
|
private drain;
|
|
607
731
|
}
|
|
608
732
|
|
|
733
|
+
/**
|
|
734
|
+
* A voice a text to speech can speak with, as its `listVoices()` describes it.
|
|
735
|
+
*
|
|
736
|
+
* Providers describe their voices unevenly, so only `id` and `name` are always
|
|
737
|
+
* there. The rest is filled when the provider says it.
|
|
738
|
+
*/
|
|
739
|
+
interface MicdropVoice {
|
|
740
|
+
/** What the `voice` option of the text to speech takes */
|
|
741
|
+
id: string;
|
|
742
|
+
name: string;
|
|
743
|
+
/**
|
|
744
|
+
* Languages the voice speaks, as codes such as `en` or `fr-FR`. Absent when
|
|
745
|
+
* the provider does not say, or when the voice reads every language of its
|
|
746
|
+
* model, as the OpenAI and Gemini voices do.
|
|
747
|
+
*/
|
|
748
|
+
languages?: string[];
|
|
749
|
+
gender?: 'female' | 'male' | 'neutral';
|
|
750
|
+
description?: string;
|
|
751
|
+
/** Address of a short sample of the voice */
|
|
752
|
+
previewUrl?: string;
|
|
753
|
+
/** Created in the account, cloned or designed, rather than offered to all */
|
|
754
|
+
custom?: boolean;
|
|
755
|
+
}
|
|
756
|
+
/** Turns the gender words providers use into the three values above */
|
|
757
|
+
declare function voiceGender(value: string | null | undefined): MicdropVoice['gender'];
|
|
758
|
+
/**
|
|
759
|
+
* Lists voices once per provider, account and options, then shares the list
|
|
760
|
+
* with every caller of the process for a day. See `cachedList()`.
|
|
761
|
+
*/
|
|
762
|
+
declare function cachedVoices(key: unknown[], apiKey: string, load: () => Promise<MicdropVoice[]>, refresh?: boolean): Promise<MicdropVoice[]>;
|
|
763
|
+
/**
|
|
764
|
+
* The voices that speak a language, `fr` or `fr-FR` alike: a voice of any
|
|
765
|
+
* French locale counts. A voice declaring no language reads them all, so it
|
|
766
|
+
* stays in the list.
|
|
767
|
+
*/
|
|
768
|
+
declare function voicesForLanguage(voices: MicdropVoice[], language: string | undefined): MicdropVoice[];
|
|
769
|
+
|
|
609
770
|
interface MicdropServerEvents {
|
|
610
771
|
End: [MicdropCallSummary];
|
|
611
772
|
/** A message was added to the conversation, by the agent or by the server */
|
|
612
773
|
Message: [MicdropConversationItem];
|
|
774
|
+
/** Audio of the user, in PCM16 mono at 16 kHz */
|
|
613
775
|
UserAudio: [Buffer];
|
|
614
|
-
|
|
776
|
+
/** Voice of the assistant, in PCM16 mono at the rate given with it, in Hz */
|
|
777
|
+
AssistantAudio: [Buffer, number];
|
|
615
778
|
}
|
|
616
779
|
interface MicdropConfig {
|
|
617
780
|
/**
|
|
@@ -724,6 +887,9 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
|
|
|
724
887
|
logger?: Logger;
|
|
725
888
|
private startTime;
|
|
726
889
|
private lastMessageSpeeched?;
|
|
890
|
+
private cancelRound;
|
|
891
|
+
private assistantMessages;
|
|
892
|
+
private assistantSampleRate;
|
|
727
893
|
private ownConversation;
|
|
728
894
|
private operationQueue;
|
|
729
895
|
private isProcessingQueue;
|
|
@@ -731,6 +897,7 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
|
|
|
731
897
|
private userSpeechChunks;
|
|
732
898
|
private turnComplete?;
|
|
733
899
|
private heldTurnTimer?;
|
|
900
|
+
private turnRound;
|
|
734
901
|
constructor(socket: WebSocket$1, config: MicdropConfig);
|
|
735
902
|
/** Everything said so far, whether an agent or the server keeps it */
|
|
736
903
|
get conversation(): MicdropConversation;
|
|
@@ -772,7 +939,14 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
|
|
|
772
939
|
private sendFirstMessage;
|
|
773
940
|
answer(): void;
|
|
774
941
|
private _answer;
|
|
775
|
-
/**
|
|
942
|
+
/**
|
|
943
|
+
* Sends the answer to the client as it is written, ahead of the voice.
|
|
944
|
+
*
|
|
945
|
+
* The stream reaches here a moment after the agent wrote it, so the agent
|
|
946
|
+
* can record the complete message first, which the client then shows. A
|
|
947
|
+
* partial answer arriving after it would show the answer twice, so the
|
|
948
|
+
* forwarding stops once the message is sent.
|
|
949
|
+
*/
|
|
776
950
|
private forwardPartialAnswer;
|
|
777
951
|
/**
|
|
778
952
|
* Classifies the turn that just ended, with the one before, and keeps the
|
|
@@ -786,7 +960,10 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
|
|
|
786
960
|
}
|
|
787
961
|
|
|
788
962
|
interface AudioMessage {
|
|
963
|
+
/** PCM16 mono audio, at `sampleRate` */
|
|
789
964
|
buffer: Buffer;
|
|
965
|
+
/** Rate of `buffer`, in Hz: 16 kHz for the user, the voice's for the assistant */
|
|
966
|
+
sampleRate: number;
|
|
790
967
|
messageIndex: number;
|
|
791
968
|
message: string;
|
|
792
969
|
role: 'user' | 'assistant';
|
|
@@ -801,12 +978,16 @@ declare class MicdropRecorder extends EventEmitter<MicdropRecorderEvents> {
|
|
|
801
978
|
private audioMessages;
|
|
802
979
|
private currentUserChunks;
|
|
803
980
|
private currentAssistantChunks;
|
|
981
|
+
private currentAssistantRate;
|
|
982
|
+
private assistantResampler?;
|
|
804
983
|
private lastUserMessageIndex;
|
|
805
984
|
private lastAssistantMessageIndex;
|
|
806
985
|
constructor(server: MicdropServer);
|
|
807
986
|
private setupListeners;
|
|
808
987
|
private onUserAudio;
|
|
809
988
|
private onAssistantAudio;
|
|
989
|
+
/** A message holds one rate, the one its first chunk came at */
|
|
990
|
+
private toAssistantRate;
|
|
810
991
|
private onMessage;
|
|
811
992
|
private finalizeUserAudio;
|
|
812
993
|
private finalizeAssistantAudio;
|
|
@@ -827,7 +1008,8 @@ declare class MicdropRecorder extends EventEmitter<MicdropRecorderEvents> {
|
|
|
827
1008
|
*
|
|
828
1009
|
* Instances are keyed by the options they were built from, so changing a model
|
|
829
1010
|
* or a device loads that one and leaves the others alone. They stay loaded for
|
|
830
|
-
* the lifetime of the process
|
|
1011
|
+
* the lifetime of the process, or for `ttl` milliseconds when it is set, which
|
|
1012
|
+
* suits a list fetched from an API that changes now and then.
|
|
831
1013
|
*
|
|
832
1014
|
* ```typescript
|
|
833
1015
|
* const synthesizers = new SharedInstances<Synthesizer>()
|
|
@@ -839,8 +1021,15 @@ declare class MicdropRecorder extends EventEmitter<MicdropRecorderEvents> {
|
|
|
839
1021
|
* }
|
|
840
1022
|
* ```
|
|
841
1023
|
*/
|
|
1024
|
+
interface SharedInstancesOptions {
|
|
1025
|
+
/** How long an instance is kept once loaded, in ms. Forever by default. */
|
|
1026
|
+
ttl?: number;
|
|
1027
|
+
}
|
|
842
1028
|
declare class SharedInstances<T> {
|
|
1029
|
+
private readonly options;
|
|
843
1030
|
private entries;
|
|
1031
|
+
private expiries;
|
|
1032
|
+
constructor(options?: SharedInstancesOptions);
|
|
844
1033
|
/**
|
|
845
1034
|
* Returns the instance for this key, calling `create` the first time.
|
|
846
1035
|
*
|
|
@@ -849,12 +1038,164 @@ declare class SharedInstances<T> {
|
|
|
849
1038
|
* instead of loading the model twice.
|
|
850
1039
|
*/
|
|
851
1040
|
load(key: unknown, create: () => Promise<T>): Promise<T>;
|
|
1041
|
+
/** Forgets an instance whose time is up */
|
|
1042
|
+
private expire;
|
|
1043
|
+
/** The instance for this key, loaded or being loaded, without loading it. */
|
|
1044
|
+
get(key: unknown): Promise<T> | undefined;
|
|
852
1045
|
/** Whether an instance is loaded, or being loaded, for this key. */
|
|
853
1046
|
has(key: unknown): boolean;
|
|
1047
|
+
/**
|
|
1048
|
+
* Forgets the instance for this key, so the next call loads a new one. For
|
|
1049
|
+
* an instance that broke, a crashed worker for instance. Releasing what it
|
|
1050
|
+
* holds is left to the caller.
|
|
1051
|
+
*/
|
|
1052
|
+
delete(key: unknown): void;
|
|
854
1053
|
/** Drops every instance, so the next call loads again. */
|
|
855
1054
|
clear(): void;
|
|
856
1055
|
}
|
|
857
1056
|
|
|
1057
|
+
/**
|
|
1058
|
+
* Where a {@link ReconnectingSocket} stands.
|
|
1059
|
+
*
|
|
1060
|
+
* - `idle`: not connected yet, `connect()` has not been called
|
|
1061
|
+
* - `connecting`: an attempt is under way, the first one or a retry
|
|
1062
|
+
* - `open`: connected, and done with its handshake if it has one
|
|
1063
|
+
* - `reconnecting`: the connection dropped, a new attempt waits for its delay
|
|
1064
|
+
* - `failed`: every retry failed, `connect()` starts over
|
|
1065
|
+
* - `destroyed`: closed for good
|
|
1066
|
+
*/
|
|
1067
|
+
type ReconnectingSocketStatus = 'idle' | 'connecting' | 'open' | 'reconnecting' | 'failed' | 'destroyed';
|
|
1068
|
+
interface ReconnectingSocketOptions {
|
|
1069
|
+
/**
|
|
1070
|
+
* Address to connect to, or a function giving it for each attempt, for a
|
|
1071
|
+
* session URL or a token that has to be fresh.
|
|
1072
|
+
*/
|
|
1073
|
+
url: string | (() => string | Promise<string>);
|
|
1074
|
+
/** Options of the `ws` client, headers included */
|
|
1075
|
+
clientOptions?: ClientOptions;
|
|
1076
|
+
/** How long an attempt may take, handshake included, 5000 ms by default */
|
|
1077
|
+
connectionTimeout?: number;
|
|
1078
|
+
/** How long to wait before a new attempt, 1000 ms by default */
|
|
1079
|
+
retryDelay?: number;
|
|
1080
|
+
/** How many attempts in a row may fail before giving up, 3 by default */
|
|
1081
|
+
maxRetry?: number;
|
|
1082
|
+
/**
|
|
1083
|
+
* Sets the session up before the connection counts as open, for a protocol
|
|
1084
|
+
* that takes nothing else until then. See {@link ReconnectingSocketHandshake}.
|
|
1085
|
+
*/
|
|
1086
|
+
handshake?: ReconnectingSocketHandshake;
|
|
1087
|
+
/** Sends a message on a regular basis, to keep an idle connection open */
|
|
1088
|
+
keepAlive?: {
|
|
1089
|
+
/** Interval between two messages, in milliseconds */
|
|
1090
|
+
interval: number;
|
|
1091
|
+
message: string | (() => string);
|
|
1092
|
+
};
|
|
1093
|
+
/** Receives the log lines, the `log` method of the owner usually */
|
|
1094
|
+
log?: (...message: any[]) => void;
|
|
1095
|
+
}
|
|
1096
|
+
/**
|
|
1097
|
+
* What a connection does between the socket opening and the session being
|
|
1098
|
+
* ready.
|
|
1099
|
+
*
|
|
1100
|
+
* Everything runs in the tick of the message it reads, so the connection is
|
|
1101
|
+
* open before the next message arrives and none is lost in between.
|
|
1102
|
+
*/
|
|
1103
|
+
interface ReconnectingSocketHandshake {
|
|
1104
|
+
/** Sends what opens the session, as soon as the socket opens */
|
|
1105
|
+
start?(socket: WebSocket): void;
|
|
1106
|
+
/**
|
|
1107
|
+
* Reads the messages that come before the session is ready, which are not
|
|
1108
|
+
* emitted as `Message`, and returns true on the one saying it is. Throwing
|
|
1109
|
+
* fails the attempt, for a session the server refused.
|
|
1110
|
+
*
|
|
1111
|
+
* Without it, the connection opens right after `start`, and proves itself
|
|
1112
|
+
* like one without a handshake: by a first message, or by staying open.
|
|
1113
|
+
*/
|
|
1114
|
+
read?(data: RawData, isBinary: boolean): boolean;
|
|
1115
|
+
}
|
|
1116
|
+
interface ReconnectingSocketEvents {
|
|
1117
|
+
/** The status changed */
|
|
1118
|
+
StatusChange: [ReconnectingSocketStatus];
|
|
1119
|
+
/**
|
|
1120
|
+
* The connection is open. A reconnection is the moment to send again what
|
|
1121
|
+
* the previous connection did not finish.
|
|
1122
|
+
*/
|
|
1123
|
+
Open: [{
|
|
1124
|
+
reconnected: boolean;
|
|
1125
|
+
}];
|
|
1126
|
+
Message: [RawData, boolean];
|
|
1127
|
+
/** The connection dropped, `reconnecting` or `failed` follows */
|
|
1128
|
+
Close: [number, string];
|
|
1129
|
+
/** Every retry failed */
|
|
1130
|
+
Failed: [Error];
|
|
1131
|
+
}
|
|
1132
|
+
/**
|
|
1133
|
+
* A WebSocket to a provider that reconnects on its own.
|
|
1134
|
+
*
|
|
1135
|
+
* Every connection a provider holds goes through the same states, and they
|
|
1136
|
+
* live here, in one place: one attempt at a time, a failure counted once
|
|
1137
|
+
* whatever the socket reports about it, a retry budget, and a `ready()` that
|
|
1138
|
+
* settles whatever happens. A socket left behind by a reconnection is silenced,
|
|
1139
|
+
* so it never acts on the one that replaced it.
|
|
1140
|
+
*
|
|
1141
|
+
* The owner keeps what belongs to its protocol: what to send again on
|
|
1142
|
+
* `Open`, and what the messages mean.
|
|
1143
|
+
*/
|
|
1144
|
+
declare class ReconnectingSocket extends EventEmitter<ReconnectingSocketEvents> {
|
|
1145
|
+
private readonly options;
|
|
1146
|
+
private _status;
|
|
1147
|
+
private socket?;
|
|
1148
|
+
private everOpened;
|
|
1149
|
+
private retryCount;
|
|
1150
|
+
private proofTimer?;
|
|
1151
|
+
private attemptId;
|
|
1152
|
+
private retryTimer?;
|
|
1153
|
+
private keepAliveTimer?;
|
|
1154
|
+
private waiters;
|
|
1155
|
+
constructor(options: ReconnectingSocketOptions);
|
|
1156
|
+
get status(): ReconnectingSocketStatus;
|
|
1157
|
+
/** Whether a message sent now goes out */
|
|
1158
|
+
get isOpen(): boolean;
|
|
1159
|
+
/**
|
|
1160
|
+
* Opens the connection. Called again after `failed`, it starts over with a
|
|
1161
|
+
* full retry budget, and does nothing in any other status.
|
|
1162
|
+
*/
|
|
1163
|
+
connect(): void;
|
|
1164
|
+
/**
|
|
1165
|
+
* Waits for the connection to be open.
|
|
1166
|
+
* @returns True once open, false if it failed or was destroyed meanwhile
|
|
1167
|
+
*/
|
|
1168
|
+
ready(): Promise<boolean>;
|
|
1169
|
+
/**
|
|
1170
|
+
* Sends a message if the connection is open
|
|
1171
|
+
* @returns False when it did not go out
|
|
1172
|
+
*/
|
|
1173
|
+
send(data: string | Buffer): boolean;
|
|
1174
|
+
/**
|
|
1175
|
+
* Drops the connection and opens a new one right away, for a provider that
|
|
1176
|
+
* can only stop what it is doing by closing. It costs no retry, and the new
|
|
1177
|
+
* connection is not a reconnection.
|
|
1178
|
+
*/
|
|
1179
|
+
restart(): void;
|
|
1180
|
+
/** Closes the connection for good */
|
|
1181
|
+
destroy(): void;
|
|
1182
|
+
private attempt;
|
|
1183
|
+
private onOpen;
|
|
1184
|
+
/** The connection works, the retry budget is whole again */
|
|
1185
|
+
private prove;
|
|
1186
|
+
private clearProofTimer;
|
|
1187
|
+
private onAttemptFailed;
|
|
1188
|
+
private scheduleRetry;
|
|
1189
|
+
/** Lets go of the current socket, which will never be heard from again */
|
|
1190
|
+
private drop;
|
|
1191
|
+
private startKeepAlive;
|
|
1192
|
+
private stopKeepAlive;
|
|
1193
|
+
private clearRetryTimer;
|
|
1194
|
+
private settleWaiters;
|
|
1195
|
+
private setStatus;
|
|
1196
|
+
private log;
|
|
1197
|
+
}
|
|
1198
|
+
|
|
858
1199
|
declare function waitForParams<CallParams>(socket: WebSocket$1, validate: (params: any) => CallParams): Promise<CallParams>;
|
|
859
1200
|
|
|
860
|
-
export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, Classifier, type ClassifierEvents, type ClassifierInput, type DeepPartial, type ExtractJsonOptions, type ExtractOptions,
|
|
1201
|
+
export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, Classifier, type ClassifierEvents, type ClassifierInput, DEFAULT_SAMPLE_RATE, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, type MicdropClassification, type MicdropClassifierOptions, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, type MicdropModel, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, type MicdropTurnInput, type MicdropVoice, MockAgent, MockSTT, MockTTS, Pcm16Aligner, Pcm16Resampler, Realtime, type RealtimeEvents, type RealtimeOptions, ReconnectingSocket, type ReconnectingSocketEvents, type ReconnectingSocketHandshake, type ReconnectingSocketOptions, type ReconnectingSocketStatus, STT, type STTEvents, SentenceSplitter, SentenceTTS, SharedInstances, type SharedInstancesOptions, TTS, type TTSEvents, type Tool, type TurnDetector, cachedList, cachedModels, cachedVoices, currentTurn, float32ToPcm16, getTurnClassification, handleError, pcm16ToFloat32, turnInput, voiceGender, voicesForLanguage, waitForParams };
|