@micdrop/server 2.6.0 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/dist/index.d.mts +191 -5
- package/dist/index.d.ts +191 -5
- package/dist/index.js +380 -25
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +378 -25
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -72,11 +72,15 @@ A server given a speech to text alone transcribes and stays quiet, which is all
|
|
|
72
72
|
a dictation tool needs. One given an agent and no voice answers in writing. See
|
|
73
73
|
[dictation and text-only calls](https://micdrop.dev/docs/server/dictation).
|
|
74
74
|
|
|
75
|
+
A `realtime` model hears the user and answers with its own voice, in place of
|
|
76
|
+
all three. See [realtime models](https://micdrop.dev/docs/server/realtime).
|
|
77
|
+
|
|
75
78
|
### Available implementations
|
|
76
79
|
|
|
77
80
|
Micdrop provides ready-to-use implementations for the following AI providers:
|
|
78
81
|
|
|
79
82
|
- [@micdrop/openai](https://micdrop.dev/docs/ai-integration/provided-integrations/openai)
|
|
83
|
+
- [@micdrop/gemini](https://micdrop.dev/docs/ai-integration/provided-integrations/gemini)
|
|
80
84
|
- [@micdrop/ai-sdk](https://micdrop.dev/docs/ai-integration/provided-integrations/ai-sdk)
|
|
81
85
|
- [@micdrop/elevenlabs](https://micdrop.dev/docs/ai-integration/provided-integrations/elevenlabs)
|
|
82
86
|
- [@micdrop/cartesia](https://micdrop.dev/docs/ai-integration/provided-integrations/cartesia)
|
|
@@ -90,6 +94,7 @@ You can use provided abstractions to write your own implementation:
|
|
|
90
94
|
- **[Agent](https://micdrop.dev/docs/ai-integration/custom-integrations/custom-agent)** - Abstract class for answer generation
|
|
91
95
|
- **[STT](https://micdrop.dev/docs/ai-integration/custom-integrations/custom-stt)** - Abstract class for speech-to-text
|
|
92
96
|
- **[TTS](https://micdrop.dev/docs/ai-integration/custom-integrations/custom-tts)** - Abstract class for text-to-speech
|
|
97
|
+
- **[Realtime](https://micdrop.dev/docs/server/realtime#writing-your-own)** - Abstract class for speech-to-speech models
|
|
93
98
|
|
|
94
99
|
## Documentation
|
|
95
100
|
|
package/dist/index.d.mts
CHANGED
|
@@ -122,7 +122,7 @@ interface ExtractTagOptions extends ExtractOptions {
|
|
|
122
122
|
startTag: string;
|
|
123
123
|
endTag: string;
|
|
124
124
|
}
|
|
125
|
-
declare abstract class Agent<Options extends AgentOptions = AgentOptions> extends EventEmitter<
|
|
125
|
+
declare abstract class Agent<Options extends AgentOptions = AgentOptions, Events extends AgentEvents = AgentEvents> extends EventEmitter<Events> {
|
|
126
126
|
protected options: Options;
|
|
127
127
|
logger?: Logger;
|
|
128
128
|
conversation: MicdropConversation;
|
|
@@ -158,6 +158,13 @@ declare abstract class Agent<Options extends AgentOptions = AgentOptions> extend
|
|
|
158
158
|
message: string;
|
|
159
159
|
metadata: MicdropAnswerMetadata | undefined;
|
|
160
160
|
};
|
|
161
|
+
/**
|
|
162
|
+
* Emits one of the events every agent has.
|
|
163
|
+
*
|
|
164
|
+
* A subclass can add events of its own, and TypeScript then no longer knows
|
|
165
|
+
* the shared ones are among them.
|
|
166
|
+
*/
|
|
167
|
+
protected emitAgentEvent<Name extends keyof AgentEvents>(event: Name, ...args: AgentEvents[Name]): void;
|
|
161
168
|
protected log(...message: any[]): void;
|
|
162
169
|
destroy(): void;
|
|
163
170
|
}
|
|
@@ -242,6 +249,134 @@ declare class MicdropError extends Error {
|
|
|
242
249
|
}
|
|
243
250
|
declare function handleError(socket: WebSocket, error: unknown): void;
|
|
244
251
|
|
|
252
|
+
type RealtimeOptions = Pick<AgentOptions, 'systemPrompt' | 'autoEndCall' | 'autoSemanticTurn' | 'autoIgnoreUserNoise'>;
|
|
253
|
+
interface RealtimeEvents extends AgentEvents {
|
|
254
|
+
/** Voice of the assistant, in PCM16 at 16 kHz */
|
|
255
|
+
Audio: [Buffer];
|
|
256
|
+
/** What the assistant has said so far in the answer it is giving */
|
|
257
|
+
PartialMessage: [string];
|
|
258
|
+
}
|
|
259
|
+
/**
|
|
260
|
+
* A model that hears the user and answers with its own voice, standing in for
|
|
261
|
+
* the speech to text, the agent and the text to speech of a call.
|
|
262
|
+
*
|
|
263
|
+
* It is an agent: it holds the conversation and the tools, and emits the same
|
|
264
|
+
* events. What differs is what goes in and out, audio in turns rather than
|
|
265
|
+
* transcripts, and audio rather than text.
|
|
266
|
+
*
|
|
267
|
+
* This class keeps what every provider shares: when a turn opens, which
|
|
268
|
+
* answer the output belongs to, dropping what arrives after an interruption,
|
|
269
|
+
* and writing the conversation in the order it was spoken. A provider
|
|
270
|
+
* implements the protected methods that talk to its API.
|
|
271
|
+
*/
|
|
272
|
+
declare abstract class Realtime<Options extends RealtimeOptions = RealtimeOptions> extends Agent<Options, RealtimeEvents> {
|
|
273
|
+
private turn;
|
|
274
|
+
private currentAnswer?;
|
|
275
|
+
private discarding;
|
|
276
|
+
private queue;
|
|
277
|
+
private answerWaiters;
|
|
278
|
+
private userTranscript;
|
|
279
|
+
private awaitingUserTranscript;
|
|
280
|
+
private userTranscriptTimer?;
|
|
281
|
+
private afterUserTranscript;
|
|
282
|
+
/**
|
|
283
|
+
* The user started speaking.
|
|
284
|
+
*
|
|
285
|
+
* The assistant stops, and the audio read from the stream goes to the model.
|
|
286
|
+
* A turn left open, because the sentence sounded unfinished, carries on.
|
|
287
|
+
*/
|
|
288
|
+
startTurn(audio: Readable): void;
|
|
289
|
+
/** Closes the turn of the user, and has the model answer it */
|
|
290
|
+
endTurn(): void;
|
|
291
|
+
/** Drops the turn of the user, without an answer */
|
|
292
|
+
discardTurn(): void;
|
|
293
|
+
/** Has the model speak without waiting for the user, to open a call */
|
|
294
|
+
respond(): void;
|
|
295
|
+
/**
|
|
296
|
+
* Has the model say this text, word for word.
|
|
297
|
+
*
|
|
298
|
+
* The caller records the text in the conversation, as with a text to speech,
|
|
299
|
+
* so the transcript of what the model says is not recorded again.
|
|
300
|
+
*/
|
|
301
|
+
say(text: string): void;
|
|
302
|
+
cancel(): void;
|
|
303
|
+
/**
|
|
304
|
+
* Adds a message the model did not hear, and tells the model about it.
|
|
305
|
+
*
|
|
306
|
+
* The transcripts of the call are recorded apart, so they are not sent back.
|
|
307
|
+
*/
|
|
308
|
+
addMessage(role: 'user' | 'assistant' | 'system', text: string, metadata?: MicdropAnswerMetadata): void;
|
|
309
|
+
addTool<Schema extends z.ZodObject>(tool: Tool<Schema>): void;
|
|
310
|
+
removeTool(name: string): void;
|
|
311
|
+
/**
|
|
312
|
+
* Lists the events that have listeners.
|
|
313
|
+
*
|
|
314
|
+
* Typed as the events of any agent, so a realtime model goes wherever an
|
|
315
|
+
* agent is expected. The two extra events do not change what that code reads.
|
|
316
|
+
*/
|
|
317
|
+
eventNames(): Array<keyof AgentEvents>;
|
|
318
|
+
destroy(): void;
|
|
319
|
+
protected generateAnswer(stream: PassThrough): Promise<void>;
|
|
320
|
+
/** Sends audio of the user, PCM16 at 16 kHz */
|
|
321
|
+
protected abstract appendAudio(chunk: Buffer): void;
|
|
322
|
+
/** Opens a turn of the user, right before its first audio */
|
|
323
|
+
protected abstract openTurn(): void;
|
|
324
|
+
/** Closes the turn of the user, which has the model answer it */
|
|
325
|
+
protected abstract closeTurn(): void;
|
|
326
|
+
/** Drops the audio of the open turn, without an answer */
|
|
327
|
+
protected abstract clearTurn(): void;
|
|
328
|
+
/** Has the model speak with no turn of the user to answer */
|
|
329
|
+
protected abstract generate(): void;
|
|
330
|
+
/** Has the model say this text, word for word */
|
|
331
|
+
protected abstract speakText(text: string): void;
|
|
332
|
+
/** Stops the answer the model is giving */
|
|
333
|
+
protected abstract cancelAnswer(): void;
|
|
334
|
+
/** Sends the tools to the model again, after one was added or removed */
|
|
335
|
+
protected abstract updateTools(): void;
|
|
336
|
+
/** Tells the model about a message it did not hear */
|
|
337
|
+
protected abstract injectMessage(role: 'user' | 'assistant' | 'system', text: string): void;
|
|
338
|
+
/** Nobody is speaking, nor about to, so the connection can be renewed */
|
|
339
|
+
protected get isIdle(): boolean;
|
|
340
|
+
/** Voice of the answer, PCM16 at 16 kHz */
|
|
341
|
+
protected emitAudio(chunk: Buffer): void;
|
|
342
|
+
/** Transcript of the answer, as it is spoken */
|
|
343
|
+
protected addAnswerTranscript(text: string): void;
|
|
344
|
+
/**
|
|
345
|
+
* The model finished its answer, or finished dropping an interrupted one.
|
|
346
|
+
*
|
|
347
|
+
* An answer without audio tells the client to stop waiting for one.
|
|
348
|
+
*/
|
|
349
|
+
protected endAnswer(): void;
|
|
350
|
+
/** Runs a tool the model called, and records it in the conversation */
|
|
351
|
+
protected runTool(toolCall: MicdropConversationToolCall): Promise<{
|
|
352
|
+
output: any;
|
|
353
|
+
skipAnswer: boolean | undefined;
|
|
354
|
+
} | {
|
|
355
|
+
output: {
|
|
356
|
+
error: any;
|
|
357
|
+
};
|
|
358
|
+
skipAnswer?: undefined;
|
|
359
|
+
}>;
|
|
360
|
+
/** Transcript of the turn of the user, possibly in several pieces */
|
|
361
|
+
protected addUserTranscript(text: string): void;
|
|
362
|
+
/** The transcript of the turn is complete, it goes in the conversation */
|
|
363
|
+
protected commitUserTranscript(): void;
|
|
364
|
+
private sendAudio;
|
|
365
|
+
private beginAnswer;
|
|
366
|
+
/**
|
|
367
|
+
* The answer output belongs to, if it is still wanted.
|
|
368
|
+
*
|
|
369
|
+
* The model can speak without being asked, to follow up on a tool call for
|
|
370
|
+
* instance, which opens an answer of its own.
|
|
371
|
+
*/
|
|
372
|
+
private activeAnswer;
|
|
373
|
+
private whenIdle;
|
|
374
|
+
private recordAnswer;
|
|
375
|
+
private awaitUserTranscript;
|
|
376
|
+
private clearUserTranscriptTimer;
|
|
377
|
+
private resolveAnswerWaiters;
|
|
378
|
+
}
|
|
379
|
+
|
|
245
380
|
interface STTEvents {
|
|
246
381
|
Transcript: [string];
|
|
247
382
|
Failed: [Buffer[]];
|
|
@@ -407,12 +542,12 @@ interface MicdropConfig {
|
|
|
407
542
|
agent?: Agent;
|
|
408
543
|
/**
|
|
409
544
|
* Turns what the user says into text, the one component a call cannot do
|
|
410
|
-
* without.
|
|
545
|
+
* without, unless a realtime model hears the user itself.
|
|
411
546
|
*
|
|
412
547
|
* It is fed the audio of each turn as it arrives, and the transcripts it
|
|
413
548
|
* returns drive everything that follows.
|
|
414
549
|
*/
|
|
415
|
-
stt
|
|
550
|
+
stt?: STT;
|
|
416
551
|
/**
|
|
417
552
|
* Gives the answers a voice.
|
|
418
553
|
*
|
|
@@ -440,6 +575,14 @@ interface MicdropConfig {
|
|
|
440
575
|
turnDetector?: TurnDetector;
|
|
441
576
|
/** How long to wait for the rest of a sentence, 4000 ms by default */
|
|
442
577
|
turnMaxWait?: number;
|
|
578
|
+
/**
|
|
579
|
+
* Hears the user and answers with its own voice, in place of `stt`, `agent`
|
|
580
|
+
* and `tts`, which are then ignored.
|
|
581
|
+
*
|
|
582
|
+
* The client does not change: it still detects when the user speaks, and the
|
|
583
|
+
* turn it sends goes to the model as audio.
|
|
584
|
+
*/
|
|
585
|
+
realtime?: Realtime;
|
|
443
586
|
}
|
|
444
587
|
declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
|
|
445
588
|
socket: WebSocket$1 | null;
|
|
@@ -457,6 +600,8 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
|
|
|
457
600
|
constructor(socket: WebSocket$1, config: MicdropConfig);
|
|
458
601
|
/** Everything said so far, whether an agent or the server keeps it */
|
|
459
602
|
get conversation(): MicdropConversation;
|
|
603
|
+
/** What holds the conversation and answers, a realtime model included */
|
|
604
|
+
private get agent();
|
|
460
605
|
private log;
|
|
461
606
|
private processQueue;
|
|
462
607
|
private queueOperation;
|
|
@@ -477,9 +622,11 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
|
|
|
477
622
|
* none leaves the call silent for good.
|
|
478
623
|
*/
|
|
479
624
|
private holdTurn;
|
|
625
|
+
/** Answers what the user just said */
|
|
626
|
+
private answerTurn;
|
|
480
627
|
private releaseHeldTurn;
|
|
481
628
|
private onTranscriptSTT;
|
|
482
|
-
private
|
|
629
|
+
private onAssistantAudio;
|
|
483
630
|
private onMessageAgent;
|
|
484
631
|
/** Adds a message to the conversation and sends it to the client */
|
|
485
632
|
addUserMessage(text: string, metadata?: MicdropAnswerMetadata): void;
|
|
@@ -528,6 +675,45 @@ declare class MicdropRecorder extends EventEmitter<MicdropRecorderEvents> {
|
|
|
528
675
|
protected log(...message: any[]): void;
|
|
529
676
|
}
|
|
530
677
|
|
|
678
|
+
/**
|
|
679
|
+
* Keeps one loaded instance per configuration, shared by every call.
|
|
680
|
+
*
|
|
681
|
+
* A local model costs from a few hundred milliseconds to a minute to load, and
|
|
682
|
+
* holding a second copy of the same weights wastes the memory the rest of the
|
|
683
|
+
* pipeline needs. Inference is bound to a few threads anyway, so two calls
|
|
684
|
+
* generating at once compete for the same cores whether they share the
|
|
685
|
+
* instance or not.
|
|
686
|
+
*
|
|
687
|
+
* Instances are keyed by the options they were built from, so changing a model
|
|
688
|
+
* or a device loads that one and leaves the others alone. They stay loaded for
|
|
689
|
+
* the lifetime of the process.
|
|
690
|
+
*
|
|
691
|
+
* ```typescript
|
|
692
|
+
* const synthesizers = new SharedInstances<Synthesizer>()
|
|
693
|
+
*
|
|
694
|
+
* function loadSynthesizer(options: Options) {
|
|
695
|
+
* return synthesizers.load([options.model, options.device], () =>
|
|
696
|
+
* Synthesizer.from_pretrained(options.model, { device: options.device })
|
|
697
|
+
* )
|
|
698
|
+
* }
|
|
699
|
+
* ```
|
|
700
|
+
*/
|
|
701
|
+
declare class SharedInstances<T> {
|
|
702
|
+
private entries;
|
|
703
|
+
/**
|
|
704
|
+
* Returns the instance for this key, calling `create` the first time.
|
|
705
|
+
*
|
|
706
|
+
* The key can be any JSON-serializable value, usually an array of the
|
|
707
|
+
* options that define the instance. Concurrent callers get the same promise
|
|
708
|
+
* instead of loading the model twice.
|
|
709
|
+
*/
|
|
710
|
+
load(key: unknown, create: () => Promise<T>): Promise<T>;
|
|
711
|
+
/** Whether an instance is loaded, or being loaded, for this key. */
|
|
712
|
+
has(key: unknown): boolean;
|
|
713
|
+
/** Drops every instance, so the next call loads again. */
|
|
714
|
+
clear(): void;
|
|
715
|
+
}
|
|
716
|
+
|
|
531
717
|
declare function waitForParams<CallParams>(socket: WebSocket$1, validate: (params: any) => CallParams): Promise<CallParams>;
|
|
532
718
|
|
|
533
|
-
export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, STT, type STTEvents, SentenceSplitter, SentenceTTS, TTS, type TTSEvents, type Tool, type TurnDetector, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };
|
|
719
|
+
export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, Realtime, type RealtimeEvents, type RealtimeOptions, STT, type STTEvents, SentenceSplitter, SentenceTTS, SharedInstances, TTS, type TTSEvents, type Tool, type TurnDetector, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };
|
package/dist/index.d.ts
CHANGED
|
@@ -122,7 +122,7 @@ interface ExtractTagOptions extends ExtractOptions {
|
|
|
122
122
|
startTag: string;
|
|
123
123
|
endTag: string;
|
|
124
124
|
}
|
|
125
|
-
declare abstract class Agent<Options extends AgentOptions = AgentOptions> extends EventEmitter<
|
|
125
|
+
declare abstract class Agent<Options extends AgentOptions = AgentOptions, Events extends AgentEvents = AgentEvents> extends EventEmitter<Events> {
|
|
126
126
|
protected options: Options;
|
|
127
127
|
logger?: Logger;
|
|
128
128
|
conversation: MicdropConversation;
|
|
@@ -158,6 +158,13 @@ declare abstract class Agent<Options extends AgentOptions = AgentOptions> extend
|
|
|
158
158
|
message: string;
|
|
159
159
|
metadata: MicdropAnswerMetadata | undefined;
|
|
160
160
|
};
|
|
161
|
+
/**
|
|
162
|
+
* Emits one of the events every agent has.
|
|
163
|
+
*
|
|
164
|
+
* A subclass can add events of its own, and TypeScript then no longer knows
|
|
165
|
+
* the shared ones are among them.
|
|
166
|
+
*/
|
|
167
|
+
protected emitAgentEvent<Name extends keyof AgentEvents>(event: Name, ...args: AgentEvents[Name]): void;
|
|
161
168
|
protected log(...message: any[]): void;
|
|
162
169
|
destroy(): void;
|
|
163
170
|
}
|
|
@@ -242,6 +249,134 @@ declare class MicdropError extends Error {
|
|
|
242
249
|
}
|
|
243
250
|
declare function handleError(socket: WebSocket, error: unknown): void;
|
|
244
251
|
|
|
252
|
+
type RealtimeOptions = Pick<AgentOptions, 'systemPrompt' | 'autoEndCall' | 'autoSemanticTurn' | 'autoIgnoreUserNoise'>;
|
|
253
|
+
interface RealtimeEvents extends AgentEvents {
|
|
254
|
+
/** Voice of the assistant, in PCM16 at 16 kHz */
|
|
255
|
+
Audio: [Buffer];
|
|
256
|
+
/** What the assistant has said so far in the answer it is giving */
|
|
257
|
+
PartialMessage: [string];
|
|
258
|
+
}
|
|
259
|
+
/**
|
|
260
|
+
* A model that hears the user and answers with its own voice, standing in for
|
|
261
|
+
* the speech to text, the agent and the text to speech of a call.
|
|
262
|
+
*
|
|
263
|
+
* It is an agent: it holds the conversation and the tools, and emits the same
|
|
264
|
+
* events. What differs is what goes in and out, audio in turns rather than
|
|
265
|
+
* transcripts, and audio rather than text.
|
|
266
|
+
*
|
|
267
|
+
* This class keeps what every provider shares: when a turn opens, which
|
|
268
|
+
* answer the output belongs to, dropping what arrives after an interruption,
|
|
269
|
+
* and writing the conversation in the order it was spoken. A provider
|
|
270
|
+
* implements the protected methods that talk to its API.
|
|
271
|
+
*/
|
|
272
|
+
declare abstract class Realtime<Options extends RealtimeOptions = RealtimeOptions> extends Agent<Options, RealtimeEvents> {
|
|
273
|
+
private turn;
|
|
274
|
+
private currentAnswer?;
|
|
275
|
+
private discarding;
|
|
276
|
+
private queue;
|
|
277
|
+
private answerWaiters;
|
|
278
|
+
private userTranscript;
|
|
279
|
+
private awaitingUserTranscript;
|
|
280
|
+
private userTranscriptTimer?;
|
|
281
|
+
private afterUserTranscript;
|
|
282
|
+
/**
|
|
283
|
+
* The user started speaking.
|
|
284
|
+
*
|
|
285
|
+
* The assistant stops, and the audio read from the stream goes to the model.
|
|
286
|
+
* A turn left open, because the sentence sounded unfinished, carries on.
|
|
287
|
+
*/
|
|
288
|
+
startTurn(audio: Readable): void;
|
|
289
|
+
/** Closes the turn of the user, and has the model answer it */
|
|
290
|
+
endTurn(): void;
|
|
291
|
+
/** Drops the turn of the user, without an answer */
|
|
292
|
+
discardTurn(): void;
|
|
293
|
+
/** Has the model speak without waiting for the user, to open a call */
|
|
294
|
+
respond(): void;
|
|
295
|
+
/**
|
|
296
|
+
* Has the model say this text, word for word.
|
|
297
|
+
*
|
|
298
|
+
* The caller records the text in the conversation, as with a text to speech,
|
|
299
|
+
* so the transcript of what the model says is not recorded again.
|
|
300
|
+
*/
|
|
301
|
+
say(text: string): void;
|
|
302
|
+
cancel(): void;
|
|
303
|
+
/**
|
|
304
|
+
* Adds a message the model did not hear, and tells the model about it.
|
|
305
|
+
*
|
|
306
|
+
* The transcripts of the call are recorded apart, so they are not sent back.
|
|
307
|
+
*/
|
|
308
|
+
addMessage(role: 'user' | 'assistant' | 'system', text: string, metadata?: MicdropAnswerMetadata): void;
|
|
309
|
+
addTool<Schema extends z.ZodObject>(tool: Tool<Schema>): void;
|
|
310
|
+
removeTool(name: string): void;
|
|
311
|
+
/**
|
|
312
|
+
* Lists the events that have listeners.
|
|
313
|
+
*
|
|
314
|
+
* Typed as the events of any agent, so a realtime model goes wherever an
|
|
315
|
+
* agent is expected. The two extra events do not change what that code reads.
|
|
316
|
+
*/
|
|
317
|
+
eventNames(): Array<keyof AgentEvents>;
|
|
318
|
+
destroy(): void;
|
|
319
|
+
protected generateAnswer(stream: PassThrough): Promise<void>;
|
|
320
|
+
/** Sends audio of the user, PCM16 at 16 kHz */
|
|
321
|
+
protected abstract appendAudio(chunk: Buffer): void;
|
|
322
|
+
/** Opens a turn of the user, right before its first audio */
|
|
323
|
+
protected abstract openTurn(): void;
|
|
324
|
+
/** Closes the turn of the user, which has the model answer it */
|
|
325
|
+
protected abstract closeTurn(): void;
|
|
326
|
+
/** Drops the audio of the open turn, without an answer */
|
|
327
|
+
protected abstract clearTurn(): void;
|
|
328
|
+
/** Has the model speak with no turn of the user to answer */
|
|
329
|
+
protected abstract generate(): void;
|
|
330
|
+
/** Has the model say this text, word for word */
|
|
331
|
+
protected abstract speakText(text: string): void;
|
|
332
|
+
/** Stops the answer the model is giving */
|
|
333
|
+
protected abstract cancelAnswer(): void;
|
|
334
|
+
/** Sends the tools to the model again, after one was added or removed */
|
|
335
|
+
protected abstract updateTools(): void;
|
|
336
|
+
/** Tells the model about a message it did not hear */
|
|
337
|
+
protected abstract injectMessage(role: 'user' | 'assistant' | 'system', text: string): void;
|
|
338
|
+
/** Nobody is speaking, nor about to, so the connection can be renewed */
|
|
339
|
+
protected get isIdle(): boolean;
|
|
340
|
+
/** Voice of the answer, PCM16 at 16 kHz */
|
|
341
|
+
protected emitAudio(chunk: Buffer): void;
|
|
342
|
+
/** Transcript of the answer, as it is spoken */
|
|
343
|
+
protected addAnswerTranscript(text: string): void;
|
|
344
|
+
/**
|
|
345
|
+
* The model finished its answer, or finished dropping an interrupted one.
|
|
346
|
+
*
|
|
347
|
+
* An answer without audio tells the client to stop waiting for one.
|
|
348
|
+
*/
|
|
349
|
+
protected endAnswer(): void;
|
|
350
|
+
/** Runs a tool the model called, and records it in the conversation */
|
|
351
|
+
protected runTool(toolCall: MicdropConversationToolCall): Promise<{
|
|
352
|
+
output: any;
|
|
353
|
+
skipAnswer: boolean | undefined;
|
|
354
|
+
} | {
|
|
355
|
+
output: {
|
|
356
|
+
error: any;
|
|
357
|
+
};
|
|
358
|
+
skipAnswer?: undefined;
|
|
359
|
+
}>;
|
|
360
|
+
/** Transcript of the turn of the user, possibly in several pieces */
|
|
361
|
+
protected addUserTranscript(text: string): void;
|
|
362
|
+
/** The transcript of the turn is complete, it goes in the conversation */
|
|
363
|
+
protected commitUserTranscript(): void;
|
|
364
|
+
private sendAudio;
|
|
365
|
+
private beginAnswer;
|
|
366
|
+
/**
|
|
367
|
+
* The answer output belongs to, if it is still wanted.
|
|
368
|
+
*
|
|
369
|
+
* The model can speak without being asked, to follow up on a tool call for
|
|
370
|
+
* instance, which opens an answer of its own.
|
|
371
|
+
*/
|
|
372
|
+
private activeAnswer;
|
|
373
|
+
private whenIdle;
|
|
374
|
+
private recordAnswer;
|
|
375
|
+
private awaitUserTranscript;
|
|
376
|
+
private clearUserTranscriptTimer;
|
|
377
|
+
private resolveAnswerWaiters;
|
|
378
|
+
}
|
|
379
|
+
|
|
245
380
|
interface STTEvents {
|
|
246
381
|
Transcript: [string];
|
|
247
382
|
Failed: [Buffer[]];
|
|
@@ -407,12 +542,12 @@ interface MicdropConfig {
|
|
|
407
542
|
agent?: Agent;
|
|
408
543
|
/**
|
|
409
544
|
* Turns what the user says into text, the one component a call cannot do
|
|
410
|
-
* without.
|
|
545
|
+
* without, unless a realtime model hears the user itself.
|
|
411
546
|
*
|
|
412
547
|
* It is fed the audio of each turn as it arrives, and the transcripts it
|
|
413
548
|
* returns drive everything that follows.
|
|
414
549
|
*/
|
|
415
|
-
stt
|
|
550
|
+
stt?: STT;
|
|
416
551
|
/**
|
|
417
552
|
* Gives the answers a voice.
|
|
418
553
|
*
|
|
@@ -440,6 +575,14 @@ interface MicdropConfig {
|
|
|
440
575
|
turnDetector?: TurnDetector;
|
|
441
576
|
/** How long to wait for the rest of a sentence, 4000 ms by default */
|
|
442
577
|
turnMaxWait?: number;
|
|
578
|
+
/**
|
|
579
|
+
* Hears the user and answers with its own voice, in place of `stt`, `agent`
|
|
580
|
+
* and `tts`, which are then ignored.
|
|
581
|
+
*
|
|
582
|
+
* The client does not change: it still detects when the user speaks, and the
|
|
583
|
+
* turn it sends goes to the model as audio.
|
|
584
|
+
*/
|
|
585
|
+
realtime?: Realtime;
|
|
443
586
|
}
|
|
444
587
|
declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
|
|
445
588
|
socket: WebSocket$1 | null;
|
|
@@ -457,6 +600,8 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
|
|
|
457
600
|
constructor(socket: WebSocket$1, config: MicdropConfig);
|
|
458
601
|
/** Everything said so far, whether an agent or the server keeps it */
|
|
459
602
|
get conversation(): MicdropConversation;
|
|
603
|
+
/** What holds the conversation and answers, a realtime model included */
|
|
604
|
+
private get agent();
|
|
460
605
|
private log;
|
|
461
606
|
private processQueue;
|
|
462
607
|
private queueOperation;
|
|
@@ -477,9 +622,11 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
|
|
|
477
622
|
* none leaves the call silent for good.
|
|
478
623
|
*/
|
|
479
624
|
private holdTurn;
|
|
625
|
+
/** Answers what the user just said */
|
|
626
|
+
private answerTurn;
|
|
480
627
|
private releaseHeldTurn;
|
|
481
628
|
private onTranscriptSTT;
|
|
482
|
-
private
|
|
629
|
+
private onAssistantAudio;
|
|
483
630
|
private onMessageAgent;
|
|
484
631
|
/** Adds a message to the conversation and sends it to the client */
|
|
485
632
|
addUserMessage(text: string, metadata?: MicdropAnswerMetadata): void;
|
|
@@ -528,6 +675,45 @@ declare class MicdropRecorder extends EventEmitter<MicdropRecorderEvents> {
|
|
|
528
675
|
protected log(...message: any[]): void;
|
|
529
676
|
}
|
|
530
677
|
|
|
678
|
+
/**
|
|
679
|
+
* Keeps one loaded instance per configuration, shared by every call.
|
|
680
|
+
*
|
|
681
|
+
* A local model costs from a few hundred milliseconds to a minute to load, and
|
|
682
|
+
* holding a second copy of the same weights wastes the memory the rest of the
|
|
683
|
+
* pipeline needs. Inference is bound to a few threads anyway, so two calls
|
|
684
|
+
* generating at once compete for the same cores whether they share the
|
|
685
|
+
* instance or not.
|
|
686
|
+
*
|
|
687
|
+
* Instances are keyed by the options they were built from, so changing a model
|
|
688
|
+
* or a device loads that one and leaves the others alone. They stay loaded for
|
|
689
|
+
* the lifetime of the process.
|
|
690
|
+
*
|
|
691
|
+
* ```typescript
|
|
692
|
+
* const synthesizers = new SharedInstances<Synthesizer>()
|
|
693
|
+
*
|
|
694
|
+
* function loadSynthesizer(options: Options) {
|
|
695
|
+
* return synthesizers.load([options.model, options.device], () =>
|
|
696
|
+
* Synthesizer.from_pretrained(options.model, { device: options.device })
|
|
697
|
+
* )
|
|
698
|
+
* }
|
|
699
|
+
* ```
|
|
700
|
+
*/
|
|
701
|
+
declare class SharedInstances<T> {
|
|
702
|
+
private entries;
|
|
703
|
+
/**
|
|
704
|
+
* Returns the instance for this key, calling `create` the first time.
|
|
705
|
+
*
|
|
706
|
+
* The key can be any JSON-serializable value, usually an array of the
|
|
707
|
+
* options that define the instance. Concurrent callers get the same promise
|
|
708
|
+
* instead of loading the model twice.
|
|
709
|
+
*/
|
|
710
|
+
load(key: unknown, create: () => Promise<T>): Promise<T>;
|
|
711
|
+
/** Whether an instance is loaded, or being loaded, for this key. */
|
|
712
|
+
has(key: unknown): boolean;
|
|
713
|
+
/** Drops every instance, so the next call loads again. */
|
|
714
|
+
clear(): void;
|
|
715
|
+
}
|
|
716
|
+
|
|
531
717
|
declare function waitForParams<CallParams>(socket: WebSocket$1, validate: (params: any) => CallParams): Promise<CallParams>;
|
|
532
718
|
|
|
533
|
-
export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, STT, type STTEvents, SentenceSplitter, SentenceTTS, TTS, type TTSEvents, type Tool, type TurnDetector, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };
|
|
719
|
+
export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, Realtime, type RealtimeEvents, type RealtimeOptions, STT, type STTEvents, SentenceSplitter, SentenceTTS, SharedInstances, TTS, type TTSEvents, type Tool, type TurnDetector, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };
|