@micdrop/server 2.5.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/dist/index.d.mts +99 -3
- package/dist/index.d.ts +99 -3
- package/dist/index.js +180 -56
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +180 -57
- package/dist/index.mjs.map +1 -1
- package/package.json +5 -3
package/README.md
CHANGED
|
@@ -64,9 +64,13 @@ wss.on('connection', (socket) => {
|
|
|
64
64
|
|
|
65
65
|
Micdrop server has 3 main components:
|
|
66
66
|
|
|
67
|
-
- `Agent` - AI agent using LLM
|
|
67
|
+
- `Agent` - AI agent using LLM, optional
|
|
68
68
|
- `STT` - Speech-to-text
|
|
69
|
-
- `TTS` - Text-to-speech
|
|
69
|
+
- `TTS` - Text-to-speech, optional
|
|
70
|
+
|
|
71
|
+
A server given a speech to text alone transcribes and stays quiet, which is all
|
|
72
|
+
a dictation tool needs. One given an agent and no voice answers in writing. See
|
|
73
|
+
[dictation and text-only calls](https://micdrop.dev/docs/server/dictation).
|
|
70
74
|
|
|
71
75
|
### Available implementations
|
|
72
76
|
|
package/dist/index.d.mts
CHANGED
|
@@ -16,6 +16,7 @@ declare enum MicdropClientCommands {
|
|
|
16
16
|
}
|
|
17
17
|
declare enum MicdropServerCommands {
|
|
18
18
|
Message = "Message",
|
|
19
|
+
PartialAssistantMessage = "PartialAssistantMessage",
|
|
19
20
|
CancelLastUserMessage = "CancelLastUserMessage",
|
|
20
21
|
SkipAnswer = "SkipAnswer",
|
|
21
22
|
EndCall = "EndCall",
|
|
@@ -376,15 +377,58 @@ declare abstract class SentenceTTS extends TTS {
|
|
|
376
377
|
|
|
377
378
|
interface MicdropServerEvents {
|
|
378
379
|
End: [MicdropCallSummary];
|
|
380
|
+
/** A message was added to the conversation, by the agent or by the server */
|
|
381
|
+
Message: [MicdropConversationItem];
|
|
379
382
|
UserAudio: [Buffer];
|
|
380
383
|
AssistantAudio: [Buffer];
|
|
381
384
|
}
|
|
382
385
|
interface MicdropConfig {
|
|
386
|
+
/**
|
|
387
|
+
* Opening line, spoken before the user has said anything.
|
|
388
|
+
*
|
|
389
|
+
* It goes into the conversation like any other assistant message. Leaving it
|
|
390
|
+
* out, along with `generateFirstMessage`, hands the first turn to the user.
|
|
391
|
+
*/
|
|
383
392
|
firstMessage?: string;
|
|
393
|
+
/**
|
|
394
|
+
* Asks the agent for its opening line instead of fixing one, so the greeting
|
|
395
|
+
* can lean on the system prompt and on whatever the call was given.
|
|
396
|
+
*
|
|
397
|
+
* Ignored when `firstMessage` is set, and out of reach without an agent.
|
|
398
|
+
*/
|
|
384
399
|
generateFirstMessage?: boolean;
|
|
385
|
-
|
|
400
|
+
/**
|
|
401
|
+
* Generates the answers.
|
|
402
|
+
*
|
|
403
|
+
* Leaving it out turns the call into a one-way one: what the user says is
|
|
404
|
+
* transcribed and sent to the client, and the assistant stays quiet unless
|
|
405
|
+
* the application calls `speak()` itself.
|
|
406
|
+
*/
|
|
407
|
+
agent?: Agent;
|
|
408
|
+
/**
|
|
409
|
+
* Turns what the user says into text, the one component a call cannot do
|
|
410
|
+
* without.
|
|
411
|
+
*
|
|
412
|
+
* It is fed the audio of each turn as it arrives, and the transcripts it
|
|
413
|
+
* returns drive everything that follows.
|
|
414
|
+
*/
|
|
386
415
|
stt: STT;
|
|
387
|
-
|
|
416
|
+
/**
|
|
417
|
+
* Gives the answers a voice.
|
|
418
|
+
*
|
|
419
|
+
* Leaving it out keeps the call textual: the answer reaches the client as a
|
|
420
|
+
* message and nothing is synthesized.
|
|
421
|
+
*/
|
|
422
|
+
tts?: TTS;
|
|
423
|
+
/**
|
|
424
|
+
* Sends the answer to the client while it is still being written, on top of
|
|
425
|
+
* the complete message that follows.
|
|
426
|
+
*
|
|
427
|
+
* Works with any agent, since the text is read from the stream it writes to.
|
|
428
|
+
* Off by default, since a client that only displays finished messages has
|
|
429
|
+
* nothing to do with it.
|
|
430
|
+
*/
|
|
431
|
+
partialMessages?: boolean;
|
|
388
432
|
/**
|
|
389
433
|
* Waits for the rest of the sentence when the speaker paused in the middle
|
|
390
434
|
* of one, instead of answering an unfinished question.
|
|
@@ -403,6 +447,7 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
|
|
|
403
447
|
logger?: Logger;
|
|
404
448
|
private startTime;
|
|
405
449
|
private lastMessageSpeeched?;
|
|
450
|
+
private ownConversation;
|
|
406
451
|
private operationQueue;
|
|
407
452
|
private isProcessingQueue;
|
|
408
453
|
private currentUserStream?;
|
|
@@ -410,6 +455,8 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
|
|
|
410
455
|
private turnComplete?;
|
|
411
456
|
private heldTurnTimer?;
|
|
412
457
|
constructor(socket: WebSocket$1, config: MicdropConfig);
|
|
458
|
+
/** Everything said so far, whether an agent or the server keeps it */
|
|
459
|
+
get conversation(): MicdropConversation;
|
|
413
460
|
private log;
|
|
414
461
|
private processQueue;
|
|
415
462
|
private queueOperation;
|
|
@@ -433,9 +480,19 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
|
|
|
433
480
|
private releaseHeldTurn;
|
|
434
481
|
private onTranscriptSTT;
|
|
435
482
|
private onAudioTTS;
|
|
483
|
+
private onMessageAgent;
|
|
484
|
+
/** Adds a message to the conversation and sends it to the client */
|
|
485
|
+
addUserMessage(text: string, metadata?: MicdropAnswerMetadata): void;
|
|
486
|
+
/** Adds a message to the conversation and sends it to the client */
|
|
487
|
+
addAssistantMessage(text: string, metadata?: MicdropAnswerMetadata): void;
|
|
488
|
+
private addOwnMessage;
|
|
489
|
+
private sendMessage;
|
|
490
|
+
private sendPartialAnswer;
|
|
436
491
|
private sendFirstMessage;
|
|
437
492
|
answer(): void;
|
|
438
493
|
private _answer;
|
|
494
|
+
/** Sends the answer to the client as it is written, ahead of the voice */
|
|
495
|
+
private forwardPartialAnswer;
|
|
439
496
|
speak(message: string | Readable): void;
|
|
440
497
|
private _speak;
|
|
441
498
|
}
|
|
@@ -471,6 +528,45 @@ declare class MicdropRecorder extends EventEmitter<MicdropRecorderEvents> {
|
|
|
471
528
|
protected log(...message: any[]): void;
|
|
472
529
|
}
|
|
473
530
|
|
|
531
|
+
/**
|
|
532
|
+
* Keeps one loaded instance per configuration, shared by every call.
|
|
533
|
+
*
|
|
534
|
+
* A local model costs from a few hundred milliseconds to a minute to load, and
|
|
535
|
+
* holding a second copy of the same weights wastes the memory the rest of the
|
|
536
|
+
* pipeline needs. Inference is bound to a few threads anyway, so two calls
|
|
537
|
+
* generating at once compete for the same cores whether they share the
|
|
538
|
+
* instance or not.
|
|
539
|
+
*
|
|
540
|
+
* Instances are keyed by the options they were built from, so changing a model
|
|
541
|
+
* or a device loads that one and leaves the others alone. They stay loaded for
|
|
542
|
+
* the lifetime of the process.
|
|
543
|
+
*
|
|
544
|
+
* ```typescript
|
|
545
|
+
* const synthesizers = new SharedInstances<Synthesizer>()
|
|
546
|
+
*
|
|
547
|
+
* function loadSynthesizer(options: Options) {
|
|
548
|
+
* return synthesizers.load([options.model, options.device], () =>
|
|
549
|
+
* Synthesizer.from_pretrained(options.model, { device: options.device })
|
|
550
|
+
* )
|
|
551
|
+
* }
|
|
552
|
+
* ```
|
|
553
|
+
*/
|
|
554
|
+
declare class SharedInstances<T> {
|
|
555
|
+
private entries;
|
|
556
|
+
/**
|
|
557
|
+
* Returns the instance for this key, calling `create` the first time.
|
|
558
|
+
*
|
|
559
|
+
* The key can be any JSON-serializable value, usually an array of the
|
|
560
|
+
* options that define the instance. Concurrent callers get the same promise
|
|
561
|
+
* instead of loading the model twice.
|
|
562
|
+
*/
|
|
563
|
+
load(key: unknown, create: () => Promise<T>): Promise<T>;
|
|
564
|
+
/** Whether an instance is loaded, or being loaded, for this key. */
|
|
565
|
+
has(key: unknown): boolean;
|
|
566
|
+
/** Drops every instance, so the next call loads again. */
|
|
567
|
+
clear(): void;
|
|
568
|
+
}
|
|
569
|
+
|
|
474
570
|
declare function waitForParams<CallParams>(socket: WebSocket$1, validate: (params: any) => CallParams): Promise<CallParams>;
|
|
475
571
|
|
|
476
|
-
export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, STT, type STTEvents, SentenceSplitter, SentenceTTS, TTS, type TTSEvents, type Tool, type TurnDetector, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };
|
|
572
|
+
export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, STT, type STTEvents, SentenceSplitter, SentenceTTS, SharedInstances, TTS, type TTSEvents, type Tool, type TurnDetector, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };
|
package/dist/index.d.ts
CHANGED
|
@@ -16,6 +16,7 @@ declare enum MicdropClientCommands {
|
|
|
16
16
|
}
|
|
17
17
|
declare enum MicdropServerCommands {
|
|
18
18
|
Message = "Message",
|
|
19
|
+
PartialAssistantMessage = "PartialAssistantMessage",
|
|
19
20
|
CancelLastUserMessage = "CancelLastUserMessage",
|
|
20
21
|
SkipAnswer = "SkipAnswer",
|
|
21
22
|
EndCall = "EndCall",
|
|
@@ -376,15 +377,58 @@ declare abstract class SentenceTTS extends TTS {
|
|
|
376
377
|
|
|
377
378
|
interface MicdropServerEvents {
|
|
378
379
|
End: [MicdropCallSummary];
|
|
380
|
+
/** A message was added to the conversation, by the agent or by the server */
|
|
381
|
+
Message: [MicdropConversationItem];
|
|
379
382
|
UserAudio: [Buffer];
|
|
380
383
|
AssistantAudio: [Buffer];
|
|
381
384
|
}
|
|
382
385
|
interface MicdropConfig {
|
|
386
|
+
/**
|
|
387
|
+
* Opening line, spoken before the user has said anything.
|
|
388
|
+
*
|
|
389
|
+
* It goes into the conversation like any other assistant message. Leaving it
|
|
390
|
+
* out, along with `generateFirstMessage`, hands the first turn to the user.
|
|
391
|
+
*/
|
|
383
392
|
firstMessage?: string;
|
|
393
|
+
/**
|
|
394
|
+
* Asks the agent for its opening line instead of fixing one, so the greeting
|
|
395
|
+
* can lean on the system prompt and on whatever the call was given.
|
|
396
|
+
*
|
|
397
|
+
* Ignored when `firstMessage` is set, and out of reach without an agent.
|
|
398
|
+
*/
|
|
384
399
|
generateFirstMessage?: boolean;
|
|
385
|
-
|
|
400
|
+
/**
|
|
401
|
+
* Generates the answers.
|
|
402
|
+
*
|
|
403
|
+
* Leaving it out turns the call into a one-way one: what the user says is
|
|
404
|
+
* transcribed and sent to the client, and the assistant stays quiet unless
|
|
405
|
+
* the application calls `speak()` itself.
|
|
406
|
+
*/
|
|
407
|
+
agent?: Agent;
|
|
408
|
+
/**
|
|
409
|
+
* Turns what the user says into text, the one component a call cannot do
|
|
410
|
+
* without.
|
|
411
|
+
*
|
|
412
|
+
* It is fed the audio of each turn as it arrives, and the transcripts it
|
|
413
|
+
* returns drive everything that follows.
|
|
414
|
+
*/
|
|
386
415
|
stt: STT;
|
|
387
|
-
|
|
416
|
+
/**
|
|
417
|
+
* Gives the answers a voice.
|
|
418
|
+
*
|
|
419
|
+
* Leaving it out keeps the call textual: the answer reaches the client as a
|
|
420
|
+
* message and nothing is synthesized.
|
|
421
|
+
*/
|
|
422
|
+
tts?: TTS;
|
|
423
|
+
/**
|
|
424
|
+
* Sends the answer to the client while it is still being written, on top of
|
|
425
|
+
* the complete message that follows.
|
|
426
|
+
*
|
|
427
|
+
* Works with any agent, since the text is read from the stream it writes to.
|
|
428
|
+
* Off by default, since a client that only displays finished messages has
|
|
429
|
+
* nothing to do with it.
|
|
430
|
+
*/
|
|
431
|
+
partialMessages?: boolean;
|
|
388
432
|
/**
|
|
389
433
|
* Waits for the rest of the sentence when the speaker paused in the middle
|
|
390
434
|
* of one, instead of answering an unfinished question.
|
|
@@ -403,6 +447,7 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
|
|
|
403
447
|
logger?: Logger;
|
|
404
448
|
private startTime;
|
|
405
449
|
private lastMessageSpeeched?;
|
|
450
|
+
private ownConversation;
|
|
406
451
|
private operationQueue;
|
|
407
452
|
private isProcessingQueue;
|
|
408
453
|
private currentUserStream?;
|
|
@@ -410,6 +455,8 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
|
|
|
410
455
|
private turnComplete?;
|
|
411
456
|
private heldTurnTimer?;
|
|
412
457
|
constructor(socket: WebSocket$1, config: MicdropConfig);
|
|
458
|
+
/** Everything said so far, whether an agent or the server keeps it */
|
|
459
|
+
get conversation(): MicdropConversation;
|
|
413
460
|
private log;
|
|
414
461
|
private processQueue;
|
|
415
462
|
private queueOperation;
|
|
@@ -433,9 +480,19 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
|
|
|
433
480
|
private releaseHeldTurn;
|
|
434
481
|
private onTranscriptSTT;
|
|
435
482
|
private onAudioTTS;
|
|
483
|
+
private onMessageAgent;
|
|
484
|
+
/** Adds a message to the conversation and sends it to the client */
|
|
485
|
+
addUserMessage(text: string, metadata?: MicdropAnswerMetadata): void;
|
|
486
|
+
/** Adds a message to the conversation and sends it to the client */
|
|
487
|
+
addAssistantMessage(text: string, metadata?: MicdropAnswerMetadata): void;
|
|
488
|
+
private addOwnMessage;
|
|
489
|
+
private sendMessage;
|
|
490
|
+
private sendPartialAnswer;
|
|
436
491
|
private sendFirstMessage;
|
|
437
492
|
answer(): void;
|
|
438
493
|
private _answer;
|
|
494
|
+
/** Sends the answer to the client as it is written, ahead of the voice */
|
|
495
|
+
private forwardPartialAnswer;
|
|
439
496
|
speak(message: string | Readable): void;
|
|
440
497
|
private _speak;
|
|
441
498
|
}
|
|
@@ -471,6 +528,45 @@ declare class MicdropRecorder extends EventEmitter<MicdropRecorderEvents> {
|
|
|
471
528
|
protected log(...message: any[]): void;
|
|
472
529
|
}
|
|
473
530
|
|
|
531
|
+
/**
|
|
532
|
+
* Keeps one loaded instance per configuration, shared by every call.
|
|
533
|
+
*
|
|
534
|
+
* A local model costs from a few hundred milliseconds to a minute to load, and
|
|
535
|
+
* holding a second copy of the same weights wastes the memory the rest of the
|
|
536
|
+
* pipeline needs. Inference is bound to a few threads anyway, so two calls
|
|
537
|
+
* generating at once compete for the same cores whether they share the
|
|
538
|
+
* instance or not.
|
|
539
|
+
*
|
|
540
|
+
* Instances are keyed by the options they were built from, so changing a model
|
|
541
|
+
* or a device loads that one and leaves the others alone. They stay loaded for
|
|
542
|
+
* the lifetime of the process.
|
|
543
|
+
*
|
|
544
|
+
* ```typescript
|
|
545
|
+
* const synthesizers = new SharedInstances<Synthesizer>()
|
|
546
|
+
*
|
|
547
|
+
* function loadSynthesizer(options: Options) {
|
|
548
|
+
* return synthesizers.load([options.model, options.device], () =>
|
|
549
|
+
* Synthesizer.from_pretrained(options.model, { device: options.device })
|
|
550
|
+
* )
|
|
551
|
+
* }
|
|
552
|
+
* ```
|
|
553
|
+
*/
|
|
554
|
+
declare class SharedInstances<T> {
|
|
555
|
+
private entries;
|
|
556
|
+
/**
|
|
557
|
+
* Returns the instance for this key, calling `create` the first time.
|
|
558
|
+
*
|
|
559
|
+
* The key can be any JSON-serializable value, usually an array of the
|
|
560
|
+
* options that define the instance. Concurrent callers get the same promise
|
|
561
|
+
* instead of loading the model twice.
|
|
562
|
+
*/
|
|
563
|
+
load(key: unknown, create: () => Promise<T>): Promise<T>;
|
|
564
|
+
/** Whether an instance is loaded, or being loaded, for this key. */
|
|
565
|
+
has(key: unknown): boolean;
|
|
566
|
+
/** Drops every instance, so the next call loads again. */
|
|
567
|
+
clear(): void;
|
|
568
|
+
}
|
|
569
|
+
|
|
474
570
|
declare function waitForParams<CallParams>(socket: WebSocket$1, validate: (params: any) => CallParams): Promise<CallParams>;
|
|
475
571
|
|
|
476
|
-
export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, STT, type STTEvents, SentenceSplitter, SentenceTTS, TTS, type TTSEvents, type Tool, type TurnDetector, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };
|
|
572
|
+
export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, STT, type STTEvents, SentenceSplitter, SentenceTTS, SharedInstances, TTS, type TTSEvents, type Tool, type TurnDetector, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };
|