@micdrop/server 2.5.0 → 2.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -64,9 +64,13 @@ wss.on('connection', (socket) => {
64
64
 
65
65
  Micdrop server has 3 main components:
66
66
 
67
- - `Agent` - AI agent using LLM
67
+ - `Agent` - AI agent using LLM, optional
68
68
  - `STT` - Speech-to-text
69
- - `TTS` - Text-to-speech
69
+ - `TTS` - Text-to-speech, optional
70
+
71
+ A server given a speech to text alone transcribes and stays quiet, which is all
72
+ a dictation tool needs. One given an agent and no voice answers in writing. See
73
+ [dictation and text-only calls](https://micdrop.dev/docs/server/dictation).
70
74
 
71
75
  ### Available implementations
72
76
 
package/dist/index.d.mts CHANGED
@@ -16,6 +16,7 @@ declare enum MicdropClientCommands {
16
16
  }
17
17
  declare enum MicdropServerCommands {
18
18
  Message = "Message",
19
+ PartialAssistantMessage = "PartialAssistantMessage",
19
20
  CancelLastUserMessage = "CancelLastUserMessage",
20
21
  SkipAnswer = "SkipAnswer",
21
22
  EndCall = "EndCall",
@@ -376,15 +377,58 @@ declare abstract class SentenceTTS extends TTS {
376
377
 
377
378
  interface MicdropServerEvents {
378
379
  End: [MicdropCallSummary];
380
+ /** A message was added to the conversation, by the agent or by the server */
381
+ Message: [MicdropConversationItem];
379
382
  UserAudio: [Buffer];
380
383
  AssistantAudio: [Buffer];
381
384
  }
382
385
  interface MicdropConfig {
386
+ /**
387
+ * Opening line, spoken before the user has said anything.
388
+ *
389
+ * It goes into the conversation like any other assistant message. Leaving it
390
+ * out, along with `generateFirstMessage`, hands the first turn to the user.
391
+ */
383
392
  firstMessage?: string;
393
+ /**
394
+ * Asks the agent for its opening line instead of fixing one, so the greeting
395
+ * can lean on the system prompt and on whatever the call was given.
396
+ *
397
+ * Ignored when `firstMessage` is set, and out of reach without an agent.
398
+ */
384
399
  generateFirstMessage?: boolean;
385
- agent: Agent;
400
+ /**
401
+ * Generates the answers.
402
+ *
403
+ * Leaving it out turns the call into a one-way one: what the user says is
404
+ * transcribed and sent to the client, and the assistant stays quiet unless
405
+ * the application calls `speak()` itself.
406
+ */
407
+ agent?: Agent;
408
+ /**
409
+ * Turns what the user says into text, the one component a call cannot do
410
+ * without.
411
+ *
412
+ * It is fed the audio of each turn as it arrives, and the transcripts it
413
+ * returns drive everything that follows.
414
+ */
386
415
  stt: STT;
387
- tts: TTS;
416
+ /**
417
+ * Gives the answers a voice.
418
+ *
419
+ * Leaving it out keeps the call textual: the answer reaches the client as a
420
+ * message and nothing is synthesized.
421
+ */
422
+ tts?: TTS;
423
+ /**
424
+ * Sends the answer to the client while it is still being written, on top of
425
+ * the complete message that follows.
426
+ *
427
+ * Works with any agent, since the text is read from the stream it writes to.
428
+ * Off by default, since a client that only displays finished messages has
429
+ * nothing to do with it.
430
+ */
431
+ partialMessages?: boolean;
388
432
  /**
389
433
  * Waits for the rest of the sentence when the speaker paused in the middle
390
434
  * of one, instead of answering an unfinished question.
@@ -403,6 +447,7 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
403
447
  logger?: Logger;
404
448
  private startTime;
405
449
  private lastMessageSpeeched?;
450
+ private ownConversation;
406
451
  private operationQueue;
407
452
  private isProcessingQueue;
408
453
  private currentUserStream?;
@@ -410,6 +455,8 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
410
455
  private turnComplete?;
411
456
  private heldTurnTimer?;
412
457
  constructor(socket: WebSocket$1, config: MicdropConfig);
458
+ /** Everything said so far, whether an agent or the server keeps it */
459
+ get conversation(): MicdropConversation;
413
460
  private log;
414
461
  private processQueue;
415
462
  private queueOperation;
@@ -433,9 +480,19 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
433
480
  private releaseHeldTurn;
434
481
  private onTranscriptSTT;
435
482
  private onAudioTTS;
483
+ private onMessageAgent;
484
+ /** Adds a message to the conversation and sends it to the client */
485
+ addUserMessage(text: string, metadata?: MicdropAnswerMetadata): void;
486
+ /** Adds a message to the conversation and sends it to the client */
487
+ addAssistantMessage(text: string, metadata?: MicdropAnswerMetadata): void;
488
+ private addOwnMessage;
489
+ private sendMessage;
490
+ private sendPartialAnswer;
436
491
  private sendFirstMessage;
437
492
  answer(): void;
438
493
  private _answer;
494
+ /** Sends the answer to the client as it is written, ahead of the voice */
495
+ private forwardPartialAnswer;
439
496
  speak(message: string | Readable): void;
440
497
  private _speak;
441
498
  }
@@ -471,6 +528,45 @@ declare class MicdropRecorder extends EventEmitter<MicdropRecorderEvents> {
471
528
  protected log(...message: any[]): void;
472
529
  }
473
530
 
531
+ /**
532
+ * Keeps one loaded instance per configuration, shared by every call.
533
+ *
534
+ * A local model costs from a few hundred milliseconds to a minute to load, and
535
+ * holding a second copy of the same weights wastes the memory the rest of the
536
+ * pipeline needs. Inference is bound to a few threads anyway, so two calls
537
+ * generating at once compete for the same cores whether they share the
538
+ * instance or not.
539
+ *
540
+ * Instances are keyed by the options they were built from, so changing a model
541
+ * or a device loads that one and leaves the others alone. They stay loaded for
542
+ * the lifetime of the process.
543
+ *
544
+ * ```typescript
545
+ * const synthesizers = new SharedInstances<Synthesizer>()
546
+ *
547
+ * function loadSynthesizer(options: Options) {
548
+ * return synthesizers.load([options.model, options.device], () =>
549
+ * Synthesizer.from_pretrained(options.model, { device: options.device })
550
+ * )
551
+ * }
552
+ * ```
553
+ */
554
+ declare class SharedInstances<T> {
555
+ private entries;
556
+ /**
557
+ * Returns the instance for this key, calling `create` the first time.
558
+ *
559
+ * The key can be any JSON-serializable value, usually an array of the
560
+ * options that define the instance. Concurrent callers get the same promise
561
+ * instead of loading the model twice.
562
+ */
563
+ load(key: unknown, create: () => Promise<T>): Promise<T>;
564
+ /** Whether an instance is loaded, or being loaded, for this key. */
565
+ has(key: unknown): boolean;
566
+ /** Drops every instance, so the next call loads again. */
567
+ clear(): void;
568
+ }
569
+
474
570
  declare function waitForParams<CallParams>(socket: WebSocket$1, validate: (params: any) => CallParams): Promise<CallParams>;
475
571
 
476
- export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, STT, type STTEvents, SentenceSplitter, SentenceTTS, TTS, type TTSEvents, type Tool, type TurnDetector, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };
572
+ export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, STT, type STTEvents, SentenceSplitter, SentenceTTS, SharedInstances, TTS, type TTSEvents, type Tool, type TurnDetector, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };
package/dist/index.d.ts CHANGED
@@ -16,6 +16,7 @@ declare enum MicdropClientCommands {
16
16
  }
17
17
  declare enum MicdropServerCommands {
18
18
  Message = "Message",
19
+ PartialAssistantMessage = "PartialAssistantMessage",
19
20
  CancelLastUserMessage = "CancelLastUserMessage",
20
21
  SkipAnswer = "SkipAnswer",
21
22
  EndCall = "EndCall",
@@ -376,15 +377,58 @@ declare abstract class SentenceTTS extends TTS {
376
377
 
377
378
  interface MicdropServerEvents {
378
379
  End: [MicdropCallSummary];
380
+ /** A message was added to the conversation, by the agent or by the server */
381
+ Message: [MicdropConversationItem];
379
382
  UserAudio: [Buffer];
380
383
  AssistantAudio: [Buffer];
381
384
  }
382
385
  interface MicdropConfig {
386
+ /**
387
+ * Opening line, spoken before the user has said anything.
388
+ *
389
+ * It goes into the conversation like any other assistant message. Leaving it
390
+ * out, along with `generateFirstMessage`, hands the first turn to the user.
391
+ */
383
392
  firstMessage?: string;
393
+ /**
394
+ * Asks the agent for its opening line instead of fixing one, so the greeting
395
+ * can lean on the system prompt and on whatever the call was given.
396
+ *
397
+ * Ignored when `firstMessage` is set, and out of reach without an agent.
398
+ */
384
399
  generateFirstMessage?: boolean;
385
- agent: Agent;
400
+ /**
401
+ * Generates the answers.
402
+ *
403
+ * Leaving it out turns the call into a one-way one: what the user says is
404
+ * transcribed and sent to the client, and the assistant stays quiet unless
405
+ * the application calls `speak()` itself.
406
+ */
407
+ agent?: Agent;
408
+ /**
409
+ * Turns what the user says into text, the one component a call cannot do
410
+ * without.
411
+ *
412
+ * It is fed the audio of each turn as it arrives, and the transcripts it
413
+ * returns drive everything that follows.
414
+ */
386
415
  stt: STT;
387
- tts: TTS;
416
+ /**
417
+ * Gives the answers a voice.
418
+ *
419
+ * Leaving it out keeps the call textual: the answer reaches the client as a
420
+ * message and nothing is synthesized.
421
+ */
422
+ tts?: TTS;
423
+ /**
424
+ * Sends the answer to the client while it is still being written, on top of
425
+ * the complete message that follows.
426
+ *
427
+ * Works with any agent, since the text is read from the stream it writes to.
428
+ * Off by default, since a client that only displays finished messages has
429
+ * nothing to do with it.
430
+ */
431
+ partialMessages?: boolean;
388
432
  /**
389
433
  * Waits for the rest of the sentence when the speaker paused in the middle
390
434
  * of one, instead of answering an unfinished question.
@@ -403,6 +447,7 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
403
447
  logger?: Logger;
404
448
  private startTime;
405
449
  private lastMessageSpeeched?;
450
+ private ownConversation;
406
451
  private operationQueue;
407
452
  private isProcessingQueue;
408
453
  private currentUserStream?;
@@ -410,6 +455,8 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
410
455
  private turnComplete?;
411
456
  private heldTurnTimer?;
412
457
  constructor(socket: WebSocket$1, config: MicdropConfig);
458
+ /** Everything said so far, whether an agent or the server keeps it */
459
+ get conversation(): MicdropConversation;
413
460
  private log;
414
461
  private processQueue;
415
462
  private queueOperation;
@@ -433,9 +480,19 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
433
480
  private releaseHeldTurn;
434
481
  private onTranscriptSTT;
435
482
  private onAudioTTS;
483
+ private onMessageAgent;
484
+ /** Adds a message to the conversation and sends it to the client */
485
+ addUserMessage(text: string, metadata?: MicdropAnswerMetadata): void;
486
+ /** Adds a message to the conversation and sends it to the client */
487
+ addAssistantMessage(text: string, metadata?: MicdropAnswerMetadata): void;
488
+ private addOwnMessage;
489
+ private sendMessage;
490
+ private sendPartialAnswer;
436
491
  private sendFirstMessage;
437
492
  answer(): void;
438
493
  private _answer;
494
+ /** Sends the answer to the client as it is written, ahead of the voice */
495
+ private forwardPartialAnswer;
439
496
  speak(message: string | Readable): void;
440
497
  private _speak;
441
498
  }
@@ -471,6 +528,45 @@ declare class MicdropRecorder extends EventEmitter<MicdropRecorderEvents> {
471
528
  protected log(...message: any[]): void;
472
529
  }
473
530
 
531
+ /**
532
+ * Keeps one loaded instance per configuration, shared by every call.
533
+ *
534
+ * A local model costs from a few hundred milliseconds to a minute to load, and
535
+ * holding a second copy of the same weights wastes the memory the rest of the
536
+ * pipeline needs. Inference is bound to a few threads anyway, so two calls
537
+ * generating at once compete for the same cores whether they share the
538
+ * instance or not.
539
+ *
540
+ * Instances are keyed by the options they were built from, so changing a model
541
+ * or a device loads that one and leaves the others alone. They stay loaded for
542
+ * the lifetime of the process.
543
+ *
544
+ * ```typescript
545
+ * const synthesizers = new SharedInstances<Synthesizer>()
546
+ *
547
+ * function loadSynthesizer(options: Options) {
548
+ * return synthesizers.load([options.model, options.device], () =>
549
+ * Synthesizer.from_pretrained(options.model, { device: options.device })
550
+ * )
551
+ * }
552
+ * ```
553
+ */
554
+ declare class SharedInstances<T> {
555
+ private entries;
556
+ /**
557
+ * Returns the instance for this key, calling `create` the first time.
558
+ *
559
+ * The key can be any JSON-serializable value, usually an array of the
560
+ * options that define the instance. Concurrent callers get the same promise
561
+ * instead of loading the model twice.
562
+ */
563
+ load(key: unknown, create: () => Promise<T>): Promise<T>;
564
+ /** Whether an instance is loaded, or being loaded, for this key. */
565
+ has(key: unknown): boolean;
566
+ /** Drops every instance, so the next call loads again. */
567
+ clear(): void;
568
+ }
569
+
474
570
  declare function waitForParams<CallParams>(socket: WebSocket$1, validate: (params: any) => CallParams): Promise<CallParams>;
475
571
 
476
- export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, STT, type STTEvents, SentenceSplitter, SentenceTTS, TTS, type TTSEvents, type Tool, type TurnDetector, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };
572
+ export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, STT, type STTEvents, SentenceSplitter, SentenceTTS, SharedInstances, TTS, type TTSEvents, type Tool, type TurnDetector, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };