@micdrop/server 2.4.0 → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -64,9 +64,13 @@ wss.on('connection', (socket) => {
64
64
 
65
65
  Micdrop server has 3 main components:
66
66
 
67
- - `Agent` - AI agent using LLM
67
+ - `Agent` - AI agent using LLM, optional
68
68
  - `STT` - Speech-to-text
69
- - `TTS` - Text-to-speech
69
+ - `TTS` - Text-to-speech, optional
70
+
71
+ A server given a speech to text alone transcribes and stays quiet, which is all
72
+ a dictation tool needs. One given an agent and no voice answers in writing. See
73
+ [dictation and text-only calls](https://micdrop.dev/docs/server/dictation).
70
74
 
71
75
  ### Available implementations
72
76
 
package/dist/index.d.mts CHANGED
@@ -16,11 +16,34 @@ declare enum MicdropClientCommands {
16
16
  }
17
17
  declare enum MicdropServerCommands {
18
18
  Message = "Message",
19
+ PartialAssistantMessage = "PartialAssistantMessage",
19
20
  CancelLastUserMessage = "CancelLastUserMessage",
20
21
  SkipAnswer = "SkipAnswer",
21
22
  EndCall = "EndCall",
22
23
  ToolCall = "ToolCall"
23
24
  }
25
+ /**
26
+ * Hears whether a sentence has landed, where voice activity detection only
27
+ * hears whether someone is speaking.
28
+ *
29
+ * `SmartTurn` from `@micdrop/smart-turn` implements it, and so can anything
30
+ * else, a call to a service included. Both sides of a call can hold one, the
31
+ * client to decide when its turn ends and the server to decide when to answer.
32
+ */
33
+ interface TurnDetector {
34
+ /**
35
+ * Feeds the audio received since the last call
36
+ * @param samples - Mono samples, in the -1..1 range
37
+ * @param sampleRate - Sample rate of `samples`, in Hz
38
+ */
39
+ push(samples: Float32Array, sampleRate?: number): void;
40
+ /** Answers whether the turn pushed so far sounds finished */
41
+ predict(): Promise<{
42
+ complete: boolean;
43
+ }>;
44
+ /** Starts a new turn, forgetting the previous one */
45
+ reset(): void;
46
+ }
24
47
  interface MicdropCallSummary {
25
48
  conversation: MicdropConversation;
26
49
  duration: number;
@@ -354,15 +377,69 @@ declare abstract class SentenceTTS extends TTS {
354
377
 
355
378
  interface MicdropServerEvents {
356
379
  End: [MicdropCallSummary];
380
+ /** A message was added to the conversation, by the agent or by the server */
381
+ Message: [MicdropConversationItem];
357
382
  UserAudio: [Buffer];
358
383
  AssistantAudio: [Buffer];
359
384
  }
360
385
  interface MicdropConfig {
386
+ /**
387
+ * Opening line, spoken before the user has said anything.
388
+ *
389
+ * It goes into the conversation like any other assistant message. Leaving it
390
+ * out, along with `generateFirstMessage`, hands the first turn to the user.
391
+ */
361
392
  firstMessage?: string;
393
+ /**
394
+ * Asks the agent for its opening line instead of fixing one, so the greeting
395
+ * can lean on the system prompt and on whatever the call was given.
396
+ *
397
+ * Ignored when `firstMessage` is set, and out of reach without an agent.
398
+ */
362
399
  generateFirstMessage?: boolean;
363
- agent: Agent;
400
+ /**
401
+ * Generates the answers.
402
+ *
403
+ * Leaving it out turns the call into a one-way one: what the user says is
404
+ * transcribed and sent to the client, and the assistant stays quiet unless
405
+ * the application calls `speak()` itself.
406
+ */
407
+ agent?: Agent;
408
+ /**
409
+ * Turns what the user says into text, the one component a call cannot do
410
+ * without.
411
+ *
412
+ * It is fed the audio of each turn as it arrives, and the transcripts it
413
+ * returns drive everything that follows.
414
+ */
364
415
  stt: STT;
365
- tts: TTS;
416
+ /**
417
+ * Gives the answers a voice.
418
+ *
419
+ * Leaving it out keeps the call textual: the answer reaches the client as a
420
+ * message and nothing is synthesized.
421
+ */
422
+ tts?: TTS;
423
+ /**
424
+ * Sends the answer to the client while it is still being written, on top of
425
+ * the complete message that follows.
426
+ *
427
+ * Works with any agent, since the text is read from the stream it writes to.
428
+ * Off by default, since a client that only displays finished messages has
429
+ * nothing to do with it.
430
+ */
431
+ partialMessages?: boolean;
432
+ /**
433
+ * Waits for the rest of the sentence when the speaker paused in the middle
434
+ * of one, instead of answering an unfinished question.
435
+ *
436
+ * Prefer running the detector in the client, which reaches the same decision
437
+ * without the round trip and can then close its turns sooner. This is the
438
+ * option for the browsers where the model has nowhere to run.
439
+ */
440
+ turnDetector?: TurnDetector;
441
+ /** How long to wait for the rest of a sentence, 4000 ms by default */
442
+ turnMaxWait?: number;
366
443
  }
367
444
  declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
368
445
  socket: WebSocket$1 | null;
@@ -370,11 +447,16 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
370
447
  logger?: Logger;
371
448
  private startTime;
372
449
  private lastMessageSpeeched?;
450
+ private ownConversation;
373
451
  private operationQueue;
374
452
  private isProcessingQueue;
375
453
  private currentUserStream?;
376
454
  private userSpeechChunks;
455
+ private turnComplete?;
456
+ private heldTurnTimer?;
377
457
  constructor(socket: WebSocket$1, config: MicdropConfig);
458
+ /** Everything said so far, whether an agent or the server keeps it */
459
+ get conversation(): MicdropConversation;
378
460
  private log;
379
461
  private processQueue;
380
462
  private queueOperation;
@@ -385,11 +467,32 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
385
467
  private onMute;
386
468
  private onStartSpeaking;
387
469
  private onStopSpeaking;
470
+ private predictTurnComplete;
471
+ /** Answers, unless the sentence sounds like it has more coming */
472
+ private answerUserTurn;
473
+ /**
474
+ * Answers anyway if the rest of the sentence never comes.
475
+ *
476
+ * Without it, a detector that hears an unfinished sentence where there is
477
+ * none leaves the call silent for good.
478
+ */
479
+ private holdTurn;
480
+ private releaseHeldTurn;
388
481
  private onTranscriptSTT;
389
482
  private onAudioTTS;
483
+ private onMessageAgent;
484
+ /** Adds a message to the conversation and sends it to the client */
485
+ addUserMessage(text: string, metadata?: MicdropAnswerMetadata): void;
486
+ /** Adds a message to the conversation and sends it to the client */
487
+ addAssistantMessage(text: string, metadata?: MicdropAnswerMetadata): void;
488
+ private addOwnMessage;
489
+ private sendMessage;
490
+ private sendPartialAnswer;
390
491
  private sendFirstMessage;
391
492
  answer(): void;
392
493
  private _answer;
494
+ /** Sends the answer to the client as it is written, ahead of the voice */
495
+ private forwardPartialAnswer;
393
496
  speak(message: string | Readable): void;
394
497
  private _speak;
395
498
  }
@@ -427,4 +530,4 @@ declare class MicdropRecorder extends EventEmitter<MicdropRecorderEvents> {
427
530
 
428
531
  declare function waitForParams<CallParams>(socket: WebSocket$1, validate: (params: any) => CallParams): Promise<CallParams>;
429
532
 
430
- export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, STT, type STTEvents, SentenceSplitter, SentenceTTS, TTS, type TTSEvents, type Tool, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };
533
+ export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, STT, type STTEvents, SentenceSplitter, SentenceTTS, TTS, type TTSEvents, type Tool, type TurnDetector, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };
package/dist/index.d.ts CHANGED
@@ -16,11 +16,34 @@ declare enum MicdropClientCommands {
16
16
  }
17
17
  declare enum MicdropServerCommands {
18
18
  Message = "Message",
19
+ PartialAssistantMessage = "PartialAssistantMessage",
19
20
  CancelLastUserMessage = "CancelLastUserMessage",
20
21
  SkipAnswer = "SkipAnswer",
21
22
  EndCall = "EndCall",
22
23
  ToolCall = "ToolCall"
23
24
  }
25
+ /**
26
+ * Hears whether a sentence has landed, where voice activity detection only
27
+ * hears whether someone is speaking.
28
+ *
29
+ * `SmartTurn` from `@micdrop/smart-turn` implements it, and so can anything
30
+ * else, a call to a service included. Both sides of a call can hold one, the
31
+ * client to decide when its turn ends and the server to decide when to answer.
32
+ */
33
+ interface TurnDetector {
34
+ /**
35
+ * Feeds the audio received since the last call
36
+ * @param samples - Mono samples, in the -1..1 range
37
+ * @param sampleRate - Sample rate of `samples`, in Hz
38
+ */
39
+ push(samples: Float32Array, sampleRate?: number): void;
40
+ /** Answers whether the turn pushed so far sounds finished */
41
+ predict(): Promise<{
42
+ complete: boolean;
43
+ }>;
44
+ /** Starts a new turn, forgetting the previous one */
45
+ reset(): void;
46
+ }
24
47
  interface MicdropCallSummary {
25
48
  conversation: MicdropConversation;
26
49
  duration: number;
@@ -354,15 +377,69 @@ declare abstract class SentenceTTS extends TTS {
354
377
 
355
378
  interface MicdropServerEvents {
356
379
  End: [MicdropCallSummary];
380
+ /** A message was added to the conversation, by the agent or by the server */
381
+ Message: [MicdropConversationItem];
357
382
  UserAudio: [Buffer];
358
383
  AssistantAudio: [Buffer];
359
384
  }
360
385
  interface MicdropConfig {
386
+ /**
387
+ * Opening line, spoken before the user has said anything.
388
+ *
389
+ * It goes into the conversation like any other assistant message. Leaving it
390
+ * out, along with `generateFirstMessage`, hands the first turn to the user.
391
+ */
361
392
  firstMessage?: string;
393
+ /**
394
+ * Asks the agent for its opening line instead of fixing one, so the greeting
395
+ * can lean on the system prompt and on whatever the call was given.
396
+ *
397
+ * Ignored when `firstMessage` is set, and out of reach without an agent.
398
+ */
362
399
  generateFirstMessage?: boolean;
363
- agent: Agent;
400
+ /**
401
+ * Generates the answers.
402
+ *
403
+ * Leaving it out turns the call into a one-way one: what the user says is
404
+ * transcribed and sent to the client, and the assistant stays quiet unless
405
+ * the application calls `speak()` itself.
406
+ */
407
+ agent?: Agent;
408
+ /**
409
+ * Turns what the user says into text, the one component a call cannot do
410
+ * without.
411
+ *
412
+ * It is fed the audio of each turn as it arrives, and the transcripts it
413
+ * returns drive everything that follows.
414
+ */
364
415
  stt: STT;
365
- tts: TTS;
416
+ /**
417
+ * Gives the answers a voice.
418
+ *
419
+ * Leaving it out keeps the call textual: the answer reaches the client as a
420
+ * message and nothing is synthesized.
421
+ */
422
+ tts?: TTS;
423
+ /**
424
+ * Sends the answer to the client while it is still being written, on top of
425
+ * the complete message that follows.
426
+ *
427
+ * Works with any agent, since the text is read from the stream it writes to.
428
+ * Off by default, since a client that only displays finished messages has
429
+ * nothing to do with it.
430
+ */
431
+ partialMessages?: boolean;
432
+ /**
433
+ * Waits for the rest of the sentence when the speaker paused in the middle
434
+ * of one, instead of answering an unfinished question.
435
+ *
436
+ * Prefer running the detector in the client, which reaches the same decision
437
+ * without the round trip and can then close its turns sooner. This is the
438
+ * option for the browsers where the model has nowhere to run.
439
+ */
440
+ turnDetector?: TurnDetector;
441
+ /** How long to wait for the rest of a sentence, 4000 ms by default */
442
+ turnMaxWait?: number;
366
443
  }
367
444
  declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
368
445
  socket: WebSocket$1 | null;
@@ -370,11 +447,16 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
370
447
  logger?: Logger;
371
448
  private startTime;
372
449
  private lastMessageSpeeched?;
450
+ private ownConversation;
373
451
  private operationQueue;
374
452
  private isProcessingQueue;
375
453
  private currentUserStream?;
376
454
  private userSpeechChunks;
455
+ private turnComplete?;
456
+ private heldTurnTimer?;
377
457
  constructor(socket: WebSocket$1, config: MicdropConfig);
458
+ /** Everything said so far, whether an agent or the server keeps it */
459
+ get conversation(): MicdropConversation;
378
460
  private log;
379
461
  private processQueue;
380
462
  private queueOperation;
@@ -385,11 +467,32 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
385
467
  private onMute;
386
468
  private onStartSpeaking;
387
469
  private onStopSpeaking;
470
+ private predictTurnComplete;
471
+ /** Answers, unless the sentence sounds like it has more coming */
472
+ private answerUserTurn;
473
+ /**
474
+ * Answers anyway if the rest of the sentence never comes.
475
+ *
476
+ * Without it, a detector that hears an unfinished sentence where there is
477
+ * none leaves the call silent for good.
478
+ */
479
+ private holdTurn;
480
+ private releaseHeldTurn;
388
481
  private onTranscriptSTT;
389
482
  private onAudioTTS;
483
+ private onMessageAgent;
484
+ /** Adds a message to the conversation and sends it to the client */
485
+ addUserMessage(text: string, metadata?: MicdropAnswerMetadata): void;
486
+ /** Adds a message to the conversation and sends it to the client */
487
+ addAssistantMessage(text: string, metadata?: MicdropAnswerMetadata): void;
488
+ private addOwnMessage;
489
+ private sendMessage;
490
+ private sendPartialAnswer;
390
491
  private sendFirstMessage;
391
492
  answer(): void;
392
493
  private _answer;
494
+ /** Sends the answer to the client as it is written, ahead of the voice */
495
+ private forwardPartialAnswer;
393
496
  speak(message: string | Readable): void;
394
497
  private _speak;
395
498
  }
@@ -427,4 +530,4 @@ declare class MicdropRecorder extends EventEmitter<MicdropRecorderEvents> {
427
530
 
428
531
  declare function waitForParams<CallParams>(socket: WebSocket$1, validate: (params: any) => CallParams): Promise<CallParams>;
429
532
 
430
- export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, STT, type STTEvents, SentenceSplitter, SentenceTTS, TTS, type TTSEvents, type Tool, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };
533
+ export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, STT, type STTEvents, SentenceSplitter, SentenceTTS, TTS, type TTSEvents, type Tool, type TurnDetector, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };