@micdrop/server 2.3.0 → 2.5.0

package/README.md

@@ -1,6 +1,6 @@
 # 🖐️🎤 Micdrop: Real-Time Voice Conversations with AI
 
-[Micdrop website](https://micdrop.dev) | [Documentation](https://micdrop.dev/docs/server) | [Demo](../../examples/demo-server)
+[Micdrop website](https://micdrop.dev) | [Documentation](https://micdrop.dev/docs/server) | [Basic example](../../examples/basic) | [Demo](../../examples/advanced)
 
 Micdrop is a set of open source Typescript packages to build real-time voice conversations with AI agents. It handles all the complexities on the browser and server side (microphone, speaker, VAD, network communication, etc) and provides ready-to-use implementations for various AI providers.
 

package/dist/index.d.mts

@@ -21,6 +21,28 @@ declare enum MicdropServerCommands {
     EndCall = "EndCall",
     ToolCall = "ToolCall"
 }
+/**
+ * Hears whether a sentence has landed, where voice activity detection only
+ * hears whether someone is speaking.
+ *
+ * `SmartTurn` from `@micdrop/smart-turn` implements it, and so can anything
+ * else, a call to a service included. Both sides of a call can hold one, the
+ * client to decide when its turn ends and the server to decide when to answer.
+ */
+interface TurnDetector {
+    /**
+     * Feeds the audio received since the last call
+     * @param samples - Mono samples, in the -1..1 range
+     * @param sampleRate - Sample rate of `samples`, in Hz
+     */
+    push(samples: Float32Array, sampleRate?: number): void;
+    /** Answers whether the turn pushed so far sounds finished */
+    predict(): Promise<{
+        complete: boolean;
+    }>;
+    /** Starts a new turn, forgetting the previous one */
+    reset(): void;
+}
 interface MicdropCallSummary {
     conversation: MicdropConversation;
     duration: number;
@@ -327,6 +349,7 @@ declare abstract class SentenceTTS extends TTS {
     private controller?;
     private generation;
     private counter;
+    private synthesizing;
     /**
      * Turns one sentence into PCM16 audio at the client's sample rate.
      *
@@ -335,6 +358,16 @@ declare abstract class SentenceTTS extends TTS {
      * nothing emits nothing, which is how a cancelled synthesis reports back.
      */
     protected abstract synthesize(text: string, signal: AbortSignal): Promise<Buffer | undefined>;
+    /**
+     * Emits a piece of the sentence being synthesized.
+     *
+     * A model that generates progressively can hand its chunks over as they
+     * come rather than waiting for the sentence to be finished, which brings
+     * the first word forward by the duration of that sentence. The false it
+     * returns says the utterance was cancelled or replaced, so the generation
+     * it comes from can be stopped there.
+     */
+    protected emitAudio(audio: Buffer): boolean;
     speak(textStream: Readable): void;
     cancel(): void;
     private enqueue;
@@ -352,6 +385,17 @@ interface MicdropConfig {
     agent: Agent;
     stt: STT;
     tts: TTS;
+    /**
+     * Waits for the rest of the sentence when the speaker paused in the middle
+     * of one, instead of answering an unfinished question.
+     *
+     * Prefer running the detector in the client, which reaches the same decision
+     * without the round trip and can then close its turns sooner. This is the
+     * option for the browsers where the model has nowhere to run.
+     */
+    turnDetector?: TurnDetector;
+    /** How long to wait for the rest of a sentence, 4000 ms by default */
+    turnMaxWait?: number;
 }
 declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
     socket: WebSocket$1 | null;
@@ -363,6 +407,8 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
     private isProcessingQueue;
     private currentUserStream?;
     private userSpeechChunks;
+    private turnComplete?;
+    private heldTurnTimer?;
     constructor(socket: WebSocket$1, config: MicdropConfig);
     private log;
     private processQueue;
@@ -374,6 +420,17 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
     private onMute;
     private onStartSpeaking;
     private onStopSpeaking;
+    private predictTurnComplete;
+    /** Answers, unless the sentence sounds like it has more coming */
+    private answerUserTurn;
+    /**
+     * Answers anyway if the rest of the sentence never comes.
+     *
+     * Without it, a detector that hears an unfinished sentence where there is
+     * none leaves the call silent for good.
+     */
+    private holdTurn;
+    private releaseHeldTurn;
     private onTranscriptSTT;
     private onAudioTTS;
     private sendFirstMessage;
@@ -416,4 +473,4 @@ declare class MicdropRecorder extends EventEmitter<MicdropRecorderEvents> {
 
 declare function waitForParams<CallParams>(socket: WebSocket$1, validate: (params: any) => CallParams): Promise<CallParams>;
 
-export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, STT, type STTEvents, SentenceSplitter, SentenceTTS, TTS, type TTSEvents, type Tool, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };
+export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, STT, type STTEvents, SentenceSplitter, SentenceTTS, TTS, type TTSEvents, type Tool, type TurnDetector, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };

package/dist/index.d.ts

@@ -21,6 +21,28 @@ declare enum MicdropServerCommands {
     EndCall = "EndCall",
     ToolCall = "ToolCall"
 }
+/**
+ * Hears whether a sentence has landed, where voice activity detection only
+ * hears whether someone is speaking.
+ *
+ * `SmartTurn` from `@micdrop/smart-turn` implements it, and so can anything
+ * else, a call to a service included. Both sides of a call can hold one, the
+ * client to decide when its turn ends and the server to decide when to answer.
+ */
+interface TurnDetector {
+    /**
+     * Feeds the audio received since the last call
+     * @param samples - Mono samples, in the -1..1 range
+     * @param sampleRate - Sample rate of `samples`, in Hz
+     */
+    push(samples: Float32Array, sampleRate?: number): void;
+    /** Answers whether the turn pushed so far sounds finished */
+    predict(): Promise<{
+        complete: boolean;
+    }>;
+    /** Starts a new turn, forgetting the previous one */
+    reset(): void;
+}
 interface MicdropCallSummary {
     conversation: MicdropConversation;
     duration: number;
@@ -327,6 +349,7 @@ declare abstract class SentenceTTS extends TTS {
     private controller?;
     private generation;
     private counter;
+    private synthesizing;
     /**
      * Turns one sentence into PCM16 audio at the client's sample rate.
      *
@@ -335,6 +358,16 @@ declare abstract class SentenceTTS extends TTS {
      * nothing emits nothing, which is how a cancelled synthesis reports back.
      */
     protected abstract synthesize(text: string, signal: AbortSignal): Promise<Buffer | undefined>;
+    /**
+     * Emits a piece of the sentence being synthesized.
+     *
+     * A model that generates progressively can hand its chunks over as they
+     * come rather than waiting for the sentence to be finished, which brings
+     * the first word forward by the duration of that sentence. The false it
+     * returns says the utterance was cancelled or replaced, so the generation
+     * it comes from can be stopped there.
+     */
+    protected emitAudio(audio: Buffer): boolean;
     speak(textStream: Readable): void;
     cancel(): void;
     private enqueue;
@@ -352,6 +385,17 @@ interface MicdropConfig {
     agent: Agent;
     stt: STT;
     tts: TTS;
+    /**
+     * Waits for the rest of the sentence when the speaker paused in the middle
+     * of one, instead of answering an unfinished question.
+     *
+     * Prefer running the detector in the client, which reaches the same decision
+     * without the round trip and can then close its turns sooner. This is the
+     * option for the browsers where the model has nowhere to run.
+     */
+    turnDetector?: TurnDetector;
+    /** How long to wait for the rest of a sentence, 4000 ms by default */
+    turnMaxWait?: number;
 }
 declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
     socket: WebSocket$1 | null;
@@ -363,6 +407,8 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
     private isProcessingQueue;
     private currentUserStream?;
     private userSpeechChunks;
+    private turnComplete?;
+    private heldTurnTimer?;
     constructor(socket: WebSocket$1, config: MicdropConfig);
     private log;
     private processQueue;
@@ -374,6 +420,17 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
     private onMute;
     private onStartSpeaking;
     private onStopSpeaking;
+    private predictTurnComplete;
+    /** Answers, unless the sentence sounds like it has more coming */
+    private answerUserTurn;
+    /**
+     * Answers anyway if the rest of the sentence never comes.
+     *
+     * Without it, a detector that hears an unfinished sentence where there is
+     * none leaves the call silent for good.
+     */
+    private holdTurn;
+    private releaseHeldTurn;
     private onTranscriptSTT;
     private onAudioTTS;
     private sendFirstMessage;
@@ -416,4 +473,4 @@ declare class MicdropRecorder extends EventEmitter<MicdropRecorderEvents> {
 
 declare function waitForParams<CallParams>(socket: WebSocket$1, validate: (params: any) => CallParams): Promise<CallParams>;
 
-export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, STT, type STTEvents, SentenceSplitter, SentenceTTS, TTS, type TTSEvents, type Tool, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };
+export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, STT, type STTEvents, SentenceSplitter, SentenceTTS, TTS, type TTSEvents, type Tool, type TurnDetector, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };

package/dist/index.js

@@ -511,6 +511,8 @@ var MicdropServerCommands = /* @__PURE__ */ ((MicdropServerCommands2) => {
 })(MicdropServerCommands || {});
 
 // src/MicdropServer.ts
+var USER_SAMPLE_RATE = 16e3;
+var DEFAULT_TURN_MAX_WAIT = 4e3;
 var MicdropServer = class extends import_eventemitter32.EventEmitter {
   constructor(socket, config) {
     super();
@@ -522,6 +524,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
     this.isProcessingQueue = false;
     this.userSpeechChunks = 0;
     this.onClose = () => {
+      this.releaseHeldTurn();
       if (!this.config) return;
       this.log("Connection closed");
       const duration = Math.round((Date.now() - this.startTime) / 1e3);
@@ -565,8 +568,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
       this.config.agent.addUserMessage(transcript);
       if (!this.currentUserStream) {
         this.log("User stopped speaking, answering");
-        this.cancel();
-        this.answer();
+        this.answerUserTurn();
       }
     };
     this.onAudioTTS = (audio) => {
@@ -639,6 +641,7 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
     this.log(`Received chunk (${chunk.byteLength} bytes)`);
     this.currentUserStream?.write(chunk);
     this.userSpeechChunks++;
+    this.config?.turnDetector?.push(pcm16ToFloat32(chunk), USER_SAMPLE_RATE);
     this.emit("UserAudio", chunk);
   }
   onMute() {
@@ -652,6 +655,9 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
     this.userSpeechChunks = 0;
     this.currentUserStream?.end();
     this.currentUserStream = new import_stream2.PassThrough();
+    this.config.turnDetector?.reset();
+    this.turnComplete = void 0;
+    this.releaseHeldTurn();
     this.config.stt.transcribe(this.currentUserStream);
     this.cancel();
   }
@@ -664,15 +670,60 @@ var MicdropServer = class extends import_eventemitter32.EventEmitter {
       this.socket?.send("SkipAnswer" /* SkipAnswer */);
       return;
     }
+    this.turnComplete = this.predictTurnComplete();
     const conversation = this.config?.agent.conversation;
     const lastMessage = conversation?.[conversation.length - 1];
     if (lastMessage?.role === "user" && this.lastMessageSpeeched !== lastMessage) {
       this.log(
         "User stopped speaking and a transcript already exists, answering"
       );
+      this.answerUserTurn();
+    }
+  }
+  async predictTurnComplete() {
+    const detector = this.config?.turnDetector;
+    if (!detector) return true;
+    try {
+      const { complete } = await detector.predict();
+      this.log(`Turn sounds ${complete ? "finished" : "unfinished"}`);
+      return complete;
+    } catch (error) {
+      this.log(`Turn detection failed: ${error}`);
+      return true;
+    }
+  }
+  /** Answers, unless the sentence sounds like it has more coming */
+  async answerUserTurn() {
+    const complete = await (this.turnComplete ?? Promise.resolve(true));
+    if (!complete) {
+      this.log("Waiting for the rest of the sentence");
+      this.socket?.send("SkipAnswer" /* SkipAnswer */);
+      this.holdTurn();
+      return;
+    }
+    this.releaseHeldTurn();
+    this.cancel();
+    this.answer();
+  }
+  /**
+   * Answers anyway if the rest of the sentence never comes.
+   *
+   * Without it, a detector that hears an unfinished sentence where there is
+   * none leaves the call silent for good.
+   */
+  holdTurn() {
+    this.releaseHeldTurn();
+    this.heldTurnTimer = setTimeout(() => {
+      this.heldTurnTimer = void 0;
+      this.log("Nothing more came, answering");
       this.cancel();
       this.answer();
-    }
+    }, this.config?.turnMaxWait ?? DEFAULT_TURN_MAX_WAIT);
+  }
+  releaseHeldTurn() {
+    if (!this.heldTurnTimer) return;
+    clearTimeout(this.heldTurnTimer);
+    this.heldTurnTimer = void 0;
   }
   sendFirstMessage() {
     if (!this.config) return;
@@ -1101,6 +1152,22 @@ var SentenceTTS = class extends TTS {
     // whether it is still the one that should be heard.
     this.generation = 0;
     this.counter = 0;
+    // Identifies the current speak() call
+    this.synthesizing = 0;
+  }
+  /**
+   * Emits a piece of the sentence being synthesized.
+   *
+   * A model that generates progressively can hand its chunks over as they
+   * come rather than waiting for the sentence to be finished, which brings
+   * the first word forward by the duration of that sentence. The false it
+   * returns says the utterance was cancelled or replaced, so the generation
+   * it comes from can be stopped there.
+   */
+  emitAudio(audio) {
+    if (this.synthesizing !== this.counter) return false;
+    if (audio.length) this.emit("Audio", audio);
+    return true;
   }
   speak(textStream) {
     const generation = ++this.generation;
@@ -1146,6 +1213,7 @@ var SentenceTTS = class extends TTS {
     this.draining = true;
     while (this.queue.length > 0) {
       const counter = this.counter;
+      this.synthesizing = counter;
       const text = this.queue.shift();
       const controller = new AbortController();
       this.controller = controller;