@micdrop/server 2.7.0 → 2.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -20,7 +20,8 @@ declare enum MicdropServerCommands {
20
20
  CancelLastUserMessage = "CancelLastUserMessage",
21
21
  SkipAnswer = "SkipAnswer",
22
22
  EndCall = "EndCall",
23
- ToolCall = "ToolCall"
23
+ ToolCall = "ToolCall",
24
+ Classification = "Classification"
24
25
  }
25
26
  /**
26
27
  * Hears whether a sentence has landed, where voice activity detection only
@@ -70,6 +71,15 @@ interface MicdropConversationToolResult {
70
71
  toolName: string;
71
72
  output: string;
72
73
  }
74
+ /** What a classifier made of an input, a turn of the user in a call */
75
+ interface MicdropClassification<Result = any, Input = any> {
76
+ /** What was classified */
77
+ input: Input;
78
+ /** Answers of the classifier, in the shape its provider returns */
79
+ result: Result;
80
+ /** Time the classification took, in ms */
81
+ duration: number;
82
+ }
73
83
  interface MicdropToolCall {
74
84
  name: string;
75
85
  parameters: any;
@@ -78,6 +88,22 @@ interface MicdropToolCall {
78
88
  type DeepPartial<T> = T extends object ? {
79
89
  [P in keyof T]?: DeepPartial<T[P]>;
80
90
  } : T;
91
+ /**
92
+ * The turn of the user so far: the user messages at the end of the
93
+ * conversation, one per transcript, joined with a space. It starts after the
94
+ * last answer, or after the last user message already classified, which is
95
+ * what tells two turns apart when no agent answers them.
96
+ */
97
+ declare function currentTurn(conversation: MicdropConversation): {
98
+ transcript: string;
99
+ messages: MicdropConversationMessage<MicdropAnswerMetadata>[];
100
+ before: MicdropConversationItem[];
101
+ };
102
+ /**
103
+ * The classification of the last turn of the user, which the server keeps in
104
+ * the metadata of its last message. Undefined until one read the whole turn.
105
+ */
106
+ declare function getTurnClassification<Result = any>(conversation: MicdropConversation): MicdropClassification<Result> | undefined;
81
107
 
82
108
  interface Tool<Schema extends z.ZodObject = z.ZodObject> {
83
109
  name: string;
@@ -100,7 +126,7 @@ interface AgentOptions {
100
126
  autoSemanticTurn?: boolean | string;
101
127
  autoIgnoreUserNoise?: boolean | string;
102
128
  extract?: ExtractJsonOptions | ExtractTagOptions;
103
- onBeforeAnswer?: (this: Agent, stream: Writable) => void | boolean | Promise<boolean>;
129
+ onBeforeAnswer?: (this: Agent, stream: Writable) => void | boolean | string | Promise<void | boolean | string>;
104
130
  }
105
131
  interface AgentEvents {
106
132
  Message: [MicdropConversationItem];
@@ -122,7 +148,7 @@ interface ExtractTagOptions extends ExtractOptions {
122
148
  startTag: string;
123
149
  endTag: string;
124
150
  }
125
- declare abstract class Agent<Options extends AgentOptions = AgentOptions> extends EventEmitter<AgentEvents> {
151
+ declare abstract class Agent<Options extends AgentOptions = AgentOptions, Events extends AgentEvents = AgentEvents> extends EventEmitter<Events> {
126
152
  protected options: Options;
127
153
  logger?: Logger;
128
154
  conversation: MicdropConversation;
@@ -158,6 +184,13 @@ declare abstract class Agent<Options extends AgentOptions = AgentOptions> extend
158
184
  message: string;
159
185
  metadata: MicdropAnswerMetadata | undefined;
160
186
  };
187
+ /**
188
+ * Emits one of the events every agent has.
189
+ *
190
+ * A subclass can add events of its own, and TypeScript then no longer knows
191
+ * the shared ones are among them.
192
+ */
193
+ protected emitAgentEvent<Name extends keyof AgentEvents>(event: Name, ...args: AgentEvents[Name]): void;
161
194
  protected log(...message: any[]): void;
162
195
  destroy(): void;
163
196
  }
@@ -193,6 +226,76 @@ declare class MockAgent extends Agent {
193
226
  cancel(): void;
194
227
  }
195
228
 
229
+ /** What a classifier reads: a text, or JSON with named fields */
230
+ type ClassifierInput = string | {
231
+ [key: string]: any;
232
+ } | any[];
233
+ interface ClassifierEvents<Result, Input> {
234
+ Classification: [MicdropClassification<Result, Input>];
235
+ }
236
+ /**
237
+ * Turns an input into typed decisions: intent, mood, risk. It answers
238
+ * questions about what the user said, where an agent answers the user.
239
+ *
240
+ * `MicdropServer` classifies each turn of the user when it ends, with a
241
+ * `MicdropTurnInput`, and a classifier works on its own just as well: call
242
+ * `classify()` with any text or JSON.
243
+ *
244
+ * This class keeps track of the classifications in progress, to wait for them
245
+ * or cancel them. A provider implements `evaluate()`.
246
+ */
247
+ declare abstract class Classifier<Result = any, Input extends ClassifierInput = ClassifierInput> extends EventEmitter<ClassifierEvents<Result, Input>> {
248
+ logger?: Logger;
249
+ /** The last classification */
250
+ lastClassification?: MicdropClassification<Result, Input>;
251
+ private controllers;
252
+ private _pending?;
253
+ /**
254
+ * Answers the questions of this classifier about an input
255
+ * @param input - What to classify
256
+ * @param signal - Aborted once the result is no longer wanted
257
+ */
258
+ protected abstract evaluate(input: Input, signal: AbortSignal): Promise<Result>;
259
+ /**
260
+ * Classifies an input, and emits the result as `Classification`
261
+ * @param input - What to classify: a text, or JSON with named fields
262
+ * @returns The classification, or undefined when cancelled or failed
263
+ */
264
+ classify(input: Input): Promise<MicdropClassification<Result, Input> | undefined>;
265
+ /** The last classification started, until it is done */
266
+ get pending(): Promise<MicdropClassification<Result, Input> | undefined> | undefined;
267
+ /** Drops every classification in progress, which then emit nothing */
268
+ cancel(): void;
269
+ protected log(...message: any[]): void;
270
+ destroy(): void;
271
+ private run;
272
+ }
273
+ /** What `MicdropServer` classifies at the end of each turn of the user */
274
+ interface MicdropTurnInput {
275
+ /**
276
+ * The turn before, and the answer that followed it, so words like "it" or
277
+ * "there" can be understood
278
+ */
279
+ history: Array<{
280
+ role: 'user' | 'assistant';
281
+ text: string;
282
+ }>;
283
+ /** What the user said in this turn, one or more transcripts joined */
284
+ turn: string;
285
+ }
286
+
287
+ /**
288
+ * Builds what `MicdropServer` classifies at the end of a turn: the turn
289
+ * itself, and the turns before it with the answers that followed them.
290
+ * @param conversation - The conversation, the turn at its end
291
+ * @param history - How many turns of the user before this one to include
292
+ * @returns The input, and the last message of the turn, which keeps the result
293
+ */
294
+ declare function turnInput(conversation: MicdropConversation, history?: number): {
295
+ input: MicdropTurnInput;
296
+ message: MicdropConversationMessage<MicdropAnswerMetadata>;
297
+ };
298
+
196
299
  /**
197
300
  * Streaming linear-interpolation resampler for PCM16 mono audio.
198
301
  *
@@ -242,6 +345,134 @@ declare class MicdropError extends Error {
242
345
  }
243
346
  declare function handleError(socket: WebSocket, error: unknown): void;
244
347
 
348
+ type RealtimeOptions = Pick<AgentOptions, 'systemPrompt' | 'autoEndCall' | 'autoSemanticTurn' | 'autoIgnoreUserNoise'>;
349
+ interface RealtimeEvents extends AgentEvents {
350
+ /** Voice of the assistant, in PCM16 at 16 kHz */
351
+ Audio: [Buffer];
352
+ /** What the assistant has said so far in the answer it is giving */
353
+ PartialMessage: [string];
354
+ }
355
+ /**
356
+ * A model that hears the user and answers with its own voice, standing in for
357
+ * the speech to text, the agent and the text to speech of a call.
358
+ *
359
+ * It is an agent: it holds the conversation and the tools, and emits the same
360
+ * events. What differs is what goes in and out, audio in turns rather than
361
+ * transcripts, and audio rather than text.
362
+ *
363
+ * This class keeps what every provider shares: when a turn opens, which
364
+ * answer the output belongs to, dropping what arrives after an interruption,
365
+ * and writing the conversation in the order it was spoken. A provider
366
+ * implements the protected methods that talk to its API.
367
+ */
368
+ declare abstract class Realtime<Options extends RealtimeOptions = RealtimeOptions> extends Agent<Options, RealtimeEvents> {
369
+ private turn;
370
+ private currentAnswer?;
371
+ private discarding;
372
+ private queue;
373
+ private answerWaiters;
374
+ private userTranscript;
375
+ private awaitingUserTranscript;
376
+ private userTranscriptTimer?;
377
+ private afterUserTranscript;
378
+ /**
379
+ * The user started speaking.
380
+ *
381
+ * The assistant stops, and the audio read from the stream goes to the model.
382
+ * A turn left open, because the sentence sounded unfinished, carries on.
383
+ */
384
+ startTurn(audio: Readable): void;
385
+ /** Closes the turn of the user, and has the model answer it */
386
+ endTurn(): void;
387
+ /** Drops the turn of the user, without an answer */
388
+ discardTurn(): void;
389
+ /** Has the model speak without waiting for the user, to open a call */
390
+ respond(): void;
391
+ /**
392
+ * Has the model say this text, word for word.
393
+ *
394
+ * The caller records the text in the conversation, as with a text to speech,
395
+ * so the transcript of what the model says is not recorded again.
396
+ */
397
+ say(text: string): void;
398
+ cancel(): void;
399
+ /**
400
+ * Adds a message the model did not hear, and tells the model about it.
401
+ *
402
+ * The transcripts of the call are recorded apart, so they are not sent back.
403
+ */
404
+ addMessage(role: 'user' | 'assistant' | 'system', text: string, metadata?: MicdropAnswerMetadata): void;
405
+ addTool<Schema extends z.ZodObject>(tool: Tool<Schema>): void;
406
+ removeTool(name: string): void;
407
+ /**
408
+ * Lists the events that have listeners.
409
+ *
410
+ * Typed as the events of any agent, so a realtime model goes wherever an
411
+ * agent is expected. The two extra events do not change what that code reads.
412
+ */
413
+ eventNames(): Array<keyof AgentEvents>;
414
+ destroy(): void;
415
+ protected generateAnswer(stream: PassThrough): Promise<void>;
416
+ /** Sends audio of the user, PCM16 at 16 kHz */
417
+ protected abstract appendAudio(chunk: Buffer): void;
418
+ /** Opens a turn of the user, right before its first audio */
419
+ protected abstract openTurn(): void;
420
+ /** Closes the turn of the user, which has the model answer it */
421
+ protected abstract closeTurn(): void;
422
+ /** Drops the audio of the open turn, without an answer */
423
+ protected abstract clearTurn(): void;
424
+ /** Has the model speak with no turn of the user to answer */
425
+ protected abstract generate(): void;
426
+ /** Has the model say this text, word for word */
427
+ protected abstract speakText(text: string): void;
428
+ /** Stops the answer the model is giving */
429
+ protected abstract cancelAnswer(): void;
430
+ /** Sends the tools to the model again, after one was added or removed */
431
+ protected abstract updateTools(): void;
432
+ /** Tells the model about a message it did not hear */
433
+ protected abstract injectMessage(role: 'user' | 'assistant' | 'system', text: string): void;
434
+ /** Nobody is speaking, nor about to, so the connection can be renewed */
435
+ protected get isIdle(): boolean;
436
+ /** Voice of the answer, PCM16 at 16 kHz */
437
+ protected emitAudio(chunk: Buffer): void;
438
+ /** Transcript of the answer, as it is spoken */
439
+ protected addAnswerTranscript(text: string): void;
440
+ /**
441
+ * The model finished its answer, or finished dropping an interrupted one.
442
+ *
443
+ * An answer without audio tells the client to stop waiting for one.
444
+ */
445
+ protected endAnswer(): void;
446
+ /** Runs a tool the model called, and records it in the conversation */
447
+ protected runTool(toolCall: MicdropConversationToolCall): Promise<{
448
+ output: any;
449
+ skipAnswer: boolean | undefined;
450
+ } | {
451
+ output: {
452
+ error: any;
453
+ };
454
+ skipAnswer?: undefined;
455
+ }>;
456
+ /** Transcript of the turn of the user, possibly in several pieces */
457
+ protected addUserTranscript(text: string): void;
458
+ /** The transcript of the turn is complete, it goes in the conversation */
459
+ protected commitUserTranscript(): void;
460
+ private sendAudio;
461
+ private beginAnswer;
462
+ /**
463
+ * The answer output belongs to, if it is still wanted.
464
+ *
465
+ * The model can speak without being asked, to follow up on a tool call for
466
+ * instance, which opens an answer of its own.
467
+ */
468
+ private activeAnswer;
469
+ private whenIdle;
470
+ private recordAnswer;
471
+ private awaitUserTranscript;
472
+ private clearUserTranscriptTimer;
473
+ private resolveAnswerWaiters;
474
+ }
475
+
245
476
  interface STTEvents {
246
477
  Transcript: [string];
247
478
  Failed: [Buffer[]];
@@ -407,12 +638,12 @@ interface MicdropConfig {
407
638
  agent?: Agent;
408
639
  /**
409
640
  * Turns what the user says into text, the one component a call cannot do
410
- * without.
641
+ * without, unless a realtime model hears the user itself.
411
642
  *
412
643
  * It is fed the audio of each turn as it arrives, and the transcripts it
413
644
  * returns drive everything that follows.
414
645
  */
415
- stt: STT;
646
+ stt?: STT;
416
647
  /**
417
648
  * Gives the answers a voice.
418
649
  *
@@ -440,6 +671,52 @@ interface MicdropConfig {
440
671
  turnDetector?: TurnDetector;
441
672
  /** How long to wait for the rest of a sentence, 4000 ms by default */
442
673
  turnMaxWait?: number;
674
+ /**
675
+ * Hears the user and answers with its own voice, in place of `stt`, `agent`
676
+ * and `tts`, which are then ignored.
677
+ *
678
+ * The client does not change: it still detects when the user speaks, and the
679
+ * turn it sends goes to the model as audio.
680
+ */
681
+ realtime?: Realtime;
682
+ /**
683
+ * Classifies each turn of the user when it ends, with a `MicdropTurnInput`:
684
+ * the turn, and the turn before with its answer.
685
+ *
686
+ * It works next to the agent and leaves the answer to it. A classification
687
+ * of the turn goes in the metadata of its last user message, under
688
+ * `classification`, where `getTurnClassification()` reads it.
689
+ */
690
+ classifier?: Classifier<any, any>;
691
+ /** How the server uses the classifier */
692
+ classifierOptions?: MicdropClassifierOptions;
693
+ }
694
+ interface MicdropClassifierOptions {
695
+ /**
696
+ * Sends every classification to the client, which emits it as
697
+ * `Classification`.
698
+ *
699
+ * Off by default, since a result can hold what the user should not see, a
700
+ * manipulation score for instance.
701
+ */
702
+ sendToClient?: boolean;
703
+ /**
704
+ * Holds each answer until the classification of its turn is done, so the
705
+ * agent can read it in `onBeforeAnswer`. Off by default: the answer starts
706
+ * right away and the classification runs next to it.
707
+ */
708
+ waitBeforeAnswer?: boolean;
709
+ /**
710
+ * How long `waitBeforeAnswer` holds an answer, 1000 ms by default. A
711
+ * classification that takes longer lets the answer go without it.
712
+ */
713
+ maxWait?: number;
714
+ /**
715
+ * How many turns of the user before this one the input holds, with the
716
+ * answers that followed them. 1 by default: the turn before, its answer,
717
+ * and this turn.
718
+ */
719
+ history?: number;
443
720
  }
444
721
  declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
445
722
  socket: WebSocket$1 | null;
@@ -457,6 +734,8 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
457
734
  constructor(socket: WebSocket$1, config: MicdropConfig);
458
735
  /** Everything said so far, whether an agent or the server keeps it */
459
736
  get conversation(): MicdropConversation;
737
+ /** What holds the conversation and answers, a realtime model included */
738
+ private get agent();
460
739
  private log;
461
740
  private processQueue;
462
741
  private queueOperation;
@@ -477,9 +756,11 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
477
756
  * none leaves the call silent for good.
478
757
  */
479
758
  private holdTurn;
759
+ /** Answers what the user just said */
760
+ private answerTurn;
480
761
  private releaseHeldTurn;
481
762
  private onTranscriptSTT;
482
- private onAudioTTS;
763
+ private onAssistantAudio;
483
764
  private onMessageAgent;
484
765
  /** Adds a message to the conversation and sends it to the client */
485
766
  addUserMessage(text: string, metadata?: MicdropAnswerMetadata): void;
@@ -493,6 +774,13 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
493
774
  private _answer;
494
775
  /** Sends the answer to the client as it is written, ahead of the voice */
495
776
  private forwardPartialAnswer;
777
+ /**
778
+ * Classifies the turn that just ended, with the one before, and keeps the
779
+ * result in the metadata of its last message
780
+ */
781
+ private classifyTurn;
782
+ /** Waits for the classification in progress, never longer than its limit */
783
+ private waitForClassification;
496
784
  speak(message: string | Readable): void;
497
785
  private _speak;
498
786
  }
@@ -569,4 +857,4 @@ declare class SharedInstances<T> {
569
857
 
570
858
  declare function waitForParams<CallParams>(socket: WebSocket$1, validate: (params: any) => CallParams): Promise<CallParams>;
571
859
 
572
- export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, STT, type STTEvents, SentenceSplitter, SentenceTTS, SharedInstances, TTS, type TTSEvents, type Tool, type TurnDetector, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };
860
+ export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, Classifier, type ClassifierEvents, type ClassifierInput, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, type MicdropClassification, type MicdropClassifierOptions, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, type MicdropTurnInput, MockAgent, MockSTT, MockTTS, Pcm16Resampler, Realtime, type RealtimeEvents, type RealtimeOptions, STT, type STTEvents, SentenceSplitter, SentenceTTS, SharedInstances, TTS, type TTSEvents, type Tool, type TurnDetector, currentTurn, float32ToPcm16, getTurnClassification, handleError, pcm16ToFloat32, turnInput, waitForParams };