@micdrop/server 2.8.0 → 2.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.mts CHANGED
@@ -1,7 +1,7 @@
1
1
  import { EventEmitter } from 'eventemitter3';
2
2
  import { Writable, PassThrough, Readable } from 'stream';
3
3
  import { z } from 'zod';
4
- import WebSocket, { WebSocket as WebSocket$1 } from 'ws';
4
+ import WebSocket, { WebSocket as WebSocket$1, RawData, ClientOptions } from 'ws';
5
5
 
6
6
  declare class Logger {
7
7
  name: string;
@@ -20,7 +20,8 @@ declare enum MicdropServerCommands {
20
20
  CancelLastUserMessage = "CancelLastUserMessage",
21
21
  SkipAnswer = "SkipAnswer",
22
22
  EndCall = "EndCall",
23
- ToolCall = "ToolCall"
23
+ ToolCall = "ToolCall",
24
+ Classification = "Classification"
24
25
  }
25
26
  /**
26
27
  * Hears whether a sentence has landed, where voice activity detection only
@@ -70,6 +71,15 @@ interface MicdropConversationToolResult {
70
71
  toolName: string;
71
72
  output: string;
72
73
  }
74
+ /** What a classifier made of an input, a turn of the user in a call */
75
+ interface MicdropClassification<Result = any, Input = any> {
76
+ /** What was classified */
77
+ input: Input;
78
+ /** Answers of the classifier, in the shape its provider returns */
79
+ result: Result;
80
+ /** Time the classification took, in ms */
81
+ duration: number;
82
+ }
73
83
  interface MicdropToolCall {
74
84
  name: string;
75
85
  parameters: any;
@@ -78,6 +88,22 @@ interface MicdropToolCall {
78
88
  type DeepPartial<T> = T extends object ? {
79
89
  [P in keyof T]?: DeepPartial<T[P]>;
80
90
  } : T;
91
+ /**
92
+ * The turn of the user so far: the user messages at the end of the
93
+ * conversation, one per transcript, joined with a space. It starts after the
94
+ * last answer, or after the last user message already classified, which is
95
+ * what tells two turns apart when no agent answers them.
96
+ */
97
+ declare function currentTurn(conversation: MicdropConversation): {
98
+ transcript: string;
99
+ messages: MicdropConversationMessage<MicdropAnswerMetadata>[];
100
+ before: MicdropConversationItem[];
101
+ };
102
+ /**
103
+ * The classification of the last turn of the user, which the server keeps in
104
+ * the metadata of its last message. Undefined until one read the whole turn.
105
+ */
106
+ declare function getTurnClassification<Result = any>(conversation: MicdropConversation): MicdropClassification<Result> | undefined;
81
107
 
82
108
  interface Tool<Schema extends z.ZodObject = z.ZodObject> {
83
109
  name: string;
@@ -100,7 +126,7 @@ interface AgentOptions {
100
126
  autoSemanticTurn?: boolean | string;
101
127
  autoIgnoreUserNoise?: boolean | string;
102
128
  extract?: ExtractJsonOptions | ExtractTagOptions;
103
- onBeforeAnswer?: (this: Agent, stream: Writable) => void | boolean | Promise<boolean>;
129
+ onBeforeAnswer?: (this: Agent, stream: Writable) => void | boolean | string | Promise<void | boolean | string>;
104
130
  }
105
131
  interface AgentEvents {
106
132
  Message: [MicdropConversationItem];
@@ -127,11 +153,34 @@ declare abstract class Agent<Options extends AgentOptions = AgentOptions, Events
127
153
  logger?: Logger;
128
154
  conversation: MicdropConversation;
129
155
  tools: Tool[];
156
+ private answerController?;
157
+ /**
158
+ * @deprecated Read the signal given to `generateAnswer` instead. Still
159
+ * bumped by every answer and every cancel, for agents written before it.
160
+ */
130
161
  protected answerCount: number;
131
- protected answering: boolean;
132
162
  constructor(options: Options);
133
- protected abstract generateAnswer(stream: PassThrough): Promise<void>;
134
- abstract cancel(): void;
163
+ /**
164
+ * Writes the answer to the stream, and records it in the conversation.
165
+ *
166
+ * The signal is aborted when the answer is cancelled or replaced by the next
167
+ * one. From then on nothing may be written nor recorded: hand it to the
168
+ * request, and check it before a retry and before adding a message.
169
+ */
170
+ protected abstract generateAnswer(stream: PassThrough, signal: AbortSignal): Promise<void>;
171
+ /** Whether an answer is being generated */
172
+ get isAnswering(): boolean;
173
+ /** @deprecated Read `isAnswering` instead */
174
+ protected get answering(): boolean;
175
+ /** @deprecated The answer is over once its stream ends or it is cancelled */
176
+ protected set answering(_value: boolean);
177
+ /**
178
+ * Stops the answer being generated, by aborting its signal.
179
+ *
180
+ * An agent overriding it to stop more than the signal does calls
181
+ * `super.cancel()`.
182
+ */
183
+ cancel(): void;
135
184
  answer(): Readable;
136
185
  addUserMessage(text: string, metadata?: MicdropAnswerMetadata): void;
137
186
  addAssistantMessage(text: string, metadata?: MicdropAnswerMetadata): void;
@@ -181,7 +230,7 @@ declare class FallbackAgent extends Agent {
181
230
  message: string;
182
231
  metadata: MicdropAnswerMetadata | undefined;
183
232
  };
184
- protected generateAnswer(stream: PassThrough): Promise<void>;
233
+ protected generateAnswer(stream: PassThrough, signal: AbortSignal): Promise<void>;
185
234
  cancel(): void;
186
235
  destroy(): void;
187
236
  private pipeAnswer;
@@ -197,8 +246,77 @@ declare class MockAgent extends Agent {
197
246
  private i;
198
247
  constructor();
199
248
  protected generateAnswer(stream: PassThrough): Promise<void>;
249
+ }
250
+
251
+ /** What a classifier reads: a text, or JSON with named fields */
252
+ type ClassifierInput = string | {
253
+ [key: string]: any;
254
+ } | any[];
255
+ interface ClassifierEvents<Result, Input> {
256
+ Classification: [MicdropClassification<Result, Input>];
257
+ }
258
+ /**
259
+ * Turns an input into typed decisions: intent, mood, risk. It answers
260
+ * questions about what the user said, where an agent answers the user.
261
+ *
262
+ * `MicdropServer` classifies each turn of the user when it ends, with a
263
+ * `MicdropTurnInput`, and a classifier works on its own just as well: call
264
+ * `classify()` with any text or JSON.
265
+ *
266
+ * This class keeps track of the classifications in progress, to wait for them
267
+ * or cancel them. A provider implements `evaluate()`.
268
+ */
269
+ declare abstract class Classifier<Result = any, Input extends ClassifierInput = ClassifierInput> extends EventEmitter<ClassifierEvents<Result, Input>> {
270
+ logger?: Logger;
271
+ /** The last classification */
272
+ lastClassification?: MicdropClassification<Result, Input>;
273
+ private controllers;
274
+ private _pending?;
275
+ /**
276
+ * Answers the questions of this classifier about an input
277
+ * @param input - What to classify
278
+ * @param signal - Aborted once the result is no longer wanted
279
+ */
280
+ protected abstract evaluate(input: Input, signal: AbortSignal): Promise<Result>;
281
+ /**
282
+ * Classifies an input, and emits the result as `Classification`
283
+ * @param input - What to classify: a text, or JSON with named fields
284
+ * @returns The classification, or undefined when cancelled or failed
285
+ */
286
+ classify(input: Input): Promise<MicdropClassification<Result, Input> | undefined>;
287
+ /** The last classification started, until it is done */
288
+ get pending(): Promise<MicdropClassification<Result, Input> | undefined> | undefined;
289
+ /** Drops every classification in progress, which then emit nothing */
200
290
  cancel(): void;
291
+ protected log(...message: any[]): void;
292
+ destroy(): void;
293
+ private run;
201
294
  }
295
+ /** What `MicdropServer` classifies at the end of each turn of the user */
296
+ interface MicdropTurnInput {
297
+ /**
298
+ * The turn before, and the answer that followed it, so words like "it" or
299
+ * "there" can be understood
300
+ */
301
+ history: Array<{
302
+ role: 'user' | 'assistant';
303
+ text: string;
304
+ }>;
305
+ /** What the user said in this turn, one or more transcripts joined */
306
+ turn: string;
307
+ }
308
+
309
+ /**
310
+ * Builds what `MicdropServer` classifies at the end of a turn: the turn
311
+ * itself, and the turns before it with the answers that followed them.
312
+ * @param conversation - The conversation, the turn at its end
313
+ * @param history - How many turns of the user before this one to include
314
+ * @returns The input, and the last message of the turn, which keeps the result
315
+ */
316
+ declare function turnInput(conversation: MicdropConversation, history?: number): {
317
+ input: MicdropTurnInput;
318
+ message: MicdropConversationMessage<MicdropAnswerMetadata>;
319
+ };
202
320
 
203
321
  /**
204
322
  * Streaming linear-interpolation resampler for PCM16 mono audio.
@@ -400,11 +518,16 @@ declare class FallbackSTT extends STT {
400
518
  private readonly options;
401
519
  private stt;
402
520
  private sttIndex;
521
+ private pending;
403
522
  constructor(options: FallbackSTTOptions);
404
523
  transcribe(audioStream: Readable): void;
405
524
  destroy(): void;
406
525
  private startNextSTT;
407
526
  private onTranscript;
527
+ /**
528
+ * Hands every utterance still waiting to the next STT, one stream each so
529
+ * each gets its own transcript. One still being spoken keeps going there.
530
+ */
408
531
  private onFailed;
409
532
  }
410
533
 
@@ -427,19 +550,29 @@ declare class FallbackTTS extends TTS {
427
550
  private readonly options;
428
551
  private tts;
429
552
  private ttsIndex;
553
+ private current?;
430
554
  constructor(options: FallbackTTSOptions);
431
555
  speak(textStream: Readable): void;
432
556
  cancel(): void;
433
557
  destroy(): void;
434
558
  private startNextTTS;
435
559
  private onAudio;
560
+ /**
561
+ * Hands the unspoken text to the next TTS. An utterance whose text is still
562
+ * being written goes on there, so its end is spoken too.
563
+ */
436
564
  private onFailed;
437
565
  }
438
566
 
567
+ /**
568
+ * Plays the same recorded chunks for any answer, to run a call without a
569
+ * voice provider. The files hold raw PCM 16 bits at 16 kHz.
570
+ */
439
571
  declare class MockTTS extends TTS {
440
572
  private audioFilePaths;
573
+ private generation;
441
574
  constructor(audioFilePaths: string[]);
442
- speak(textStream: Readable): PassThrough;
575
+ speak(textStream: Readable): void;
443
576
  cancel(): void;
444
577
  }
445
578
 
@@ -583,6 +716,44 @@ interface MicdropConfig {
583
716
  * turn it sends goes to the model as audio.
584
717
  */
585
718
  realtime?: Realtime;
719
+ /**
720
+ * Classifies each turn of the user when it ends, with a `MicdropTurnInput`:
721
+ * the turn, and the turn before with its answer.
722
+ *
723
+ * It works next to the agent and leaves the answer to it. A classification
724
+ * of the turn goes in the metadata of its last user message, under
725
+ * `classification`, where `getTurnClassification()` reads it.
726
+ */
727
+ classifier?: Classifier<any, any>;
728
+ /** How the server uses the classifier */
729
+ classifierOptions?: MicdropClassifierOptions;
730
+ }
731
+ interface MicdropClassifierOptions {
732
+ /**
733
+ * Sends every classification to the client, which emits it as
734
+ * `Classification`.
735
+ *
736
+ * Off by default, since a result can hold what the user should not see, a
737
+ * manipulation score for instance.
738
+ */
739
+ sendToClient?: boolean;
740
+ /**
741
+ * Holds each answer until the classification of its turn is done, so the
742
+ * agent can read it in `onBeforeAnswer`. Off by default: the answer starts
743
+ * right away and the classification runs next to it.
744
+ */
745
+ waitBeforeAnswer?: boolean;
746
+ /**
747
+ * How long `waitBeforeAnswer` holds an answer, 1000 ms by default. A
748
+ * classification that takes longer lets the answer go without it.
749
+ */
750
+ maxWait?: number;
751
+ /**
752
+ * How many turns of the user before this one the input holds, with the
753
+ * answers that followed them. 1 by default: the turn before, its answer,
754
+ * and this turn.
755
+ */
756
+ history?: number;
586
757
  }
587
758
  declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
588
759
  socket: WebSocket$1 | null;
@@ -590,6 +761,8 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
590
761
  logger?: Logger;
591
762
  private startTime;
592
763
  private lastMessageSpeeched?;
764
+ private cancelRound;
765
+ private assistantMessages;
593
766
  private ownConversation;
594
767
  private operationQueue;
595
768
  private isProcessingQueue;
@@ -597,6 +770,7 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
597
770
  private userSpeechChunks;
598
771
  private turnComplete?;
599
772
  private heldTurnTimer?;
773
+ private turnRound;
600
774
  constructor(socket: WebSocket$1, config: MicdropConfig);
601
775
  /** Everything said so far, whether an agent or the server keeps it */
602
776
  get conversation(): MicdropConversation;
@@ -638,8 +812,22 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
638
812
  private sendFirstMessage;
639
813
  answer(): void;
640
814
  private _answer;
641
- /** Sends the answer to the client as it is written, ahead of the voice */
815
+ /**
816
+ * Sends the answer to the client as it is written, ahead of the voice.
817
+ *
818
+ * The stream reaches here a moment after the agent wrote it, so the agent
819
+ * can record the complete message first, which the client then shows. A
820
+ * partial answer arriving after it would show the answer twice, so the
821
+ * forwarding stops once the message is sent.
822
+ */
642
823
  private forwardPartialAnswer;
824
+ /**
825
+ * Classifies the turn that just ended, with the one before, and keeps the
826
+ * result in the metadata of its last message
827
+ */
828
+ private classifyTurn;
829
+ /** Waits for the classification in progress, never longer than its limit */
830
+ private waitForClassification;
643
831
  speak(message: string | Readable): void;
644
832
  private _speak;
645
833
  }
@@ -708,12 +896,162 @@ declare class SharedInstances<T> {
708
896
  * instead of loading the model twice.
709
897
  */
710
898
  load(key: unknown, create: () => Promise<T>): Promise<T>;
899
+ /** The instance for this key, loaded or being loaded, without loading it. */
900
+ get(key: unknown): Promise<T> | undefined;
711
901
  /** Whether an instance is loaded, or being loaded, for this key. */
712
902
  has(key: unknown): boolean;
903
+ /**
904
+ * Forgets the instance for this key, so the next call loads a new one. For
905
+ * an instance that broke, a crashed worker for instance. Releasing what it
906
+ * holds is left to the caller.
907
+ */
908
+ delete(key: unknown): void;
713
909
  /** Drops every instance, so the next call loads again. */
714
910
  clear(): void;
715
911
  }
716
912
 
913
+ /**
914
+ * Where a {@link ReconnectingSocket} stands.
915
+ *
916
+ * - `idle`: not connected yet, `connect()` has not been called
917
+ * - `connecting`: an attempt is under way, the first one or a retry
918
+ * - `open`: connected, and done with its handshake if it has one
919
+ * - `reconnecting`: the connection dropped, a new attempt waits for its delay
920
+ * - `failed`: every retry failed, `connect()` starts over
921
+ * - `destroyed`: closed for good
922
+ */
923
+ type ReconnectingSocketStatus = 'idle' | 'connecting' | 'open' | 'reconnecting' | 'failed' | 'destroyed';
924
+ interface ReconnectingSocketOptions {
925
+ /**
926
+ * Address to connect to, or a function giving it for each attempt, for a
927
+ * session URL or a token that has to be fresh.
928
+ */
929
+ url: string | (() => string | Promise<string>);
930
+ /** Options of the `ws` client, headers included */
931
+ clientOptions?: ClientOptions;
932
+ /** How long an attempt may take, handshake included, 5000 ms by default */
933
+ connectionTimeout?: number;
934
+ /** How long to wait before a new attempt, 1000 ms by default */
935
+ retryDelay?: number;
936
+ /** How many attempts in a row may fail before giving up, 3 by default */
937
+ maxRetry?: number;
938
+ /**
939
+ * Sets the session up before the connection counts as open, for a protocol
940
+ * that takes nothing else until then. See {@link ReconnectingSocketHandshake}.
941
+ */
942
+ handshake?: ReconnectingSocketHandshake;
943
+ /** Sends a message on a regular basis, to keep an idle connection open */
944
+ keepAlive?: {
945
+ /** Interval between two messages, in milliseconds */
946
+ interval: number;
947
+ message: string | (() => string);
948
+ };
949
+ /** Receives the log lines, the `log` method of the owner usually */
950
+ log?: (...message: any[]) => void;
951
+ }
952
+ /**
953
+ * What a connection does between the socket opening and the session being
954
+ * ready.
955
+ *
956
+ * Everything runs in the tick of the message it reads, so the connection is
957
+ * open before the next message arrives and none is lost in between.
958
+ */
959
+ interface ReconnectingSocketHandshake {
960
+ /** Sends what opens the session, as soon as the socket opens */
961
+ start?(socket: WebSocket): void;
962
+ /**
963
+ * Reads the messages that come before the session is ready, which are not
964
+ * emitted as `Message`, and returns true on the one saying it is. Throwing
965
+ * fails the attempt, for a session the server refused.
966
+ *
967
+ * Without it, the connection opens right after `start`, and proves itself
968
+ * like one without a handshake: by a first message, or by staying open.
969
+ */
970
+ read?(data: RawData, isBinary: boolean): boolean;
971
+ }
972
+ interface ReconnectingSocketEvents {
973
+ /** The status changed */
974
+ StatusChange: [ReconnectingSocketStatus];
975
+ /**
976
+ * The connection is open. A reconnection is the moment to send again what
977
+ * the previous connection did not finish.
978
+ */
979
+ Open: [{
980
+ reconnected: boolean;
981
+ }];
982
+ Message: [RawData, boolean];
983
+ /** The connection dropped, `reconnecting` or `failed` follows */
984
+ Close: [number, string];
985
+ /** Every retry failed */
986
+ Failed: [Error];
987
+ }
988
+ /**
989
+ * A WebSocket to a provider that reconnects on its own.
990
+ *
991
+ * Every connection a provider holds goes through the same states, and they
992
+ * live here, in one place: one attempt at a time, a failure counted once
993
+ * whatever the socket reports about it, a retry budget, and a `ready()` that
994
+ * settles whatever happens. A socket left behind by a reconnection is silenced,
995
+ * so it never acts on the one that replaced it.
996
+ *
997
+ * The owner keeps what belongs to its protocol: what to send again on
998
+ * `Open`, and what the messages mean.
999
+ */
1000
+ declare class ReconnectingSocket extends EventEmitter<ReconnectingSocketEvents> {
1001
+ private readonly options;
1002
+ private _status;
1003
+ private socket?;
1004
+ private everOpened;
1005
+ private retryCount;
1006
+ private proofTimer?;
1007
+ private attemptId;
1008
+ private retryTimer?;
1009
+ private keepAliveTimer?;
1010
+ private waiters;
1011
+ constructor(options: ReconnectingSocketOptions);
1012
+ get status(): ReconnectingSocketStatus;
1013
+ /** Whether a message sent now goes out */
1014
+ get isOpen(): boolean;
1015
+ /**
1016
+ * Opens the connection. Called again after `failed`, it starts over with a
1017
+ * full retry budget, and does nothing in any other status.
1018
+ */
1019
+ connect(): void;
1020
+ /**
1021
+ * Waits for the connection to be open.
1022
+ * @returns True once open, false if it failed or was destroyed meanwhile
1023
+ */
1024
+ ready(): Promise<boolean>;
1025
+ /**
1026
+ * Sends a message if the connection is open
1027
+ * @returns False when it did not go out
1028
+ */
1029
+ send(data: string | Buffer): boolean;
1030
+ /**
1031
+ * Drops the connection and opens a new one right away, for a provider that
1032
+ * can only stop what it is doing by closing. It costs no retry, and the new
1033
+ * connection is not a reconnection.
1034
+ */
1035
+ restart(): void;
1036
+ /** Closes the connection for good */
1037
+ destroy(): void;
1038
+ private attempt;
1039
+ private onOpen;
1040
+ /** The connection works, the retry budget is whole again */
1041
+ private prove;
1042
+ private clearProofTimer;
1043
+ private onAttemptFailed;
1044
+ private scheduleRetry;
1045
+ /** Lets go of the current socket, which will never be heard from again */
1046
+ private drop;
1047
+ private startKeepAlive;
1048
+ private stopKeepAlive;
1049
+ private clearRetryTimer;
1050
+ private settleWaiters;
1051
+ private setStatus;
1052
+ private log;
1053
+ }
1054
+
717
1055
  declare function waitForParams<CallParams>(socket: WebSocket$1, validate: (params: any) => CallParams): Promise<CallParams>;
718
1056
 
719
- export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, Realtime, type RealtimeEvents, type RealtimeOptions, STT, type STTEvents, SentenceSplitter, SentenceTTS, SharedInstances, TTS, type TTSEvents, type Tool, type TurnDetector, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };
1057
+ export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, Classifier, type ClassifierEvents, type ClassifierInput, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, type MicdropClassification, type MicdropClassifierOptions, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, type MicdropTurnInput, MockAgent, MockSTT, MockTTS, Pcm16Resampler, Realtime, type RealtimeEvents, type RealtimeOptions, ReconnectingSocket, type ReconnectingSocketEvents, type ReconnectingSocketHandshake, type ReconnectingSocketOptions, type ReconnectingSocketStatus, STT, type STTEvents, SentenceSplitter, SentenceTTS, SharedInstances, TTS, type TTSEvents, type Tool, type TurnDetector, currentTurn, float32ToPcm16, getTurnClassification, handleError, pcm16ToFloat32, turnInput, waitForParams };