@micdrop/server 2.8.0 → 2.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.mts CHANGED
@@ -20,7 +20,8 @@ declare enum MicdropServerCommands {
20
20
  CancelLastUserMessage = "CancelLastUserMessage",
21
21
  SkipAnswer = "SkipAnswer",
22
22
  EndCall = "EndCall",
23
- ToolCall = "ToolCall"
23
+ ToolCall = "ToolCall",
24
+ Classification = "Classification"
24
25
  }
25
26
  /**
26
27
  * Hears whether a sentence has landed, where voice activity detection only
@@ -70,6 +71,15 @@ interface MicdropConversationToolResult {
70
71
  toolName: string;
71
72
  output: string;
72
73
  }
74
+ /** What a classifier made of an input, a turn of the user in a call */
75
+ interface MicdropClassification<Result = any, Input = any> {
76
+ /** What was classified */
77
+ input: Input;
78
+ /** Answers of the classifier, in the shape its provider returns */
79
+ result: Result;
80
+ /** Time the classification took, in ms */
81
+ duration: number;
82
+ }
73
83
  interface MicdropToolCall {
74
84
  name: string;
75
85
  parameters: any;
@@ -78,6 +88,22 @@ interface MicdropToolCall {
78
88
  type DeepPartial<T> = T extends object ? {
79
89
  [P in keyof T]?: DeepPartial<T[P]>;
80
90
  } : T;
91
+ /**
92
+ * The turn of the user so far: the user messages at the end of the
93
+ * conversation, one per transcript, joined with a space. It starts after the
94
+ * last answer, or after the last user message already classified, which is
95
+ * what tells two turns apart when no agent answers them.
96
+ */
97
+ declare function currentTurn(conversation: MicdropConversation): {
98
+ transcript: string;
99
+ messages: MicdropConversationMessage<MicdropAnswerMetadata>[];
100
+ before: MicdropConversationItem[];
101
+ };
102
+ /**
103
+ * The classification of the last turn of the user, which the server keeps in
104
+ * the metadata of its last message. Undefined until one read the whole turn.
105
+ */
106
+ declare function getTurnClassification<Result = any>(conversation: MicdropConversation): MicdropClassification<Result> | undefined;
81
107
 
82
108
  interface Tool<Schema extends z.ZodObject = z.ZodObject> {
83
109
  name: string;
@@ -100,7 +126,7 @@ interface AgentOptions {
100
126
  autoSemanticTurn?: boolean | string;
101
127
  autoIgnoreUserNoise?: boolean | string;
102
128
  extract?: ExtractJsonOptions | ExtractTagOptions;
103
- onBeforeAnswer?: (this: Agent, stream: Writable) => void | boolean | Promise<boolean>;
129
+ onBeforeAnswer?: (this: Agent, stream: Writable) => void | boolean | string | Promise<void | boolean | string>;
104
130
  }
105
131
  interface AgentEvents {
106
132
  Message: [MicdropConversationItem];
@@ -200,6 +226,76 @@ declare class MockAgent extends Agent {
200
226
  cancel(): void;
201
227
  }
202
228
 
229
+ /** What a classifier reads: a text, or JSON with named fields */
230
+ type ClassifierInput = string | {
231
+ [key: string]: any;
232
+ } | any[];
233
+ interface ClassifierEvents<Result, Input> {
234
+ Classification: [MicdropClassification<Result, Input>];
235
+ }
236
+ /**
237
+ * Turns an input into typed decisions: intent, mood, risk. It answers
238
+ * questions about what the user said, where an agent answers the user.
239
+ *
240
+ * `MicdropServer` classifies each turn of the user when it ends, with a
241
+ * `MicdropTurnInput`, and a classifier works on its own just as well: call
242
+ * `classify()` with any text or JSON.
243
+ *
244
+ * This class keeps track of the classifications in progress, to wait for them
245
+ * or cancel them. A provider implements `evaluate()`.
246
+ */
247
+ declare abstract class Classifier<Result = any, Input extends ClassifierInput = ClassifierInput> extends EventEmitter<ClassifierEvents<Result, Input>> {
248
+ logger?: Logger;
249
+ /** The last classification */
250
+ lastClassification?: MicdropClassification<Result, Input>;
251
+ private controllers;
252
+ private _pending?;
253
+ /**
254
+ * Answers the questions of this classifier about an input
255
+ * @param input - What to classify
256
+ * @param signal - Aborted once the result is no longer wanted
257
+ */
258
+ protected abstract evaluate(input: Input, signal: AbortSignal): Promise<Result>;
259
+ /**
260
+ * Classifies an input, and emits the result as `Classification`
261
+ * @param input - What to classify: a text, or JSON with named fields
262
+ * @returns The classification, or undefined when cancelled or failed
263
+ */
264
+ classify(input: Input): Promise<MicdropClassification<Result, Input> | undefined>;
265
+ /** The last classification started, until it is done */
266
+ get pending(): Promise<MicdropClassification<Result, Input> | undefined> | undefined;
267
+ /** Drops every classification in progress, which then emit nothing */
268
+ cancel(): void;
269
+ protected log(...message: any[]): void;
270
+ destroy(): void;
271
+ private run;
272
+ }
273
+ /** What `MicdropServer` classifies at the end of each turn of the user */
274
+ interface MicdropTurnInput {
275
+ /**
276
+ * The turn before, and the answer that followed it, so words like "it" or
277
+ * "there" can be understood
278
+ */
279
+ history: Array<{
280
+ role: 'user' | 'assistant';
281
+ text: string;
282
+ }>;
283
+ /** What the user said in this turn, one or more transcripts joined */
284
+ turn: string;
285
+ }
286
+
287
+ /**
288
+ * Builds what `MicdropServer` classifies at the end of a turn: the turn
289
+ * itself, and the turns before it with the answers that followed them.
290
+ * @param conversation - The conversation, the turn at its end
291
+ * @param history - How many turns of the user before this one to include
292
+ * @returns The input, and the last message of the turn, which keeps the result
293
+ */
294
+ declare function turnInput(conversation: MicdropConversation, history?: number): {
295
+ input: MicdropTurnInput;
296
+ message: MicdropConversationMessage<MicdropAnswerMetadata>;
297
+ };
298
+
203
299
  /**
204
300
  * Streaming linear-interpolation resampler for PCM16 mono audio.
205
301
  *
@@ -583,6 +679,44 @@ interface MicdropConfig {
583
679
  * turn it sends goes to the model as audio.
584
680
  */
585
681
  realtime?: Realtime;
682
+ /**
683
+ * Classifies each turn of the user when it ends, with a `MicdropTurnInput`:
684
+ * the turn, and the turn before with its answer.
685
+ *
686
+ * It works next to the agent and leaves the answer to it. A classification
687
+ * of the turn goes in the metadata of its last user message, under
688
+ * `classification`, where `getTurnClassification()` reads it.
689
+ */
690
+ classifier?: Classifier<any, any>;
691
+ /** How the server uses the classifier */
692
+ classifierOptions?: MicdropClassifierOptions;
693
+ }
694
+ interface MicdropClassifierOptions {
695
+ /**
696
+ * Sends every classification to the client, which emits it as
697
+ * `Classification`.
698
+ *
699
+ * Off by default, since a result can hold what the user should not see, a
700
+ * manipulation score for instance.
701
+ */
702
+ sendToClient?: boolean;
703
+ /**
704
+ * Holds each answer until the classification of its turn is done, so the
705
+ * agent can read it in `onBeforeAnswer`. Off by default: the answer starts
706
+ * right away and the classification runs next to it.
707
+ */
708
+ waitBeforeAnswer?: boolean;
709
+ /**
710
+ * How long `waitBeforeAnswer` holds an answer, 1000 ms by default. A
711
+ * classification that takes longer lets the answer go without it.
712
+ */
713
+ maxWait?: number;
714
+ /**
715
+ * How many turns of the user before this one the input holds, with the
716
+ * answers that followed them. 1 by default: the turn before, its answer,
717
+ * and this turn.
718
+ */
719
+ history?: number;
586
720
  }
587
721
  declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
588
722
  socket: WebSocket$1 | null;
@@ -640,6 +774,13 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
640
774
  private _answer;
641
775
  /** Sends the answer to the client as it is written, ahead of the voice */
642
776
  private forwardPartialAnswer;
777
+ /**
778
+ * Classifies the turn that just ended, with the one before, and keeps the
779
+ * result in the metadata of its last message
780
+ */
781
+ private classifyTurn;
782
+ /** Waits for the classification in progress, never longer than its limit */
783
+ private waitForClassification;
643
784
  speak(message: string | Readable): void;
644
785
  private _speak;
645
786
  }
@@ -716,4 +857,4 @@ declare class SharedInstances<T> {
716
857
 
717
858
  declare function waitForParams<CallParams>(socket: WebSocket$1, validate: (params: any) => CallParams): Promise<CallParams>;
718
859
 
719
- export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, Realtime, type RealtimeEvents, type RealtimeOptions, STT, type STTEvents, SentenceSplitter, SentenceTTS, SharedInstances, TTS, type TTSEvents, type Tool, type TurnDetector, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };
860
+ export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, Classifier, type ClassifierEvents, type ClassifierInput, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, type MicdropClassification, type MicdropClassifierOptions, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, type MicdropTurnInput, MockAgent, MockSTT, MockTTS, Pcm16Resampler, Realtime, type RealtimeEvents, type RealtimeOptions, STT, type STTEvents, SentenceSplitter, SentenceTTS, SharedInstances, TTS, type TTSEvents, type Tool, type TurnDetector, currentTurn, float32ToPcm16, getTurnClassification, handleError, pcm16ToFloat32, turnInput, waitForParams };
package/dist/index.d.ts CHANGED
@@ -20,7 +20,8 @@ declare enum MicdropServerCommands {
20
20
  CancelLastUserMessage = "CancelLastUserMessage",
21
21
  SkipAnswer = "SkipAnswer",
22
22
  EndCall = "EndCall",
23
- ToolCall = "ToolCall"
23
+ ToolCall = "ToolCall",
24
+ Classification = "Classification"
24
25
  }
25
26
  /**
26
27
  * Hears whether a sentence has landed, where voice activity detection only
@@ -70,6 +71,15 @@ interface MicdropConversationToolResult {
70
71
  toolName: string;
71
72
  output: string;
72
73
  }
74
+ /** What a classifier made of an input, a turn of the user in a call */
75
+ interface MicdropClassification<Result = any, Input = any> {
76
+ /** What was classified */
77
+ input: Input;
78
+ /** Answers of the classifier, in the shape its provider returns */
79
+ result: Result;
80
+ /** Time the classification took, in ms */
81
+ duration: number;
82
+ }
73
83
  interface MicdropToolCall {
74
84
  name: string;
75
85
  parameters: any;
@@ -78,6 +88,22 @@ interface MicdropToolCall {
78
88
  type DeepPartial<T> = T extends object ? {
79
89
  [P in keyof T]?: DeepPartial<T[P]>;
80
90
  } : T;
91
+ /**
92
+ * The turn of the user so far: the user messages at the end of the
93
+ * conversation, one per transcript, joined with a space. It starts after the
94
+ * last answer, or after the last user message already classified, which is
95
+ * what tells two turns apart when no agent answers them.
96
+ */
97
+ declare function currentTurn(conversation: MicdropConversation): {
98
+ transcript: string;
99
+ messages: MicdropConversationMessage<MicdropAnswerMetadata>[];
100
+ before: MicdropConversationItem[];
101
+ };
102
+ /**
103
+ * The classification of the last turn of the user, which the server keeps in
104
+ * the metadata of its last message. Undefined until one read the whole turn.
105
+ */
106
+ declare function getTurnClassification<Result = any>(conversation: MicdropConversation): MicdropClassification<Result> | undefined;
81
107
 
82
108
  interface Tool<Schema extends z.ZodObject = z.ZodObject> {
83
109
  name: string;
@@ -100,7 +126,7 @@ interface AgentOptions {
100
126
  autoSemanticTurn?: boolean | string;
101
127
  autoIgnoreUserNoise?: boolean | string;
102
128
  extract?: ExtractJsonOptions | ExtractTagOptions;
103
- onBeforeAnswer?: (this: Agent, stream: Writable) => void | boolean | Promise<boolean>;
129
+ onBeforeAnswer?: (this: Agent, stream: Writable) => void | boolean | string | Promise<void | boolean | string>;
104
130
  }
105
131
  interface AgentEvents {
106
132
  Message: [MicdropConversationItem];
@@ -200,6 +226,76 @@ declare class MockAgent extends Agent {
200
226
  cancel(): void;
201
227
  }
202
228
 
229
+ /** What a classifier reads: a text, or JSON with named fields */
230
+ type ClassifierInput = string | {
231
+ [key: string]: any;
232
+ } | any[];
233
+ interface ClassifierEvents<Result, Input> {
234
+ Classification: [MicdropClassification<Result, Input>];
235
+ }
236
+ /**
237
+ * Turns an input into typed decisions: intent, mood, risk. It answers
238
+ * questions about what the user said, where an agent answers the user.
239
+ *
240
+ * `MicdropServer` classifies each turn of the user when it ends, with a
241
+ * `MicdropTurnInput`, and a classifier works on its own just as well: call
242
+ * `classify()` with any text or JSON.
243
+ *
244
+ * This class keeps track of the classifications in progress, to wait for them
245
+ * or cancel them. A provider implements `evaluate()`.
246
+ */
247
+ declare abstract class Classifier<Result = any, Input extends ClassifierInput = ClassifierInput> extends EventEmitter<ClassifierEvents<Result, Input>> {
248
+ logger?: Logger;
249
+ /** The last classification */
250
+ lastClassification?: MicdropClassification<Result, Input>;
251
+ private controllers;
252
+ private _pending?;
253
+ /**
254
+ * Answers the questions of this classifier about an input
255
+ * @param input - What to classify
256
+ * @param signal - Aborted once the result is no longer wanted
257
+ */
258
+ protected abstract evaluate(input: Input, signal: AbortSignal): Promise<Result>;
259
+ /**
260
+ * Classifies an input, and emits the result as `Classification`
261
+ * @param input - What to classify: a text, or JSON with named fields
262
+ * @returns The classification, or undefined when cancelled or failed
263
+ */
264
+ classify(input: Input): Promise<MicdropClassification<Result, Input> | undefined>;
265
+ /** The last classification started, until it is done */
266
+ get pending(): Promise<MicdropClassification<Result, Input> | undefined> | undefined;
267
+ /** Drops every classification in progress, which then emit nothing */
268
+ cancel(): void;
269
+ protected log(...message: any[]): void;
270
+ destroy(): void;
271
+ private run;
272
+ }
273
+ /** What `MicdropServer` classifies at the end of each turn of the user */
274
+ interface MicdropTurnInput {
275
+ /**
276
+ * The turn before, and the answer that followed it, so words like "it" or
277
+ * "there" can be understood
278
+ */
279
+ history: Array<{
280
+ role: 'user' | 'assistant';
281
+ text: string;
282
+ }>;
283
+ /** What the user said in this turn, one or more transcripts joined */
284
+ turn: string;
285
+ }
286
+
287
+ /**
288
+ * Builds what `MicdropServer` classifies at the end of a turn: the turn
289
+ * itself, and the turns before it with the answers that followed them.
290
+ * @param conversation - The conversation, the turn at its end
291
+ * @param history - How many turns of the user before this one to include
292
+ * @returns The input, and the last message of the turn, which keeps the result
293
+ */
294
+ declare function turnInput(conversation: MicdropConversation, history?: number): {
295
+ input: MicdropTurnInput;
296
+ message: MicdropConversationMessage<MicdropAnswerMetadata>;
297
+ };
298
+
203
299
  /**
204
300
  * Streaming linear-interpolation resampler for PCM16 mono audio.
205
301
  *
@@ -583,6 +679,44 @@ interface MicdropConfig {
583
679
  * turn it sends goes to the model as audio.
584
680
  */
585
681
  realtime?: Realtime;
682
+ /**
683
+ * Classifies each turn of the user when it ends, with a `MicdropTurnInput`:
684
+ * the turn, and the turn before with its answer.
685
+ *
686
+ * It works next to the agent and leaves the answer to it. A classification
687
+ * of the turn goes in the metadata of its last user message, under
688
+ * `classification`, where `getTurnClassification()` reads it.
689
+ */
690
+ classifier?: Classifier<any, any>;
691
+ /** How the server uses the classifier */
692
+ classifierOptions?: MicdropClassifierOptions;
693
+ }
694
+ interface MicdropClassifierOptions {
695
+ /**
696
+ * Sends every classification to the client, which emits it as
697
+ * `Classification`.
698
+ *
699
+ * Off by default, since a result can hold what the user should not see, a
700
+ * manipulation score for instance.
701
+ */
702
+ sendToClient?: boolean;
703
+ /**
704
+ * Holds each answer until the classification of its turn is done, so the
705
+ * agent can read it in `onBeforeAnswer`. Off by default: the answer starts
706
+ * right away and the classification runs next to it.
707
+ */
708
+ waitBeforeAnswer?: boolean;
709
+ /**
710
+ * How long `waitBeforeAnswer` holds an answer, 1000 ms by default. A
711
+ * classification that takes longer lets the answer go without it.
712
+ */
713
+ maxWait?: number;
714
+ /**
715
+ * How many turns of the user before this one the input holds, with the
716
+ * answers that followed them. 1 by default: the turn before, its answer,
717
+ * and this turn.
718
+ */
719
+ history?: number;
586
720
  }
587
721
  declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
588
722
  socket: WebSocket$1 | null;
@@ -640,6 +774,13 @@ declare class MicdropServer extends EventEmitter<MicdropServerEvents> {
640
774
  private _answer;
641
775
  /** Sends the answer to the client as it is written, ahead of the voice */
642
776
  private forwardPartialAnswer;
777
+ /**
778
+ * Classifies the turn that just ended, with the one before, and keeps the
779
+ * result in the metadata of its last message
780
+ */
781
+ private classifyTurn;
782
+ /** Waits for the classification in progress, never longer than its limit */
783
+ private waitForClassification;
643
784
  speak(message: string | Readable): void;
644
785
  private _speak;
645
786
  }
@@ -716,4 +857,4 @@ declare class SharedInstances<T> {
716
857
 
717
858
  declare function waitForParams<CallParams>(socket: WebSocket$1, validate: (params: any) => CallParams): Promise<CallParams>;
718
859
 
719
- export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, MockAgent, MockSTT, MockTTS, Pcm16Resampler, Realtime, type RealtimeEvents, type RealtimeOptions, STT, type STTEvents, SentenceSplitter, SentenceTTS, SharedInstances, TTS, type TTSEvents, type Tool, type TurnDetector, float32ToPcm16, handleError, pcm16ToFloat32, waitForParams };
860
+ export { AUTO_END_CALL_PROMPT, AUTO_END_CALL_TOOL_NAME, AUTO_IGNORE_USER_NOISE_PROMPT, AUTO_IGNORE_USER_NOISE_TOOL_NAME, AUTO_SEMANTIC_TURN_PROMPT, AUTO_SEMANTIC_TURN_TOOL_NAME, Agent, type AgentEvents, type AgentOptions, type AudioMessage, Classifier, type ClassifierEvents, type ClassifierInput, type DeepPartial, type ExtractJsonOptions, type ExtractOptions, type ExtractTagOptions, FallbackAgent, type FallbackAgentOptions, FallbackSTT, type FallbackSTTOptions, FallbackTTS, type FallbackTTSOptions, Logger, type MicdropAnswerMetadata, type MicdropCallSummary, type MicdropClassification, type MicdropClassifierOptions, MicdropClientCommands, type MicdropConfig, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, MicdropError, MicdropErrorCode, MicdropRecorder, type MicdropRecorderEvents, MicdropServer, MicdropServerCommands, type MicdropServerEvents, type MicdropToolCall, type MicdropTurnInput, MockAgent, MockSTT, MockTTS, Pcm16Resampler, Realtime, type RealtimeEvents, type RealtimeOptions, STT, type STTEvents, SentenceSplitter, SentenceTTS, SharedInstances, TTS, type TTSEvents, type Tool, type TurnDetector, currentTurn, float32ToPcm16, getTurnClassification, handleError, pcm16ToFloat32, turnInput, waitForParams };