@micdrop/client 3.2.0 → 3.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.mts CHANGED
@@ -41,6 +41,11 @@ declare abstract class MicDriver extends EventEmitter<MicDriverEvents> {
41
41
  abstract get isStarted(): boolean;
42
42
  /** Identifier of the input device in use, when the system reports one */
43
43
  abstract get deviceId(): string | undefined;
44
+ /**
45
+ * Whether the system removes the speaker from what the microphone hears.
46
+ * False when it is known to be off, undefined when the system does not say.
47
+ */
48
+ get echoCancellation(): boolean | undefined;
44
49
  /**
45
50
  * Asks for the microphone permission and starts capturing
46
51
  * @param deviceId - Input device to record from, defaults to the system one
@@ -132,6 +137,8 @@ declare class MicController extends EventEmitter<MicEvents> {
132
137
  getDriver(): MicDriver;
133
138
  get isStarted(): boolean;
134
139
  get deviceId(): string | undefined;
140
+ /** Whether echo cancellation is on, undefined when nobody knows */
141
+ get echoCancellation(): boolean | undefined;
135
142
  /**
136
143
  * Asks for the microphone permission and starts capturing
137
144
  * @param deviceId - Input device to record from, defaults to the system one
@@ -217,7 +224,8 @@ declare enum MicdropServerCommands {
217
224
  CancelLastUserMessage = "CancelLastUserMessage",
218
225
  SkipAnswer = "SkipAnswer",
219
226
  EndCall = "EndCall",
220
- ToolCall = "ToolCall"
227
+ ToolCall = "ToolCall",
228
+ Classification = "Classification"
221
229
  }
222
230
  /**
223
231
  * Hears whether a sentence has landed, where voice activity detection only
@@ -267,6 +275,15 @@ interface MicdropConversationToolResult {
267
275
  toolName: string;
268
276
  output: string;
269
277
  }
278
+ /** What a classifier made of an input, a turn of the user in a call */
279
+ interface MicdropClassification<Result = any, Input = any> {
280
+ /** What was classified */
281
+ input: Input;
282
+ /** Answers of the classifier, in the shape its provider returns */
283
+ result: Result;
284
+ /** Time the classification took, in ms */
285
+ duration: number;
286
+ }
270
287
  interface MicdropToolCall {
271
288
  name: string;
272
289
  parameters: any;
@@ -275,6 +292,22 @@ interface MicdropToolCall {
275
292
  type DeepPartial<T> = T extends object ? {
276
293
  [P in keyof T]?: DeepPartial<T[P]>;
277
294
  } : T;
295
+ /**
296
+ * The turn of the user so far: the user messages at the end of the
297
+ * conversation, one per transcript, joined with a space. It starts after the
298
+ * last answer, or after the last user message already classified, which is
299
+ * what tells two turns apart when no agent answers them.
300
+ */
301
+ declare function currentTurn(conversation: MicdropConversation): {
302
+ transcript: string;
303
+ messages: MicdropConversationMessage<MicdropAnswerMetadata>[];
304
+ before: MicdropConversationItem[];
305
+ };
306
+ /**
307
+ * The classification of the last turn of the user, which the server keeps in
308
+ * the metadata of its last message. Undefined until one read the whole turn.
309
+ */
310
+ declare function getTurnClassification<Result = any>(conversation: MicdropConversation): MicdropClassification<Result> | undefined;
278
311
 
279
312
  interface VADEvents {
280
313
  StartSpeaking: void;
@@ -334,7 +367,7 @@ declare abstract class VAD extends EventEmitter<VADEvents> {
334
367
  * Sets the status of the VAD
335
368
  * @param status - The status to set
336
369
  */
337
- private setStatus;
370
+ protected setStatus(status: VADStatus): void;
338
371
  }
339
372
 
340
373
  type VADConfigName = 'volume' | 'silero';
@@ -349,6 +382,23 @@ declare function equalVADConfig(a?: VADConfig, b?: VADConfig): boolean;
349
382
  * enough that a wrong verdict costs one awkward silence rather than the call.
350
383
  */
351
384
  declare const DEFAULT_TURN_MAX_WAIT = 4000;
385
+ /**
386
+ * How much longer than the VAD delay a turn may wait for its confirmation.
387
+ *
388
+ * The audio heard before the speech is confirmed is held back, and only its
389
+ * most recent part is kept: a VAD that stays unsure for a long time (a noisy
390
+ * room keeps the volume VAD up) would otherwise send everything since then.
391
+ */
392
+ declare const CONFIRM_MARGIN = 300;
393
+ /**
394
+ * How long the microphone stays deaf after the speaker went quiet, when the
395
+ * user may not interrupt the assistant.
396
+ *
397
+ * Without echo cancellation, what the speaker plays comes back through the
398
+ * microphone. The sound leaves the speaker a little after it was played, and
399
+ * lingers in the room.
400
+ */
401
+ declare const ECHO_TAIL = 300;
352
402
  interface MicRecorderState {
353
403
  isStarting: boolean;
354
404
  isStarted: boolean;
@@ -391,7 +441,8 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
391
441
  private reserveLength;
392
442
  private buffer;
393
443
  private bufferLength;
394
- private queuedChunks;
444
+ private echoSamples;
445
+ private speakerPlaying;
395
446
  private turnOpen;
396
447
  private turnListening;
397
448
  private turnTimer;
@@ -412,6 +463,15 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
412
463
  */
413
464
  changeTurnDetector: (turnDetector?: TurnDetector) => void;
414
465
  changeVad: (vadConfig: VADConfig) => void;
466
+ /**
467
+ * Leaves out what the microphone hears while the speaker plays, so the
468
+ * assistant is never sent back to the server as if the user had said it.
469
+ *
470
+ * Only for calls the user may not interrupt: an interruption relies on echo
471
+ * cancellation, and needs every word the user says over the assistant.
472
+ * @param playing - True while the assistant is heard
473
+ */
474
+ setSpeakerPlaying: (playing: boolean) => void;
415
475
  private onFrames;
416
476
  private onStartSpeaking;
417
477
  private onConfirmSpeaking;
@@ -600,6 +660,7 @@ declare class SileroVAD extends VAD {
600
660
  private speechFrames;
601
661
  private silenceFrames;
602
662
  private processing;
663
+ private round;
603
664
  constructor(options?: Partial<SileroVADOptions>);
604
665
  get isStarted(): boolean;
605
666
  get isPaused(): boolean;
@@ -729,6 +790,8 @@ interface MicdropEvents {
729
790
  Message: [MicdropConversationItem];
730
791
  /** What the assistant is answering, before it is finished */
731
792
  PartialAssistantMessage: [string];
793
+ /** What the classifier of the server made of what the user says */
794
+ Classification: [MicdropClassification];
732
795
  }
733
796
  interface MicdropReconnectOptions {
734
797
  maxAttempts?: number;
@@ -747,6 +810,10 @@ interface MicdropOptions {
747
810
  turnDetector?: TurnDetector;
748
811
  /** How long a turn may stay open after the detector asked to wait */
749
812
  turnMaxWait?: number;
813
+ /**
814
+ * Keeps the user from interrupting the assistant, whose voice they cannot
815
+ * speak over. Forced on when the microphone has no echo cancellation.
816
+ */
750
817
  disableInterruption?: boolean;
751
818
  debugLog?: boolean;
752
819
  reconnect?: MicdropReconnectOptions;
@@ -791,6 +858,7 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
791
858
  private _isMuted;
792
859
  private _isPaused;
793
860
  private _isReconnecting;
861
+ private _isStartingCall;
794
862
  private reconnectAttempt;
795
863
  private reconnectTimer?;
796
864
  private connectionTimer?;
@@ -809,6 +877,12 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
809
877
  get isMicMuted(): boolean;
810
878
  get isUserSpeaking(): boolean;
811
879
  get isAssistantSpeaking(): boolean;
880
+ /**
881
+ * Whether the user is kept from speaking over the assistant. Without echo
882
+ * cancellation the microphone hears the assistant, which would interrupt
883
+ * itself and end up in the transcript.
884
+ */
885
+ get isInterruptionDisabled(): boolean;
812
886
  get micDeviceId(): string | undefined;
813
887
  get speakerDeviceId(): string | undefined;
814
888
  get state(): MicdropState;
@@ -882,4 +956,4 @@ declare enum MicdropStorageKeys {
882
956
  declare function setMicdropStorage(storage: MicdropStorage): void;
883
957
  declare const storage: MicdropStorage;
884
958
 
885
- export { type AudioSink, DEFAULT_TURN_MAX_WAIT, type DeepPartial, Mic, MicController, MicDriver, type MicDriverEvents, type MicEvents, MicRecorder, type MicRecorderEvents, type MicRecorderState, type MicSource, Micdrop, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClient, MicdropClientCommands, MicdropClientError, MicdropClientErrorCode, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, type MicdropDevice, type MicdropEvents, type MicdropOptions, type MicdropReconnectOptions, MicdropServerCommands, type MicdropState, type MicdropStorage, MicdropStorageKeys, type MicdropToolCall, MultipleVAD, Pcm16AudioStream, type Pcm16AudioStreamEvents, SILERO_FRAME_SAMPLES, SILERO_SAMPLE_RATE, type ScheduledAudio, type SileroModel, type SileroModelLoader, SileroVAD, type SileroVADOptions, Speaker, SpeakerController, SpeakerDriver, type SpeakerDriverEvents, type SpeakerEvents, type TurnDetector, VAD, type VADConfig, type VADConfigName, type VADEvents, VADStatus, VOLUME_SAMPLE_RATE, VolumeMeter, VolumeVAD, type VolumeVADOptions, type WSCloseEvent, concatFloat32, equalVADConfig, floatToPcm16, getClientErrorFromWSCloseEvent, getClientErrorFromWSCloseEventCode, getVAD, getVolume, isRecoverableError, pcm16ToArrayBuffer, pcm16ToFloat, resample, setMicdropStorage, setSileroModelLoader, storage };
959
+ export { type AudioSink, CONFIRM_MARGIN, DEFAULT_TURN_MAX_WAIT, type DeepPartial, ECHO_TAIL, Mic, MicController, MicDriver, type MicDriverEvents, type MicEvents, MicRecorder, type MicRecorderEvents, type MicRecorderState, type MicSource, Micdrop, type MicdropAnswerMetadata, type MicdropCallSummary, type MicdropClassification, MicdropClient, MicdropClientCommands, MicdropClientError, MicdropClientErrorCode, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, type MicdropDevice, type MicdropEvents, type MicdropOptions, type MicdropReconnectOptions, MicdropServerCommands, type MicdropState, type MicdropStorage, MicdropStorageKeys, type MicdropToolCall, MultipleVAD, Pcm16AudioStream, type Pcm16AudioStreamEvents, SILERO_FRAME_SAMPLES, SILERO_SAMPLE_RATE, type ScheduledAudio, type SileroModel, type SileroModelLoader, SileroVAD, type SileroVADOptions, Speaker, SpeakerController, SpeakerDriver, type SpeakerDriverEvents, type SpeakerEvents, type TurnDetector, VAD, type VADConfig, type VADConfigName, type VADEvents, VADStatus, VOLUME_SAMPLE_RATE, VolumeMeter, VolumeVAD, type VolumeVADOptions, type WSCloseEvent, concatFloat32, currentTurn, equalVADConfig, floatToPcm16, getClientErrorFromWSCloseEvent, getClientErrorFromWSCloseEventCode, getTurnClassification, getVAD, getVolume, isRecoverableError, pcm16ToArrayBuffer, pcm16ToFloat, resample, setMicdropStorage, setSileroModelLoader, storage };
package/dist/index.d.ts CHANGED
@@ -41,6 +41,11 @@ declare abstract class MicDriver extends EventEmitter<MicDriverEvents> {
41
41
  abstract get isStarted(): boolean;
42
42
  /** Identifier of the input device in use, when the system reports one */
43
43
  abstract get deviceId(): string | undefined;
44
+ /**
45
+ * Whether the system removes the speaker from what the microphone hears.
46
+ * False when it is known to be off, undefined when the system does not say.
47
+ */
48
+ get echoCancellation(): boolean | undefined;
44
49
  /**
45
50
  * Asks for the microphone permission and starts capturing
46
51
  * @param deviceId - Input device to record from, defaults to the system one
@@ -132,6 +137,8 @@ declare class MicController extends EventEmitter<MicEvents> {
132
137
  getDriver(): MicDriver;
133
138
  get isStarted(): boolean;
134
139
  get deviceId(): string | undefined;
140
+ /** Whether echo cancellation is on, undefined when nobody knows */
141
+ get echoCancellation(): boolean | undefined;
135
142
  /**
136
143
  * Asks for the microphone permission and starts capturing
137
144
  * @param deviceId - Input device to record from, defaults to the system one
@@ -217,7 +224,8 @@ declare enum MicdropServerCommands {
217
224
  CancelLastUserMessage = "CancelLastUserMessage",
218
225
  SkipAnswer = "SkipAnswer",
219
226
  EndCall = "EndCall",
220
- ToolCall = "ToolCall"
227
+ ToolCall = "ToolCall",
228
+ Classification = "Classification"
221
229
  }
222
230
  /**
223
231
  * Hears whether a sentence has landed, where voice activity detection only
@@ -267,6 +275,15 @@ interface MicdropConversationToolResult {
267
275
  toolName: string;
268
276
  output: string;
269
277
  }
278
+ /** What a classifier made of an input, a turn of the user in a call */
279
+ interface MicdropClassification<Result = any, Input = any> {
280
+ /** What was classified */
281
+ input: Input;
282
+ /** Answers of the classifier, in the shape its provider returns */
283
+ result: Result;
284
+ /** Time the classification took, in ms */
285
+ duration: number;
286
+ }
270
287
  interface MicdropToolCall {
271
288
  name: string;
272
289
  parameters: any;
@@ -275,6 +292,22 @@ interface MicdropToolCall {
275
292
  type DeepPartial<T> = T extends object ? {
276
293
  [P in keyof T]?: DeepPartial<T[P]>;
277
294
  } : T;
295
+ /**
296
+ * The turn of the user so far: the user messages at the end of the
297
+ * conversation, one per transcript, joined with a space. It starts after the
298
+ * last answer, or after the last user message already classified, which is
299
+ * what tells two turns apart when no agent answers them.
300
+ */
301
+ declare function currentTurn(conversation: MicdropConversation): {
302
+ transcript: string;
303
+ messages: MicdropConversationMessage<MicdropAnswerMetadata>[];
304
+ before: MicdropConversationItem[];
305
+ };
306
+ /**
307
+ * The classification of the last turn of the user, which the server keeps in
308
+ * the metadata of its last message. Undefined until one read the whole turn.
309
+ */
310
+ declare function getTurnClassification<Result = any>(conversation: MicdropConversation): MicdropClassification<Result> | undefined;
278
311
 
279
312
  interface VADEvents {
280
313
  StartSpeaking: void;
@@ -334,7 +367,7 @@ declare abstract class VAD extends EventEmitter<VADEvents> {
334
367
  * Sets the status of the VAD
335
368
  * @param status - The status to set
336
369
  */
337
- private setStatus;
370
+ protected setStatus(status: VADStatus): void;
338
371
  }
339
372
 
340
373
  type VADConfigName = 'volume' | 'silero';
@@ -349,6 +382,23 @@ declare function equalVADConfig(a?: VADConfig, b?: VADConfig): boolean;
349
382
  * enough that a wrong verdict costs one awkward silence rather than the call.
350
383
  */
351
384
  declare const DEFAULT_TURN_MAX_WAIT = 4000;
385
+ /**
386
+ * How much longer than the VAD delay a turn may wait for its confirmation.
387
+ *
388
+ * The audio heard before the speech is confirmed is held back, and only its
389
+ * most recent part is kept: a VAD that stays unsure for a long time (a noisy
390
+ * room keeps the volume VAD up) would otherwise send everything since then.
391
+ */
392
+ declare const CONFIRM_MARGIN = 300;
393
+ /**
394
+ * How long the microphone stays deaf after the speaker went quiet, when the
395
+ * user may not interrupt the assistant.
396
+ *
397
+ * Without echo cancellation, what the speaker plays comes back through the
398
+ * microphone. The sound leaves the speaker a little after it was played, and
399
+ * lingers in the room.
400
+ */
401
+ declare const ECHO_TAIL = 300;
352
402
  interface MicRecorderState {
353
403
  isStarting: boolean;
354
404
  isStarted: boolean;
@@ -391,7 +441,8 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
391
441
  private reserveLength;
392
442
  private buffer;
393
443
  private bufferLength;
394
- private queuedChunks;
444
+ private echoSamples;
445
+ private speakerPlaying;
395
446
  private turnOpen;
396
447
  private turnListening;
397
448
  private turnTimer;
@@ -412,6 +463,15 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
412
463
  */
413
464
  changeTurnDetector: (turnDetector?: TurnDetector) => void;
414
465
  changeVad: (vadConfig: VADConfig) => void;
466
+ /**
467
+ * Leaves out what the microphone hears while the speaker plays, so the
468
+ * assistant is never sent back to the server as if the user had said it.
469
+ *
470
+ * Only for calls the user may not interrupt: an interruption relies on echo
471
+ * cancellation, and needs every word the user says over the assistant.
472
+ * @param playing - True while the assistant is heard
473
+ */
474
+ setSpeakerPlaying: (playing: boolean) => void;
415
475
  private onFrames;
416
476
  private onStartSpeaking;
417
477
  private onConfirmSpeaking;
@@ -600,6 +660,7 @@ declare class SileroVAD extends VAD {
600
660
  private speechFrames;
601
661
  private silenceFrames;
602
662
  private processing;
663
+ private round;
603
664
  constructor(options?: Partial<SileroVADOptions>);
604
665
  get isStarted(): boolean;
605
666
  get isPaused(): boolean;
@@ -729,6 +790,8 @@ interface MicdropEvents {
729
790
  Message: [MicdropConversationItem];
730
791
  /** What the assistant is answering, before it is finished */
731
792
  PartialAssistantMessage: [string];
793
+ /** What the classifier of the server made of what the user says */
794
+ Classification: [MicdropClassification];
732
795
  }
733
796
  interface MicdropReconnectOptions {
734
797
  maxAttempts?: number;
@@ -747,6 +810,10 @@ interface MicdropOptions {
747
810
  turnDetector?: TurnDetector;
748
811
  /** How long a turn may stay open after the detector asked to wait */
749
812
  turnMaxWait?: number;
813
+ /**
814
+ * Keeps the user from interrupting the assistant, whose voice they cannot
815
+ * speak over. Forced on when the microphone has no echo cancellation.
816
+ */
750
817
  disableInterruption?: boolean;
751
818
  debugLog?: boolean;
752
819
  reconnect?: MicdropReconnectOptions;
@@ -791,6 +858,7 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
791
858
  private _isMuted;
792
859
  private _isPaused;
793
860
  private _isReconnecting;
861
+ private _isStartingCall;
794
862
  private reconnectAttempt;
795
863
  private reconnectTimer?;
796
864
  private connectionTimer?;
@@ -809,6 +877,12 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
809
877
  get isMicMuted(): boolean;
810
878
  get isUserSpeaking(): boolean;
811
879
  get isAssistantSpeaking(): boolean;
880
+ /**
881
+ * Whether the user is kept from speaking over the assistant. Without echo
882
+ * cancellation the microphone hears the assistant, which would interrupt
883
+ * itself and end up in the transcript.
884
+ */
885
+ get isInterruptionDisabled(): boolean;
812
886
  get micDeviceId(): string | undefined;
813
887
  get speakerDeviceId(): string | undefined;
814
888
  get state(): MicdropState;
@@ -882,4 +956,4 @@ declare enum MicdropStorageKeys {
882
956
  declare function setMicdropStorage(storage: MicdropStorage): void;
883
957
  declare const storage: MicdropStorage;
884
958
 
885
- export { type AudioSink, DEFAULT_TURN_MAX_WAIT, type DeepPartial, Mic, MicController, MicDriver, type MicDriverEvents, type MicEvents, MicRecorder, type MicRecorderEvents, type MicRecorderState, type MicSource, Micdrop, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClient, MicdropClientCommands, MicdropClientError, MicdropClientErrorCode, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, type MicdropDevice, type MicdropEvents, type MicdropOptions, type MicdropReconnectOptions, MicdropServerCommands, type MicdropState, type MicdropStorage, MicdropStorageKeys, type MicdropToolCall, MultipleVAD, Pcm16AudioStream, type Pcm16AudioStreamEvents, SILERO_FRAME_SAMPLES, SILERO_SAMPLE_RATE, type ScheduledAudio, type SileroModel, type SileroModelLoader, SileroVAD, type SileroVADOptions, Speaker, SpeakerController, SpeakerDriver, type SpeakerDriverEvents, type SpeakerEvents, type TurnDetector, VAD, type VADConfig, type VADConfigName, type VADEvents, VADStatus, VOLUME_SAMPLE_RATE, VolumeMeter, VolumeVAD, type VolumeVADOptions, type WSCloseEvent, concatFloat32, equalVADConfig, floatToPcm16, getClientErrorFromWSCloseEvent, getClientErrorFromWSCloseEventCode, getVAD, getVolume, isRecoverableError, pcm16ToArrayBuffer, pcm16ToFloat, resample, setMicdropStorage, setSileroModelLoader, storage };
959
+ export { type AudioSink, CONFIRM_MARGIN, DEFAULT_TURN_MAX_WAIT, type DeepPartial, ECHO_TAIL, Mic, MicController, MicDriver, type MicDriverEvents, type MicEvents, MicRecorder, type MicRecorderEvents, type MicRecorderState, type MicSource, Micdrop, type MicdropAnswerMetadata, type MicdropCallSummary, type MicdropClassification, MicdropClient, MicdropClientCommands, MicdropClientError, MicdropClientErrorCode, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, type MicdropDevice, type MicdropEvents, type MicdropOptions, type MicdropReconnectOptions, MicdropServerCommands, type MicdropState, type MicdropStorage, MicdropStorageKeys, type MicdropToolCall, MultipleVAD, Pcm16AudioStream, type Pcm16AudioStreamEvents, SILERO_FRAME_SAMPLES, SILERO_SAMPLE_RATE, type ScheduledAudio, type SileroModel, type SileroModelLoader, SileroVAD, type SileroVADOptions, Speaker, SpeakerController, SpeakerDriver, type SpeakerDriverEvents, type SpeakerEvents, type TurnDetector, VAD, type VADConfig, type VADConfigName, type VADEvents, VADStatus, VOLUME_SAMPLE_RATE, VolumeMeter, VolumeVAD, type VolumeVADOptions, type WSCloseEvent, concatFloat32, currentTurn, equalVADConfig, floatToPcm16, getClientErrorFromWSCloseEvent, getClientErrorFromWSCloseEventCode, getTurnClassification, getVAD, getVolume, isRecoverableError, pcm16ToArrayBuffer, pcm16ToFloat, resample, setMicdropStorage, setSileroModelLoader, storage };