@micdrop/client 3.2.1 → 3.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.mts CHANGED
@@ -41,6 +41,11 @@ declare abstract class MicDriver extends EventEmitter<MicDriverEvents> {
41
41
  abstract get isStarted(): boolean;
42
42
  /** Identifier of the input device in use, when the system reports one */
43
43
  abstract get deviceId(): string | undefined;
44
+ /**
45
+ * Whether the system removes the speaker from what the microphone hears.
46
+ * False when it is known to be off, undefined when the system does not say.
47
+ */
48
+ get echoCancellation(): boolean | undefined;
44
49
  /**
45
50
  * Asks for the microphone permission and starts capturing
46
51
  * @param deviceId - Input device to record from, defaults to the system one
@@ -132,6 +137,8 @@ declare class MicController extends EventEmitter<MicEvents> {
132
137
  getDriver(): MicDriver;
133
138
  get isStarted(): boolean;
134
139
  get deviceId(): string | undefined;
140
+ /** Whether echo cancellation is on, undefined when nobody knows */
141
+ get echoCancellation(): boolean | undefined;
135
142
  /**
136
143
  * Asks for the microphone permission and starts capturing
137
144
  * @param deviceId - Input device to record from, defaults to the system one
@@ -217,7 +224,8 @@ declare enum MicdropServerCommands {
217
224
  CancelLastUserMessage = "CancelLastUserMessage",
218
225
  SkipAnswer = "SkipAnswer",
219
226
  EndCall = "EndCall",
220
- ToolCall = "ToolCall"
227
+ ToolCall = "ToolCall",
228
+ Classification = "Classification"
221
229
  }
222
230
  /**
223
231
  * Hears whether a sentence has landed, where voice activity detection only
@@ -267,6 +275,15 @@ interface MicdropConversationToolResult {
267
275
  toolName: string;
268
276
  output: string;
269
277
  }
278
+ /** What a classifier made of an input, a turn of the user in a call */
279
+ interface MicdropClassification<Result = any, Input = any> {
280
+ /** What was classified */
281
+ input: Input;
282
+ /** Answers of the classifier, in the shape its provider returns */
283
+ result: Result;
284
+ /** Time the classification took, in ms */
285
+ duration: number;
286
+ }
270
287
  interface MicdropToolCall {
271
288
  name: string;
272
289
  parameters: any;
@@ -275,6 +292,22 @@ interface MicdropToolCall {
275
292
  type DeepPartial<T> = T extends object ? {
276
293
  [P in keyof T]?: DeepPartial<T[P]>;
277
294
  } : T;
295
+ /**
296
+ * The turn of the user so far: the user messages at the end of the
297
+ * conversation, one per transcript, joined with a space. It starts after the
298
+ * last answer, or after the last user message already classified, which is
299
+ * what tells two turns apart when no agent answers them.
300
+ */
301
+ declare function currentTurn(conversation: MicdropConversation): {
302
+ transcript: string;
303
+ messages: MicdropConversationMessage<MicdropAnswerMetadata>[];
304
+ before: MicdropConversationItem[];
305
+ };
306
+ /**
307
+ * The classification of the last turn of the user, which the server keeps in
308
+ * the metadata of its last message. Undefined until one read the whole turn.
309
+ */
310
+ declare function getTurnClassification<Result = any>(conversation: MicdropConversation): MicdropClassification<Result> | undefined;
278
311
 
279
312
  interface VADEvents {
280
313
  StartSpeaking: void;
@@ -349,6 +382,23 @@ declare function equalVADConfig(a?: VADConfig, b?: VADConfig): boolean;
349
382
  * enough that a wrong verdict costs one awkward silence rather than the call.
350
383
  */
351
384
  declare const DEFAULT_TURN_MAX_WAIT = 4000;
385
+ /**
386
+ * How much longer than the VAD delay a turn may wait for its confirmation.
387
+ *
388
+ * The audio heard before the speech is confirmed is held back, and only its
389
+ * most recent part is kept: a VAD that stays unsure for a long time (a noisy
390
+ * room keeps the volume VAD up) would otherwise send everything since then.
391
+ */
392
+ declare const CONFIRM_MARGIN = 300;
393
+ /**
394
+ * How long the microphone stays deaf after the speaker went quiet, when the
395
+ * user may not interrupt the assistant.
396
+ *
397
+ * Without echo cancellation, what the speaker plays comes back through the
398
+ * microphone. The sound leaves the speaker a little after it was played, and
399
+ * lingers in the room.
400
+ */
401
+ declare const ECHO_TAIL = 300;
352
402
  interface MicRecorderState {
353
403
  isStarting: boolean;
354
404
  isStarted: boolean;
@@ -391,7 +441,8 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
391
441
  private reserveLength;
392
442
  private buffer;
393
443
  private bufferLength;
394
- private queuedChunks;
444
+ private echoSamples;
445
+ private speakerPlaying;
395
446
  private turnOpen;
396
447
  private turnListening;
397
448
  private turnTimer;
@@ -412,6 +463,15 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
412
463
  */
413
464
  changeTurnDetector: (turnDetector?: TurnDetector) => void;
414
465
  changeVad: (vadConfig: VADConfig) => void;
466
+ /**
467
+ * Leaves out what the microphone hears while the speaker plays, so the
468
+ * assistant is never sent back to the server as if the user had said it.
469
+ *
470
+ * Only for calls the user may not interrupt: an interruption relies on echo
471
+ * cancellation, and needs every word the user says over the assistant.
472
+ * @param playing - True while the assistant is heard
473
+ */
474
+ setSpeakerPlaying: (playing: boolean) => void;
415
475
  private onFrames;
416
476
  private onStartSpeaking;
417
477
  private onConfirmSpeaking;
@@ -730,6 +790,8 @@ interface MicdropEvents {
730
790
  Message: [MicdropConversationItem];
731
791
  /** What the assistant is answering, before it is finished */
732
792
  PartialAssistantMessage: [string];
793
+ /** What the classifier of the server made of what the user says */
794
+ Classification: [MicdropClassification];
733
795
  }
734
796
  interface MicdropReconnectOptions {
735
797
  maxAttempts?: number;
@@ -748,6 +810,10 @@ interface MicdropOptions {
748
810
  turnDetector?: TurnDetector;
749
811
  /** How long a turn may stay open after the detector asked to wait */
750
812
  turnMaxWait?: number;
813
+ /**
814
+ * Keeps the user from interrupting the assistant, whose voice they cannot
815
+ * speak over. Forced on when the microphone has no echo cancellation.
816
+ */
751
817
  disableInterruption?: boolean;
752
818
  debugLog?: boolean;
753
819
  reconnect?: MicdropReconnectOptions;
@@ -811,6 +877,12 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
811
877
  get isMicMuted(): boolean;
812
878
  get isUserSpeaking(): boolean;
813
879
  get isAssistantSpeaking(): boolean;
880
+ /**
881
+ * Whether the user is kept from speaking over the assistant. Without echo
882
+ * cancellation the microphone hears the assistant, which would interrupt
883
+ * itself and end up in the transcript.
884
+ */
885
+ get isInterruptionDisabled(): boolean;
814
886
  get micDeviceId(): string | undefined;
815
887
  get speakerDeviceId(): string | undefined;
816
888
  get state(): MicdropState;
@@ -884,4 +956,4 @@ declare enum MicdropStorageKeys {
884
956
  declare function setMicdropStorage(storage: MicdropStorage): void;
885
957
  declare const storage: MicdropStorage;
886
958
 
887
- export { type AudioSink, DEFAULT_TURN_MAX_WAIT, type DeepPartial, Mic, MicController, MicDriver, type MicDriverEvents, type MicEvents, MicRecorder, type MicRecorderEvents, type MicRecorderState, type MicSource, Micdrop, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClient, MicdropClientCommands, MicdropClientError, MicdropClientErrorCode, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, type MicdropDevice, type MicdropEvents, type MicdropOptions, type MicdropReconnectOptions, MicdropServerCommands, type MicdropState, type MicdropStorage, MicdropStorageKeys, type MicdropToolCall, MultipleVAD, Pcm16AudioStream, type Pcm16AudioStreamEvents, SILERO_FRAME_SAMPLES, SILERO_SAMPLE_RATE, type ScheduledAudio, type SileroModel, type SileroModelLoader, SileroVAD, type SileroVADOptions, Speaker, SpeakerController, SpeakerDriver, type SpeakerDriverEvents, type SpeakerEvents, type TurnDetector, VAD, type VADConfig, type VADConfigName, type VADEvents, VADStatus, VOLUME_SAMPLE_RATE, VolumeMeter, VolumeVAD, type VolumeVADOptions, type WSCloseEvent, concatFloat32, equalVADConfig, floatToPcm16, getClientErrorFromWSCloseEvent, getClientErrorFromWSCloseEventCode, getVAD, getVolume, isRecoverableError, pcm16ToArrayBuffer, pcm16ToFloat, resample, setMicdropStorage, setSileroModelLoader, storage };
959
+ export { type AudioSink, CONFIRM_MARGIN, DEFAULT_TURN_MAX_WAIT, type DeepPartial, ECHO_TAIL, Mic, MicController, MicDriver, type MicDriverEvents, type MicEvents, MicRecorder, type MicRecorderEvents, type MicRecorderState, type MicSource, Micdrop, type MicdropAnswerMetadata, type MicdropCallSummary, type MicdropClassification, MicdropClient, MicdropClientCommands, MicdropClientError, MicdropClientErrorCode, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, type MicdropDevice, type MicdropEvents, type MicdropOptions, type MicdropReconnectOptions, MicdropServerCommands, type MicdropState, type MicdropStorage, MicdropStorageKeys, type MicdropToolCall, MultipleVAD, Pcm16AudioStream, type Pcm16AudioStreamEvents, SILERO_FRAME_SAMPLES, SILERO_SAMPLE_RATE, type ScheduledAudio, type SileroModel, type SileroModelLoader, SileroVAD, type SileroVADOptions, Speaker, SpeakerController, SpeakerDriver, type SpeakerDriverEvents, type SpeakerEvents, type TurnDetector, VAD, type VADConfig, type VADConfigName, type VADEvents, VADStatus, VOLUME_SAMPLE_RATE, VolumeMeter, VolumeVAD, type VolumeVADOptions, type WSCloseEvent, concatFloat32, currentTurn, equalVADConfig, floatToPcm16, getClientErrorFromWSCloseEvent, getClientErrorFromWSCloseEventCode, getTurnClassification, getVAD, getVolume, isRecoverableError, pcm16ToArrayBuffer, pcm16ToFloat, resample, setMicdropStorage, setSileroModelLoader, storage };
package/dist/index.d.ts CHANGED
@@ -41,6 +41,11 @@ declare abstract class MicDriver extends EventEmitter<MicDriverEvents> {
41
41
  abstract get isStarted(): boolean;
42
42
  /** Identifier of the input device in use, when the system reports one */
43
43
  abstract get deviceId(): string | undefined;
44
+ /**
45
+ * Whether the system removes the speaker from what the microphone hears.
46
+ * False when it is known to be off, undefined when the system does not say.
47
+ */
48
+ get echoCancellation(): boolean | undefined;
44
49
  /**
45
50
  * Asks for the microphone permission and starts capturing
46
51
  * @param deviceId - Input device to record from, defaults to the system one
@@ -132,6 +137,8 @@ declare class MicController extends EventEmitter<MicEvents> {
132
137
  getDriver(): MicDriver;
133
138
  get isStarted(): boolean;
134
139
  get deviceId(): string | undefined;
140
+ /** Whether echo cancellation is on, undefined when nobody knows */
141
+ get echoCancellation(): boolean | undefined;
135
142
  /**
136
143
  * Asks for the microphone permission and starts capturing
137
144
  * @param deviceId - Input device to record from, defaults to the system one
@@ -217,7 +224,8 @@ declare enum MicdropServerCommands {
217
224
  CancelLastUserMessage = "CancelLastUserMessage",
218
225
  SkipAnswer = "SkipAnswer",
219
226
  EndCall = "EndCall",
220
- ToolCall = "ToolCall"
227
+ ToolCall = "ToolCall",
228
+ Classification = "Classification"
221
229
  }
222
230
  /**
223
231
  * Hears whether a sentence has landed, where voice activity detection only
@@ -267,6 +275,15 @@ interface MicdropConversationToolResult {
267
275
  toolName: string;
268
276
  output: string;
269
277
  }
278
+ /** What a classifier made of an input, a turn of the user in a call */
279
+ interface MicdropClassification<Result = any, Input = any> {
280
+ /** What was classified */
281
+ input: Input;
282
+ /** Answers of the classifier, in the shape its provider returns */
283
+ result: Result;
284
+ /** Time the classification took, in ms */
285
+ duration: number;
286
+ }
270
287
  interface MicdropToolCall {
271
288
  name: string;
272
289
  parameters: any;
@@ -275,6 +292,22 @@ interface MicdropToolCall {
275
292
  type DeepPartial<T> = T extends object ? {
276
293
  [P in keyof T]?: DeepPartial<T[P]>;
277
294
  } : T;
295
+ /**
296
+ * The turn of the user so far: the user messages at the end of the
297
+ * conversation, one per transcript, joined with a space. It starts after the
298
+ * last answer, or after the last user message already classified, which is
299
+ * what tells two turns apart when no agent answers them.
300
+ */
301
+ declare function currentTurn(conversation: MicdropConversation): {
302
+ transcript: string;
303
+ messages: MicdropConversationMessage<MicdropAnswerMetadata>[];
304
+ before: MicdropConversationItem[];
305
+ };
306
+ /**
307
+ * The classification of the last turn of the user, which the server keeps in
308
+ * the metadata of its last message. Undefined until one read the whole turn.
309
+ */
310
+ declare function getTurnClassification<Result = any>(conversation: MicdropConversation): MicdropClassification<Result> | undefined;
278
311
 
279
312
  interface VADEvents {
280
313
  StartSpeaking: void;
@@ -349,6 +382,23 @@ declare function equalVADConfig(a?: VADConfig, b?: VADConfig): boolean;
349
382
  * enough that a wrong verdict costs one awkward silence rather than the call.
350
383
  */
351
384
  declare const DEFAULT_TURN_MAX_WAIT = 4000;
385
+ /**
386
+ * How much longer than the VAD delay a turn may wait for its confirmation.
387
+ *
388
+ * The audio heard before the speech is confirmed is held back, and only its
389
+ * most recent part is kept: a VAD that stays unsure for a long time (a noisy
390
+ * room keeps the volume VAD up) would otherwise send everything since then.
391
+ */
392
+ declare const CONFIRM_MARGIN = 300;
393
+ /**
394
+ * How long the microphone stays deaf after the speaker went quiet, when the
395
+ * user may not interrupt the assistant.
396
+ *
397
+ * Without echo cancellation, what the speaker plays comes back through the
398
+ * microphone. The sound leaves the speaker a little after it was played, and
399
+ * lingers in the room.
400
+ */
401
+ declare const ECHO_TAIL = 300;
352
402
  interface MicRecorderState {
353
403
  isStarting: boolean;
354
404
  isStarted: boolean;
@@ -391,7 +441,8 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
391
441
  private reserveLength;
392
442
  private buffer;
393
443
  private bufferLength;
394
- private queuedChunks;
444
+ private echoSamples;
445
+ private speakerPlaying;
395
446
  private turnOpen;
396
447
  private turnListening;
397
448
  private turnTimer;
@@ -412,6 +463,15 @@ declare class MicRecorder extends EventEmitter<MicRecorderEvents> {
412
463
  */
413
464
  changeTurnDetector: (turnDetector?: TurnDetector) => void;
414
465
  changeVad: (vadConfig: VADConfig) => void;
466
+ /**
467
+ * Leaves out what the microphone hears while the speaker plays, so the
468
+ * assistant is never sent back to the server as if the user had said it.
469
+ *
470
+ * Only for calls the user may not interrupt: an interruption relies on echo
471
+ * cancellation, and needs every word the user says over the assistant.
472
+ * @param playing - True while the assistant is heard
473
+ */
474
+ setSpeakerPlaying: (playing: boolean) => void;
415
475
  private onFrames;
416
476
  private onStartSpeaking;
417
477
  private onConfirmSpeaking;
@@ -730,6 +790,8 @@ interface MicdropEvents {
730
790
  Message: [MicdropConversationItem];
731
791
  /** What the assistant is answering, before it is finished */
732
792
  PartialAssistantMessage: [string];
793
+ /** What the classifier of the server made of what the user says */
794
+ Classification: [MicdropClassification];
733
795
  }
734
796
  interface MicdropReconnectOptions {
735
797
  maxAttempts?: number;
@@ -748,6 +810,10 @@ interface MicdropOptions {
748
810
  turnDetector?: TurnDetector;
749
811
  /** How long a turn may stay open after the detector asked to wait */
750
812
  turnMaxWait?: number;
813
+ /**
814
+ * Keeps the user from interrupting the assistant, whose voice they cannot
815
+ * speak over. Forced on when the microphone has no echo cancellation.
816
+ */
751
817
  disableInterruption?: boolean;
752
818
  debugLog?: boolean;
753
819
  reconnect?: MicdropReconnectOptions;
@@ -811,6 +877,12 @@ declare class MicdropClient extends EventEmitter<MicdropEvents> implements Micdr
811
877
  get isMicMuted(): boolean;
812
878
  get isUserSpeaking(): boolean;
813
879
  get isAssistantSpeaking(): boolean;
880
+ /**
881
+ * Whether the user is kept from speaking over the assistant. Without echo
882
+ * cancellation the microphone hears the assistant, which would interrupt
883
+ * itself and end up in the transcript.
884
+ */
885
+ get isInterruptionDisabled(): boolean;
814
886
  get micDeviceId(): string | undefined;
815
887
  get speakerDeviceId(): string | undefined;
816
888
  get state(): MicdropState;
@@ -884,4 +956,4 @@ declare enum MicdropStorageKeys {
884
956
  declare function setMicdropStorage(storage: MicdropStorage): void;
885
957
  declare const storage: MicdropStorage;
886
958
 
887
- export { type AudioSink, DEFAULT_TURN_MAX_WAIT, type DeepPartial, Mic, MicController, MicDriver, type MicDriverEvents, type MicEvents, MicRecorder, type MicRecorderEvents, type MicRecorderState, type MicSource, Micdrop, type MicdropAnswerMetadata, type MicdropCallSummary, MicdropClient, MicdropClientCommands, MicdropClientError, MicdropClientErrorCode, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, type MicdropDevice, type MicdropEvents, type MicdropOptions, type MicdropReconnectOptions, MicdropServerCommands, type MicdropState, type MicdropStorage, MicdropStorageKeys, type MicdropToolCall, MultipleVAD, Pcm16AudioStream, type Pcm16AudioStreamEvents, SILERO_FRAME_SAMPLES, SILERO_SAMPLE_RATE, type ScheduledAudio, type SileroModel, type SileroModelLoader, SileroVAD, type SileroVADOptions, Speaker, SpeakerController, SpeakerDriver, type SpeakerDriverEvents, type SpeakerEvents, type TurnDetector, VAD, type VADConfig, type VADConfigName, type VADEvents, VADStatus, VOLUME_SAMPLE_RATE, VolumeMeter, VolumeVAD, type VolumeVADOptions, type WSCloseEvent, concatFloat32, equalVADConfig, floatToPcm16, getClientErrorFromWSCloseEvent, getClientErrorFromWSCloseEventCode, getVAD, getVolume, isRecoverableError, pcm16ToArrayBuffer, pcm16ToFloat, resample, setMicdropStorage, setSileroModelLoader, storage };
959
+ export { type AudioSink, CONFIRM_MARGIN, DEFAULT_TURN_MAX_WAIT, type DeepPartial, ECHO_TAIL, Mic, MicController, MicDriver, type MicDriverEvents, type MicEvents, MicRecorder, type MicRecorderEvents, type MicRecorderState, type MicSource, Micdrop, type MicdropAnswerMetadata, type MicdropCallSummary, type MicdropClassification, MicdropClient, MicdropClientCommands, MicdropClientError, MicdropClientErrorCode, type MicdropConversation, type MicdropConversationItem, type MicdropConversationMessage, type MicdropConversationToolCall, type MicdropConversationToolResult, type MicdropDevice, type MicdropEvents, type MicdropOptions, type MicdropReconnectOptions, MicdropServerCommands, type MicdropState, type MicdropStorage, MicdropStorageKeys, type MicdropToolCall, MultipleVAD, Pcm16AudioStream, type Pcm16AudioStreamEvents, SILERO_FRAME_SAMPLES, SILERO_SAMPLE_RATE, type ScheduledAudio, type SileroModel, type SileroModelLoader, SileroVAD, type SileroVADOptions, Speaker, SpeakerController, SpeakerDriver, type SpeakerDriverEvents, type SpeakerEvents, type TurnDetector, VAD, type VADConfig, type VADConfigName, type VADEvents, VADStatus, VOLUME_SAMPLE_RATE, VolumeMeter, VolumeVAD, type VolumeVADOptions, type WSCloseEvent, concatFloat32, currentTurn, equalVADConfig, floatToPcm16, getClientErrorFromWSCloseEvent, getClientErrorFromWSCloseEventCode, getTurnClassification, getVAD, getVolume, isRecoverableError, pcm16ToArrayBuffer, pcm16ToFloat, resample, setMicdropStorage, setSileroModelLoader, storage };