@handsforbots/voice 2.0.0-alpha.0 → 2.0.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/providers/level.d.ts +51 -0
- package/dist/providers/level.d.ts.map +1 -0
- package/dist/providers/level.js +83 -0
- package/dist/providers/level.js.map +1 -0
- package/dist/providers/media.d.ts +21 -2
- package/dist/providers/media.d.ts.map +1 -1
- package/dist/providers/media.js +32 -4
- package/dist/providers/media.js.map +1 -1
- package/dist/providers/noise.d.ts +88 -0
- package/dist/providers/noise.d.ts.map +1 -0
- package/dist/providers/noise.js +173 -0
- package/dist/providers/noise.js.map +1 -0
- package/dist/voice.d.ts +84 -3
- package/dist/voice.d.ts.map +1 -1
- package/dist/voice.js +167 -4
- package/dist/voice.js.map +1 -1
- package/package.json +9 -4
package/dist/voice.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { type PluginContext } from '@handsforbots/core';
|
|
2
|
+
import { type NoiseReport, type NoiseThresholds } from './providers/noise.js';
|
|
3
|
+
import { type LevelGateOptions } from './providers/level.js';
|
|
2
4
|
import { SpeechError, type AudioPlayer, type EchoFilter, type EndOfTurn, type SpeechEffects, type SpeechErrorCode, type SpeechToText, type TextToSpeech, type TtsTraceEvent } from './types.js';
|
|
3
5
|
export type VoiceMode = 'push-to-talk' | 'hands-free';
|
|
4
6
|
export type InputModality = 'keyboard' | 'voice';
|
|
@@ -79,8 +81,56 @@ export type VoiceState = {
|
|
|
79
81
|
echo?: string;
|
|
80
82
|
/** Speech effects in use, if any. */
|
|
81
83
|
effects?: string;
|
|
84
|
+
/** Last noise check (`noise` option or `checkNoise()`). */
|
|
85
|
+
noise?: NoiseReport;
|
|
86
|
+
/** A noise check is running: the microphone is being measured, not listened to. */
|
|
87
|
+
checkingNoise: boolean;
|
|
88
|
+
/**
|
|
89
|
+
* Hands-free was turned into push-to-talk because the room was too noisy.
|
|
90
|
+
* Cleared when the mode is set again.
|
|
91
|
+
*/
|
|
92
|
+
noiseFallback: boolean;
|
|
93
|
+
/** Level gate: the user's learned speech level (dBFS), once an utterance was accepted. */
|
|
94
|
+
userLevelDb?: number;
|
|
82
95
|
error?: SpeechError;
|
|
83
96
|
};
|
|
97
|
+
/**
|
|
98
|
+
* Checks the room before hands-free listening (see the "Noisy rooms" section
|
|
99
|
+
* of the conversation guide). Off unless configured: it opens the microphone for a moment.
|
|
100
|
+
*/
|
|
101
|
+
export type VoiceNoiseOptions = {
|
|
102
|
+
/** `hands-free` (default): before hands-free listening starts. `manual`: only `checkNoise()`. */
|
|
103
|
+
check?: 'hands-free' | 'manual';
|
|
104
|
+
/** `push-to-talk` (default): switch to push-to-talk when too noisy and do not start listening. `warn`: only report. */
|
|
105
|
+
tooNoisy?: 'push-to-talk' | 'warn';
|
|
106
|
+
/** A report newer than this is reused instead of measuring again. Default 300000 (5 min). */
|
|
107
|
+
maxAgeMs?: number;
|
|
108
|
+
/** Passed to `checkNoise`. */
|
|
109
|
+
durationMs?: number;
|
|
110
|
+
thresholds?: NoiseThresholds;
|
|
111
|
+
processing?: boolean;
|
|
112
|
+
/** Replaces the measurement (a model, a server, a test). Default `checkNoise`. */
|
|
113
|
+
measure?: (signal: AbortSignal) => Promise<NoiseReport>;
|
|
114
|
+
};
|
|
115
|
+
/** A noise check finished; `action` says what the voice did about it. */
|
|
116
|
+
export type VoiceNoiseEvent = {
|
|
117
|
+
report: NoiseReport;
|
|
118
|
+
action?: 'push-to-talk';
|
|
119
|
+
};
|
|
120
|
+
/** What the level gate did, for debugging (levels in dBFS, no audio, no text). */
|
|
121
|
+
export type VoiceLevelEvent = {
|
|
122
|
+
event: 'monitor.open';
|
|
123
|
+
} | {
|
|
124
|
+
event: 'monitor.close';
|
|
125
|
+
} | {
|
|
126
|
+
event: 'learned';
|
|
127
|
+
levelDb: number;
|
|
128
|
+
userLevelDb: number;
|
|
129
|
+
gateDb: number;
|
|
130
|
+
} | {
|
|
131
|
+
event: 'learn.skipped';
|
|
132
|
+
reason: 'bot-speech' | 'no-audio';
|
|
133
|
+
};
|
|
84
134
|
export type ListenOptions = {
|
|
85
135
|
/**
|
|
86
136
|
* When a push-to-talk utterance ends. `silence` (default): the first pause
|
|
@@ -115,13 +165,23 @@ export type BargeInOptions = {
|
|
|
115
165
|
/** Backchannels that never interrupt ("aham", "ok"…). Default: a list for the language. */
|
|
116
166
|
ignore?: string[];
|
|
117
167
|
};
|
|
118
|
-
|
|
168
|
+
/** Level gate readings attached to a decision when the gate is on (dBFS). */
|
|
169
|
+
export type VoiceLevelReading = {
|
|
170
|
+
/** Loud end of the audio heard in the lookback window. */
|
|
171
|
+
levelDb?: number;
|
|
172
|
+
/** The user's learned level, if learned yet. */
|
|
173
|
+
userLevelDb?: number;
|
|
174
|
+
};
|
|
175
|
+
export type VoiceDecision = ({
|
|
119
176
|
kind: 'interrupt';
|
|
120
177
|
text: string;
|
|
121
|
-
} | {
|
|
178
|
+
} & VoiceLevelReading) | {
|
|
122
179
|
kind: 'ignore';
|
|
123
180
|
text: string;
|
|
124
|
-
reason: 'backchannel' | 'too-short' | 'echo';
|
|
181
|
+
reason: 'backchannel' | 'too-short' | 'echo' | 'background';
|
|
182
|
+
/** For `background`: the level heard and the user's learned level, in dBFS. */
|
|
183
|
+
levelDb?: number;
|
|
184
|
+
userLevelDb?: number;
|
|
125
185
|
/** For backchannels: the reply being spoken and how many of its characters were spoken then (ADR 0015). */
|
|
126
186
|
messageId?: string;
|
|
127
187
|
at?: number;
|
|
@@ -226,6 +286,12 @@ export type VoiceService = {
|
|
|
226
286
|
unlockAudio(): void;
|
|
227
287
|
/** Whether a reply arriving now would be spoken (`output` is `voice`, or `auto` after voice input). */
|
|
228
288
|
willSpeakReplies(): boolean;
|
|
289
|
+
/**
|
|
290
|
+
* Measures the room now (the microphone opens for a moment) and stores the
|
|
291
|
+
* report in `noise`. Resolves `undefined` if the measurement failed (the
|
|
292
|
+
* error goes to the `error` event, `source: 'voice:noise'`).
|
|
293
|
+
*/
|
|
294
|
+
checkNoise(): Promise<NoiseReport | undefined>;
|
|
229
295
|
};
|
|
230
296
|
declare module '@handsforbots/core' {
|
|
231
297
|
interface Services {
|
|
@@ -240,6 +306,10 @@ declare module '@handsforbots/core' {
|
|
|
240
306
|
'voice.track': VoiceTrackEvent;
|
|
241
307
|
/** Each speech provider loading its model and, when it traces, its internals (for logs and debugging). */
|
|
242
308
|
'voice.tts': VoiceTtsEvent;
|
|
309
|
+
/** A noise check finished, and what the voice did about it. */
|
|
310
|
+
'voice.noise': VoiceNoiseEvent;
|
|
311
|
+
/** The level gate opened or closed its monitor, learned the user's level, or skipped learning. */
|
|
312
|
+
'voice.level': VoiceLevelEvent;
|
|
243
313
|
}
|
|
244
314
|
}
|
|
245
315
|
export type VoiceOptions = {
|
|
@@ -277,6 +347,17 @@ export type VoiceOptions = {
|
|
|
277
347
|
* the reply each came (ADR 0015). They never interrupt. Default true.
|
|
278
348
|
*/
|
|
279
349
|
reactions?: boolean;
|
|
350
|
+
/**
|
|
351
|
+
* Measures the room before hands-free listening and falls back to
|
|
352
|
+
* push-to-talk when it is too noisy. `true` for the defaults. Off by default.
|
|
353
|
+
*/
|
|
354
|
+
noise?: boolean | VoiceNoiseOptions;
|
|
355
|
+
/**
|
|
356
|
+
* Hands-free: drops voices quieter than the user's own (TV, people nearby)
|
|
357
|
+
* by loudness, learned from the user's accepted utterances. Opens a second
|
|
358
|
+
* microphone capture while listening. `true` for the defaults. Off by default.
|
|
359
|
+
*/
|
|
360
|
+
levelGate?: boolean | LevelGateOptions;
|
|
280
361
|
};
|
|
281
362
|
export declare const voice: (config?: VoiceOptions | undefined) => import("@handsforbots/core").Plugin<VoiceOptions | undefined>;
|
|
282
363
|
export declare function createVoice(ctx: PluginContext, options: VoiceOptions): Promise<VoiceService>;
|
package/dist/voice.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"voice.d.ts","sourceRoot":"","sources":["../src/voice.ts"],"names":[],"mappings":"AAAA,OAAO,EAA+C,KAAK,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAGpG,OAAO,EACL,WAAW,EACX,KAAK,WAAW,EAChB,KAAK,UAAU,EACf,KAAK,SAAS,EAGd,KAAK,aAAa,EAClB,KAAK,eAAe,EACpB,KAAK,YAAY,EAGjB,KAAK,YAAY,EACjB,KAAK,aAAa,EAEnB,MAAM,YAAY,CAAA;AAEnB,MAAM,MAAM,SAAS,GAAG,cAAc,GAAG,YAAY,CAAA;AACrD,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,OAAO,CAAA;AAChD,gFAAgF;AAChF,MAAM,MAAM,gBAAgB,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAA;AACxD,kEAAkE;AAClE,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,WAAW,GAAG,cAAc,GAAG,kBAAkB,GAAG,aAAa,CAAA;AAEnG,uFAAuF;AACvF,MAAM,MAAM,SAAS,GAAG;IACtB,KAAK,EAAE,UAAU,GAAG,SAAS,GAAG,OAAO,GAAG,QAAQ,CAAA;IAClD,+BAA+B;IAC/B,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;CACf,CAAA;AAED;;;;;;GAMG;AACH,MAAM,MAAM,eAAe,GAAG,OAAO,GAAG,MAAM,GAAG,WAAW,GAAG,QAAQ,CAAA;AAEvE;;;;;GAKG;AACH,MAAM,MAAM,kBAAkB,GAAG,UAAU,GAAG,MAAM,GAAG,QAAQ,CAAA;AAE/D,MAAM,MAAM,aAAa,GAAG;IAAE,KAAK,CAAC,EAAE,eAAe,CAAC;IAAC,YAAY,CAAC,EAAE,kBAAkB,CAAA;CAAE,CAAA;AAE1F,MAAM,MAAM,UAAU,GAAG;IACvB,SAAS,EAAE;QAAE,GAAG,EAAE,OAAO,CAAC;QAAC,GAAG,EAAE,OAAO,CAAA;KAAE,CAAA;IACzC,IAAI,EAAE,SAAS,CAAA;IACf,KAAK,EAAE,UAAU,CAAA;IACjB,SAAS,EAAE,OAAO,CAAA;IAClB,QAAQ,EAAE,OAAO,CAAA;IACjB,kFAAkF;IAClF,OAAO,EAAE,MAAM,CAAA;IACf;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,gGAAgG;IAChG,WAAW,EAAE,OAAO,CAAA;IACpB,2CAA2C;IAC3C,SAAS,EAAE,aAAa,CAAA;IACxB,MAAM,EAAE,gBAAgB,CAAA;IACxB,wBAAwB;IACxB,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,yFAAyF;IACzF,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;IACpC,sFAAsF;IACtF,YAAY,EAAE,OAAO,CAAA;IACrB,2CAA2C;IAC3C,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,kCAAkC;IAClC,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,qCAAqC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,WAAW,CAAA;CACpB,CAAA;AAED,MAAM,MAAM,aAAa,GAAG;IAC1B;;;;;OAKG;IACH,KAAK,CAAC,EAAE,SAAS,GAAG,MAAM,CAAA;CAC3B,CAAA;AAED,wEAAwE;AACxE,MAAM,MAAM,UAAU,GAAG;IACvB;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,6DAA6D;IAC7D,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,2HAA2H;IAC3H,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,sFAAsF;IACtF,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,qFAAqF;IACrF,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,4DAA4D;IAC5D,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB,CAAA;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,iGAAiG;IACjG,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,2FAA2F;IAC3F,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;CAClB,CAAA;AAED,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,
|
|
1
|
+
{"version":3,"file":"voice.d.ts","sourceRoot":"","sources":["../src/voice.ts"],"names":[],"mappings":"AAAA,OAAO,EAA+C,KAAK,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAGpG,OAAO,EAAc,KAAK,WAAW,EAAE,KAAK,eAAe,EAAE,MAAM,sBAAsB,CAAA;AACzF,OAAO,EAAiC,KAAK,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AAC3F,OAAO,EACL,WAAW,EACX,KAAK,WAAW,EAChB,KAAK,UAAU,EACf,KAAK,SAAS,EAGd,KAAK,aAAa,EAClB,KAAK,eAAe,EACpB,KAAK,YAAY,EAGjB,KAAK,YAAY,EACjB,KAAK,aAAa,EAEnB,MAAM,YAAY,CAAA;AAEnB,MAAM,MAAM,SAAS,GAAG,cAAc,GAAG,YAAY,CAAA;AACrD,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,OAAO,CAAA;AAChD,gFAAgF;AAChF,MAAM,MAAM,gBAAgB,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAA;AACxD,kEAAkE;AAClE,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,WAAW,GAAG,cAAc,GAAG,kBAAkB,GAAG,aAAa,CAAA;AAEnG,uFAAuF;AACvF,MAAM,MAAM,SAAS,GAAG;IACtB,KAAK,EAAE,UAAU,GAAG,SAAS,GAAG,OAAO,GAAG,QAAQ,CAAA;IAClD,+BAA+B;IAC/B,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;CACf,CAAA;AAED;;;;;;GAMG;AACH,MAAM,MAAM,eAAe,GAAG,OAAO,GAAG,MAAM,GAAG,WAAW,GAAG,QAAQ,CAAA;AAEvE;;;;;GAKG;AACH,MAAM,MAAM,kBAAkB,GAAG,UAAU,GAAG,MAAM,GAAG,QAAQ,CAAA;AAE/D,MAAM,MAAM,aAAa,GAAG;IAAE,KAAK,CAAC,EAAE,eAAe,CAAC;IAAC,YAAY,CAAC,EAAE,kBAAkB,CAAA;CAAE,CAAA;AAE1F,MAAM,MAAM,UAAU,GAAG;IACvB,SAAS,EAAE;QAAE,GAAG,EAAE,OAAO,CAAC;QAAC,GAAG,EAAE,OAAO,CAAA;KAAE,CAAA;IACzC,IAAI,EAAE,SAAS,CAAA;IACf,KAAK,EAAE,UAAU,CAAA;IACjB,SAAS,EAAE,OAAO,CAAA;IAClB,QAAQ,EAAE,OAAO,CAAA;IACjB,kFAAkF;IAClF,OAAO,EAAE,MAAM,CAAA;IACf;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,gGAAgG;IAChG,WAAW,EAAE,OAAO,CAAA;IACpB,2CAA2C;IAC3C,SAAS,EAAE,aAAa,CAAA;IACxB,MAAM,EAAE,gBAAgB,CAAA;IACxB,wBAAwB;IACxB,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,yFAAyF;IACzF,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;IACpC,sFAAsF;IACtF,YAAY,EAAE,OAAO,CAAA;IACrB,2CAA2C;IAC3C,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,kCAAkC;IAClC,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,qCAAqC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,2DAA2D;IAC3D,KAAK,CAAC,EAAE,WAAW,CAAA;IACnB,mFAAmF;IACnF,aAAa,EAAE,OAAO,CAAA;IACtB;;;OAGG;IACH,aAAa,EAAE,OAAO,CAAA;IACtB,0FAA0F;IAC1F,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,KAAK,CAAC,EAAE,WAAW,CAAA;CACpB,CAAA;AAED;;;GAGG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,iGAAiG;IACjG,KAAK,CAAC,EAAE,YAAY,GAAG,QAAQ,CAAA;IAC/B,uHAAuH;IACvH,QAAQ,CAAC,EAAE,cAAc,GAAG,MAAM,CAAA;IAClC,6FAA6F;IAC7F,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,8BAA8B;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,eAAe,CAAA;IAC5B,UAAU,CAAC,EAAE,OAAO,CAAA;IACpB,kFAAkF;IAClF,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,WAAW,CAAC,CAAA;CACxD,CAAA;AAED,yEAAyE;AACzE,MAAM,MAAM,eAAe,GAAG;IAAE,MAAM,EAAE,WAAW,CAAC;IAAC,MAAM,CAAC,EAAE,cAAc,CAAA;CAAE,CAAA;AAE9E,kFAAkF;AAClF,MAAM,MAAM,eAAe,GACvB;IAAE,KAAK,EAAE,cAAc,CAAA;CAAE,GACzB;IAAE,KAAK,EAAE,eAAe,CAAA;CAAE,GAC1B;IAAE,KAAK,EAAE,SAAS,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAC1E;IAAE,KAAK,EAAE,eAAe,CAAC;IAAC,MAAM,EAAE,YAAY,GAAG,UAAU,CAAA;CAAE,CAAA;AAEjE,MAAM,MAAM,aAAa,GAAG;IAC1B;;;;;OAKG;IACH,KAAK,CAAC,EAAE,SAAS,GAAG,MAAM,CAAA;CAC3B,CAAA;AAED,wEAAwE;AACxE,MAAM,MAAM,UAAU,GAAG;IACvB;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,6DAA6D;IAC7D,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,2HAA2H;IAC3H,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,sFAAsF;IACtF,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,qFAAqF;IACrF,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,4DAA4D;IAC5D,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB,CAAA;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,iGAAiG;IACjG,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,2FAA2F;IAC3F,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;CAClB,CAAA;AAED,6EAA6E;AAC7E,MAAM,MAAM,iBAAiB,GAAG;IAC9B,0DAA0D;IAC1D,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,gDAAgD;IAChD,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB,CAAA;AAED,MAAM,MAAM,aAAa,GACrB,CAAC;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,iBAAiB,CAAC,GACzD;IACE,IAAI,EAAE,QAAQ,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,aAAa,GAAG,WAAW,GAAG,MAAM,GAAG,YAAY,CAAA;IAC3D,+EAA+E;IAC/E,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,2GAA2G;IAC3G,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,EAAE,CAAC,EAAE,MAAM,CAAA;CACZ,GACD;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,KAAK,GAAG,OAAO,GAAG,SAAS,GAAG,aAAa,GAAG,QAAQ,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,GAC3H;IAAE,IAAI,EAAE,yBAAyB,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAA;AAEzE;;;GAGG;AACH,MAAM,MAAM,gBAAgB,GAAG,CAC3B;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACnD;IAAE,KAAK,EAAE,KAAK,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,OAAO,CAAA;CAAE,GACrE;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,eAAe,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE;AAC/E,yGAAyG;GACvG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,aAAa,GAAG,WAAW,CAAA;CAAE,CACzD,GACC,OAAO,CAAC,WAAW,CAAC,CAAA;AAEtB;;;;GAIG;AACH,MAAM,MAAM,WAAW,GAAG;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,CAAA;AAEzE,wGAAwG;AACxG,MAAM,MAAM,eAAe,GAAG;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAA;AAEnE;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,GAAG,aAAa,CAAA;AAEhE,MAAM,MAAM,YAAY,GAAG;IACzB,QAAQ,IAAI,UAAU,CAAA;IACtB,SAAS,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAA;IAC3C,uEAAuE;IACvE,MAAM,CAAC,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC9C,2EAA2E;IAC3E,IAAI,IAAI,IAAI,CAAA;IACZ,yEAAyE;IACzE,MAAM,IAAI,IAAI,CAAA;IACd,8EAA8E;IAC9E,OAAO,IAAI,IAAI,CAAA;IACf,+DAA+D;IAC/D,WAAW,IAAI,IAAI,CAAA;IACnB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IACvB,OAAO,CAAC,IAAI,EAAE,SAAS,GAAG,IAAI,CAAA;IAC9B,SAAS,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAA;IACzC,iFAAiF;IACjF,YAAY,CAAC,QAAQ,EAAE,SAAS,GAAG,IAAI,GAAG,IAAI,CAAA;IAC9C,0DAA0D;IAC1D,OAAO,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI,GAAG,IAAI,CAAA;IACxC,+CAA+C;IAC/C,SAAS,CAAC,OAAO,EAAE;QAAE,IAAI,CAAC,EAAE,UAAU,CAAC;QAAC,OAAO,CAAC,EAAE,OAAO,GAAG,cAAc,CAAA;KAAE,GAAG,IAAI,CAAA;IACnF,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAClC,YAAY,IAAI,IAAI,CAAA;IACpB;;;;OAIG;IACH,UAAU,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;IAC3C,8EAA8E;IAC9E,MAAM,CAAC,GAAG,EAAE,YAAY,GAAG,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACzD,+DAA+D;IAC/D,UAAU,CAAC,OAAO,EAAE,aAAa,GAAG,IAAI,GAAG,IAAI,CAAA;IAC/C,uFAAuF;IACvF,WAAW,IAAI,IAAI,CAAA;IACnB,uGAAuG;IACvG,gBAAgB,IAAI,OAAO,CAAA;IAC3B;;;;OAIG;IACH,UAAU,IAAI,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAAA;CAC/C,CAAA;AAED,OAAO,QAAQ,oBAAoB,CAAC;IAClC,UAAU,QAAQ;QAChB,KAAK,EAAE,YAAY,CAAA;KACpB;IACD,UAAU,MAAM;QACd,mFAAmF;QACnF,gBAAgB,EAAE,aAAa,CAAA;QAC/B,oHAAoH;QACpH,cAAc,EAAE,gBAAgB,CAAA;QAChC,6GAA6G;QAC7G,aAAa,EAAE,eAAe,CAAA;QAC9B,0GAA0G;QAC1G,WAAW,EAAE,aAAa,CAAA;QAC1B,+DAA+D;QAC/D,aAAa,EAAE,eAAe,CAAA;QAC9B,kGAAkG;QAClG,aAAa,EAAE,eAAe,CAAA;KAC/B;CACF;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,4FAA4F;IAC5F,GAAG,CAAC,EAAE,YAAY,GAAG,YAAY,EAAE,CAAA;IACnC,GAAG,CAAC,EAAE,YAAY,GAAG,YAAY,EAAE,CAAA;IACnC,uEAAuE;IACvE,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,IAAI,CAAC,EAAE,SAAS,CAAA;IAChB,MAAM,CAAC,EAAE,gBAAgB,CAAA;IACzB;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,GAAG,cAAc,CAAA;IAClC,yFAAyF;IACzF,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI,CAAA;IAC5B;;;;OAIG;IACH,IAAI,CAAC,EAAE,UAAU,GAAG,KAAK,CAAA;IACzB,IAAI,CAAC,EAAE,UAAU,CAAA;IACjB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,uHAAuH;IACvH,IAAI,CAAC,EAAE,aAAa,CAAA;IACpB,gFAAgF;IAChF,OAAO,CAAC,EAAE,aAAa,GAAG,IAAI,CAAA;IAC9B,4DAA4D;IAC5D,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAA;IACnB;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,GAAG,iBAAiB,CAAA;IACnC;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,GAAG,gBAAgB,CAAA;CACvC,CAAA;AA0BD,eAAO,MAAM,KAAK,sGAOhB,CAAA;AAOF,wBAAsB,WAAW,CAAC,GAAG,EAAE,aAAa,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,CA+uClG"}
|
package/dist/voice.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { definePlugin, textOf } from '@handsforbots/core';
|
|
2
2
|
import { planProsody, webAudioPlayer } from './player.js';
|
|
3
3
|
import { textEchoFilter } from './providers/echo.js';
|
|
4
|
+
import { checkNoise } from './providers/noise.js';
|
|
5
|
+
import { createLevelGate, monitorLevel } from './providers/level.js';
|
|
4
6
|
import { SpeechError, } from './types.js';
|
|
5
7
|
const toArray = (value) => (value === undefined ? [] : Array.isArray(value) ? value : [value]);
|
|
6
8
|
const BACKCHANNELS = {
|
|
@@ -75,17 +77,75 @@ export async function createVoice(ctx, options) {
|
|
|
75
77
|
ttsStatus: {},
|
|
76
78
|
synthesizing: false,
|
|
77
79
|
effects: options.effects?.name,
|
|
80
|
+
checkingNoise: false,
|
|
81
|
+
noiseFallback: false,
|
|
78
82
|
};
|
|
79
83
|
let echo = options.echo === false ? undefined : (options.echo ?? textEchoFilter());
|
|
80
84
|
state.echo = echo?.name;
|
|
81
85
|
const listeners = new Set();
|
|
86
|
+
/** When the bot last stopped speaking (epoch ms): its voice may still be in the level window. */
|
|
87
|
+
let spokeUntil = 0;
|
|
82
88
|
const set = (patch) => {
|
|
83
89
|
const next = { ...state, ...patch };
|
|
90
|
+
if (state.speaking && !next.speaking)
|
|
91
|
+
spokeUntil = Date.now();
|
|
84
92
|
state = { ...next, phase: phaseOf(next) };
|
|
85
93
|
for (const listener of listeners)
|
|
86
94
|
listener();
|
|
87
95
|
};
|
|
88
96
|
const decide = (decision) => ctx.emit('voice.decision', decision);
|
|
97
|
+
/* Noise check ---------------------------------------------------------- */
|
|
98
|
+
const noiseOptions = options.noise === true ? {} : options.noise ? options.noise : undefined;
|
|
99
|
+
let noiseRun;
|
|
100
|
+
let noiseAt = 0;
|
|
101
|
+
const measureNoise = () => {
|
|
102
|
+
if (noiseRun)
|
|
103
|
+
return noiseRun.done;
|
|
104
|
+
const controller = new AbortController();
|
|
105
|
+
const opts = noiseOptions ?? {};
|
|
106
|
+
const measure = opts.measure ??
|
|
107
|
+
((signal) => checkNoise({ durationMs: opts.durationMs, thresholds: opts.thresholds, processing: opts.processing, signal }));
|
|
108
|
+
set({ checkingNoise: true });
|
|
109
|
+
const done = (async () => {
|
|
110
|
+
try {
|
|
111
|
+
const report = await measure(controller.signal);
|
|
112
|
+
noiseAt = Date.now();
|
|
113
|
+
set({ noise: report });
|
|
114
|
+
return report;
|
|
115
|
+
}
|
|
116
|
+
catch (error) {
|
|
117
|
+
if (!controller.signal.aborted)
|
|
118
|
+
ctx.emit('error', { error, source: 'voice:noise' });
|
|
119
|
+
return undefined;
|
|
120
|
+
}
|
|
121
|
+
finally {
|
|
122
|
+
noiseRun = undefined;
|
|
123
|
+
set({ checkingNoise: false });
|
|
124
|
+
}
|
|
125
|
+
})();
|
|
126
|
+
noiseRun = { controller, done };
|
|
127
|
+
return done;
|
|
128
|
+
};
|
|
129
|
+
/** Before hands-free listening: false when listening should not start. */
|
|
130
|
+
const noiseAllowsListening = async () => {
|
|
131
|
+
if (!noiseOptions || (noiseOptions.check ?? 'hands-free') !== 'hands-free' || state.mode !== 'hands-free')
|
|
132
|
+
return true;
|
|
133
|
+
// Another listen() is already waiting for this check and will start the session.
|
|
134
|
+
if (noiseRun)
|
|
135
|
+
return false;
|
|
136
|
+
const fresh = state.noise && Date.now() - noiseAt < (noiseOptions.maxAgeMs ?? 300_000);
|
|
137
|
+
const report = fresh ? state.noise : await measureNoise();
|
|
138
|
+
if (!report || !wantListening)
|
|
139
|
+
return wantListening;
|
|
140
|
+
const fallback = report.verdict === 'too-noisy' && (noiseOptions.tooNoisy ?? 'push-to-talk') === 'push-to-talk';
|
|
141
|
+
if (!fresh)
|
|
142
|
+
ctx.emit('voice.noise', fallback ? { report, action: 'push-to-talk' } : { report });
|
|
143
|
+
if (!fallback)
|
|
144
|
+
return true;
|
|
145
|
+
wantListening = false;
|
|
146
|
+
set({ mode: 'push-to-talk', noiseFallback: true });
|
|
147
|
+
return false;
|
|
148
|
+
};
|
|
89
149
|
/** Shows the error in the state (for the UI) and on the kernel bus (for the developer). */
|
|
90
150
|
const fail = (error, patch = {}, source = 'voice') => {
|
|
91
151
|
set({ ...patch, error });
|
|
@@ -266,6 +326,85 @@ export async function createVoice(ctx, options) {
|
|
|
266
326
|
return text;
|
|
267
327
|
}
|
|
268
328
|
};
|
|
329
|
+
/* Level gate ----------------------------------------------------------- */
|
|
330
|
+
const gateOptions = options.levelGate === true ? {} : options.levelGate ? options.levelGate : undefined;
|
|
331
|
+
const levelGate = gateOptions ? createLevelGate(gateOptions) : undefined;
|
|
332
|
+
let monitor;
|
|
333
|
+
const openMonitor = () => {
|
|
334
|
+
if (!levelGate || monitor || state.mode !== 'hands-free')
|
|
335
|
+
return;
|
|
336
|
+
const entry = { closed: false };
|
|
337
|
+
monitor = entry;
|
|
338
|
+
const open = gateOptions?.open ?? ((onLevel) => monitorLevel(onLevel));
|
|
339
|
+
// The bot's own voice is not the user's: it is measured, but never learned from.
|
|
340
|
+
open((db) => levelGate.push(db))
|
|
341
|
+
.then((close) => {
|
|
342
|
+
if (entry.closed)
|
|
343
|
+
return close();
|
|
344
|
+
entry.close = close;
|
|
345
|
+
ctx.emit('voice.level', { event: 'monitor.open' });
|
|
346
|
+
})
|
|
347
|
+
.catch((error) => {
|
|
348
|
+
if (monitor === entry)
|
|
349
|
+
monitor = undefined;
|
|
350
|
+
ctx.emit('error', { error, source: 'voice:level-gate' });
|
|
351
|
+
});
|
|
352
|
+
};
|
|
353
|
+
const closeMonitor = () => {
|
|
354
|
+
const entry = monitor;
|
|
355
|
+
monitor = undefined;
|
|
356
|
+
if (!entry)
|
|
357
|
+
return;
|
|
358
|
+
entry.closed = true;
|
|
359
|
+
if (!entry.close)
|
|
360
|
+
return;
|
|
361
|
+
entry.close();
|
|
362
|
+
ctx.emit('voice.level', { event: 'monitor.close' });
|
|
363
|
+
};
|
|
364
|
+
/**
|
|
365
|
+
* Hands-free text that came from a voice quieter than the user's: '' when
|
|
366
|
+
* the gate drops it. Push-to-talk is never gated: the user is pressing.
|
|
367
|
+
*/
|
|
368
|
+
const withoutBackground = (text, final) => {
|
|
369
|
+
if (!levelGate || !monitor || !text.trim() || state.mode !== 'hands-free')
|
|
370
|
+
return text;
|
|
371
|
+
if ((gateOptions?.scope ?? 'barge-in') === 'barge-in' && !botAnswering())
|
|
372
|
+
return text;
|
|
373
|
+
if (levelGate.judge() !== 'background')
|
|
374
|
+
return text;
|
|
375
|
+
if (final) {
|
|
376
|
+
decide({
|
|
377
|
+
kind: 'ignore',
|
|
378
|
+
text,
|
|
379
|
+
reason: 'background',
|
|
380
|
+
levelDb: levelGate.peak(),
|
|
381
|
+
userLevelDb: levelGate.userLevelDb,
|
|
382
|
+
});
|
|
383
|
+
}
|
|
384
|
+
return '';
|
|
385
|
+
};
|
|
386
|
+
/** Levels for a decision, when the gate is listening. */
|
|
387
|
+
const levelReading = () => levelGate && monitor ? { levelDb: levelGate.peak(), userLevelDb: levelGate.userLevelDb } : {};
|
|
388
|
+
/** An utterance from the user was accepted: learn how loud they speak. */
|
|
389
|
+
const learnLevel = () => {
|
|
390
|
+
if (!levelGate || !monitor)
|
|
391
|
+
return;
|
|
392
|
+
// The window must hold the user's voice, not the bot's.
|
|
393
|
+
if (state.speaking || Date.now() - spokeUntil < (gateOptions?.lookbackMs ?? 3000)) {
|
|
394
|
+
ctx.emit('voice.level', { event: 'learn.skipped', reason: 'bot-speech' });
|
|
395
|
+
return;
|
|
396
|
+
}
|
|
397
|
+
const levelDb = levelGate.peak();
|
|
398
|
+
if (levelDb === undefined) {
|
|
399
|
+
ctx.emit('voice.level', { event: 'learn.skipped', reason: 'no-audio' });
|
|
400
|
+
return;
|
|
401
|
+
}
|
|
402
|
+
levelGate.learn();
|
|
403
|
+
const userLevelDb = levelGate.userLevelDb;
|
|
404
|
+
ctx.emit('voice.level', { event: 'learned', levelDb, userLevelDb, gateDb: levelGate.gateDb });
|
|
405
|
+
if (userLevelDb !== state.userLevelDb)
|
|
406
|
+
set({ userLevelDb });
|
|
407
|
+
};
|
|
269
408
|
/* ---------------------------------------------------------------------- */
|
|
270
409
|
/* Barge-in */
|
|
271
410
|
/* ---------------------------------------------------------------------- */
|
|
@@ -319,7 +458,7 @@ export async function createVoice(ctx, options) {
|
|
|
319
458
|
if (!final && words.length < (bargeInOptions.minWords ?? 2))
|
|
320
459
|
return 'pending';
|
|
321
460
|
bargedIn = true;
|
|
322
|
-
decide({ kind: 'interrupt', text });
|
|
461
|
+
decide({ kind: 'interrupt', text, ...levelReading() });
|
|
323
462
|
// Interrupt first: the interceptor reads how much was spoken before speech stops.
|
|
324
463
|
void ctx.app.interrupt({ by: 'voice' });
|
|
325
464
|
service.cancelSpeech();
|
|
@@ -385,7 +524,7 @@ export async function createVoice(ctx, options) {
|
|
|
385
524
|
onPartial(heard) {
|
|
386
525
|
if (!isCurrent() || entry.stopping)
|
|
387
526
|
return;
|
|
388
|
-
const text = withoutEcho(heard, false);
|
|
527
|
+
const text = withoutBackground(withoutEcho(heard, false), false);
|
|
389
528
|
if (entry.held)
|
|
390
529
|
return set({ partial: heldText(entry.held, text) });
|
|
391
530
|
bargeInCheck(text, false);
|
|
@@ -408,9 +547,12 @@ export async function createVoice(ctx, options) {
|
|
|
408
547
|
if (entry.discarded)
|
|
409
548
|
return;
|
|
410
549
|
const heard = text.trim();
|
|
411
|
-
const
|
|
550
|
+
const afterEcho = withoutEcho(heard, true).trim();
|
|
551
|
+
const trimmed = entry.held ? afterEcho : withoutBackground(afterEcho, true).trim();
|
|
412
552
|
if (heard && !trimmed) {
|
|
413
|
-
|
|
553
|
+
// A background voice was already reported by the gate.
|
|
554
|
+
if (!afterEcho)
|
|
555
|
+
decide({ kind: 'ignore', text: heard, reason: 'echo' });
|
|
414
556
|
if (handsFree && !entry.held) {
|
|
415
557
|
turn.live = '';
|
|
416
558
|
turn.heardAt = 0;
|
|
@@ -465,6 +607,7 @@ export async function createVoice(ctx, options) {
|
|
|
465
607
|
return;
|
|
466
608
|
}
|
|
467
609
|
turn.texts.push(trimmed);
|
|
610
|
+
learnLevel();
|
|
468
611
|
turn.live = '';
|
|
469
612
|
turn.stt = provider.name;
|
|
470
613
|
// Recognizers deliver the final result some time after the user stops
|
|
@@ -535,6 +678,8 @@ export async function createVoice(ctx, options) {
|
|
|
535
678
|
else if (!wantListening && turn.texts.length > 0) {
|
|
536
679
|
sendTurn('manual');
|
|
537
680
|
}
|
|
681
|
+
if (!wantListening)
|
|
682
|
+
closeMonitor();
|
|
538
683
|
},
|
|
539
684
|
});
|
|
540
685
|
}
|
|
@@ -979,10 +1124,17 @@ export async function createVoice(ctx, options) {
|
|
|
979
1124
|
else if (state.speaking) {
|
|
980
1125
|
service.cancelSpeech();
|
|
981
1126
|
}
|
|
1127
|
+
if (!(await noiseAllowsListening()))
|
|
1128
|
+
return;
|
|
1129
|
+
if (!wantListening || active)
|
|
1130
|
+
return;
|
|
1131
|
+
openMonitor();
|
|
982
1132
|
await startSession();
|
|
983
1133
|
},
|
|
984
1134
|
stop() {
|
|
985
1135
|
wantListening = false;
|
|
1136
|
+
noiseRun?.controller.abort();
|
|
1137
|
+
closeMonitor();
|
|
986
1138
|
clearTimeout(restartTimer);
|
|
987
1139
|
const entry = active;
|
|
988
1140
|
const press = holding;
|
|
@@ -1012,6 +1164,8 @@ export async function createVoice(ctx, options) {
|
|
|
1012
1164
|
},
|
|
1013
1165
|
cancel() {
|
|
1014
1166
|
wantListening = false;
|
|
1167
|
+
noiseRun?.controller.abort();
|
|
1168
|
+
closeMonitor();
|
|
1015
1169
|
clearTimeout(restartTimer);
|
|
1016
1170
|
if (holding)
|
|
1017
1171
|
release(holding, false);
|
|
@@ -1050,6 +1204,12 @@ export async function createVoice(ctx, options) {
|
|
|
1050
1204
|
restartTimer = setTimeout(() => wantListening && !active && void startSession(), 250);
|
|
1051
1205
|
}
|
|
1052
1206
|
},
|
|
1207
|
+
async checkNoise() {
|
|
1208
|
+
const report = await measureNoise();
|
|
1209
|
+
if (report)
|
|
1210
|
+
ctx.emit('voice.noise', { report });
|
|
1211
|
+
return report;
|
|
1212
|
+
},
|
|
1053
1213
|
async toggle() {
|
|
1054
1214
|
if (state.listening || wantListening)
|
|
1055
1215
|
service.stop();
|
|
@@ -1057,6 +1217,8 @@ export async function createVoice(ctx, options) {
|
|
|
1057
1217
|
await service.listen();
|
|
1058
1218
|
},
|
|
1059
1219
|
setMode(mode) {
|
|
1220
|
+
if (state.noiseFallback)
|
|
1221
|
+
set({ noiseFallback: false });
|
|
1060
1222
|
if (mode === state.mode)
|
|
1061
1223
|
return;
|
|
1062
1224
|
const wasListening = wantListening;
|
|
@@ -1170,6 +1332,7 @@ export async function createVoice(ctx, options) {
|
|
|
1170
1332
|
});
|
|
1171
1333
|
ctx.onDispose(() => {
|
|
1172
1334
|
disposed = true;
|
|
1335
|
+
closeMonitor();
|
|
1173
1336
|
unfollowEffects?.();
|
|
1174
1337
|
followTraces([]);
|
|
1175
1338
|
for (const controller of loadControllers)
|