@burdenoff/website-sdk 2026.1105.2 → 2026.1106.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +133 -0
- package/dist/avatar-live-view/index.d.mts +1 -1
- package/dist/avatar-live-view/index.d.ts +1 -1
- package/dist/avatar-live-view/index.js +1071 -205
- package/dist/avatar-live-view/index.js.map +1 -1
- package/dist/avatar-live-view/index.mjs +1064 -198
- package/dist/avatar-live-view/index.mjs.map +1 -1
- package/dist/avatar-mic/index.d.mts +181 -62
- package/dist/avatar-mic/index.d.ts +181 -62
- package/dist/avatar-mic/index.js +717 -5
- package/dist/avatar-mic/index.js.map +1 -1
- package/dist/avatar-mic/index.mjs +717 -6
- package/dist/avatar-mic/index.mjs.map +1 -1
- package/dist/{chunk-MVNN6LLO.mjs → chunk-BHEHF3Y5.mjs} +2 -2
- package/dist/chunk-BHEHF3Y5.mjs.map +1 -0
- package/dist/{chunk-XIZE37IB.js → chunk-Z2UP22GV.js} +2 -2
- package/dist/chunk-Z2UP22GV.js.map +1 -0
- package/dist/{explore-avatar-live-config-auLKxSzx.d.ts → explore-avatar-live-config-CJstiSAN.d.ts} +10 -3
- package/dist/{explore-avatar-live-config-BL8FTm0I.d.mts → explore-avatar-live-config-D2PGF8fG.d.mts} +10 -3
- package/dist/index.d.mts +5 -4
- package/dist/index.d.ts +5 -4
- package/dist/index.js +5 -5
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1 -1
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
- package/dist/chunk-MVNN6LLO.mjs.map +0 -1
- package/dist/chunk-XIZE37IB.js.map +0 -1
|
@@ -1,62 +1,3 @@
|
|
|
1
|
-
/** What the worklet posts for each frame. */
|
|
2
|
-
interface CaptureFrameMessage {
|
|
3
|
-
/** s16le mono 16 kHz, `CAPTURE_FRAME_SAMPLES` samples. */
|
|
4
|
-
pcm: ArrayBuffer;
|
|
5
|
-
/** Microphone level over the frame, dBFS. */
|
|
6
|
-
rmsDb: number;
|
|
7
|
-
/** Reference-bus level over the same samples, dBFS (about -90 when silent, as it always is here). */
|
|
8
|
-
refDb: number;
|
|
9
|
-
/** Zero crossings per sample. */
|
|
10
|
-
zcr: number;
|
|
11
|
-
/** AudioContext time at the end of the frame, seconds. */
|
|
12
|
-
ctxTime: number;
|
|
13
|
-
seq: number;
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
/**
|
|
17
|
-
* BOFF-9797 (Stage C). The Avatar tab's own microphone capture: `getUserMedia`, through the capture worklet
|
|
18
|
-
* (`capture-worklet.ts`), into 16 kHz PCM frames.
|
|
19
|
-
*
|
|
20
|
-
* **Ported, not shared.** From `src/shared/assistant/voice/handsfree/micCapture.ts` in Algoshred/fe-libs at the
|
|
21
|
-
* commit `capture-worklet.ts` names (`CAPTURE_WORKLET_PORTED_FROM`; blob `8486c630`), with one difference: fe-libs
|
|
22
|
-
* routes everything its assistant plays into the worklet's second input (its playback reference, for barge-in), and the
|
|
23
|
-
* website plays nothing through this context (the avatar's voice is the room's own audio element, and the mic is
|
|
24
|
-
* offered only while no session plays), so no reference bus is made and the worklet has the microphone as its only
|
|
25
|
-
* input. `looksLikeHeadset`, which only fe-libs' barge-in reads, is not ported. `capture-worklet.test.ts` holds the
|
|
26
|
-
* parity tests.
|
|
27
|
-
*
|
|
28
|
-
* It opens `getUserMedia` with echo cancellation, noise suppression and auto gain on. Constraints are plain `true`,
|
|
29
|
-
* never `{ exact }`: `exact` throws OverconstrainedError on handsets that cannot honour one. The settings the browser
|
|
30
|
-
* actually applied are read back (`track.getSettings()`), because some Android handsets report `true` and apply nothing.
|
|
31
|
-
*
|
|
32
|
-
* `stop()` stops the track (the browser's recording indicator goes), disconnects the graph and closes the context. The
|
|
33
|
-
* caller does that the moment a press ends, whichever way it ends; `pagehide` does it too, whatever the caller does.
|
|
34
|
-
*/
|
|
35
|
-
|
|
36
|
-
interface MicCaptureHandle {
|
|
37
|
-
context: AudioContext;
|
|
38
|
-
micSource: MediaStreamAudioSourceNode;
|
|
39
|
-
track: MediaStreamTrack;
|
|
40
|
-
/** What the browser reports it applied. */
|
|
41
|
-
settings: MediaTrackSettings;
|
|
42
|
-
/** Stop everything and release the microphone. Safe to call more than once. */
|
|
43
|
-
stop(): void;
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
/** What the mic needs of a WebSocket; {@link openBrowserSocket} adapts the browser's. (fe-libs' seam.) */
|
|
47
|
-
interface SocketLike {
|
|
48
|
-
readonly readyState: number;
|
|
49
|
-
send(data: string | ArrayBufferLike | ArrayBufferView): void;
|
|
50
|
-
close(code?: number): void;
|
|
51
|
-
listen(handlers: SocketHandlers): void;
|
|
52
|
-
}
|
|
53
|
-
interface SocketHandlers {
|
|
54
|
-
open(): void;
|
|
55
|
-
message(data: unknown): void;
|
|
56
|
-
close(code: number): void;
|
|
57
|
-
error(): void;
|
|
58
|
-
}
|
|
59
|
-
|
|
60
1
|
/**
|
|
61
2
|
* BOFF-9797 (Stage C). The shapes the Avatar tab's mic shares with the live avatar's view.
|
|
62
3
|
*
|
|
@@ -79,7 +20,8 @@ interface AvatarMicOffer {
|
|
|
79
20
|
* Why a press did not give text:
|
|
80
21
|
*
|
|
81
22
|
* - `denied`: the microphone was refused (the visitor, the browser or the page's permissions);
|
|
82
|
-
* - `unsupported`: this browser cannot capture here, or has no microphone to use
|
|
23
|
+
* - `unsupported`: this browser cannot capture here, or has no microphone to use (with the browser's recognizer: its
|
|
24
|
+
* speech service is off or unreachable here, or it has no language to offer);
|
|
83
25
|
* - `unavailable`: the server will not take a stream from this page now: no offer, an offer it refuses twice, its mic
|
|
84
26
|
* notice is another version than this SDK's, or the website avatar is paused;
|
|
85
27
|
* - `limit`: a limit on voice questions is reached (the visitor's, the network's, or every visitor's together);
|
|
@@ -88,6 +30,19 @@ interface AvatarMicOffer {
|
|
|
88
30
|
* time).
|
|
89
31
|
*/
|
|
90
32
|
type AvatarMicFailure = "denied" | "unsupported" | "unavailable" | "limit" | "busy" | "failed";
|
|
33
|
+
/**
|
|
34
|
+
* Why a press of the voice conversation ended with text, or with none (Stage C2, a press with an
|
|
35
|
+
* {@link AvatarMicEndpoint} only):
|
|
36
|
+
*
|
|
37
|
+
* - `pause`: the visitor stopped speaking for the pause, after a question: the one way a question is sent by itself;
|
|
38
|
+
* - `tap`: the visitor stopped it;
|
|
39
|
+
* - `limit`: the question reached the longest one, counted from its first word;
|
|
40
|
+
* - `silence`: nothing that counts as speech was heard in the longest wait, or before the recognizer ended;
|
|
41
|
+
* - `failed`: the recognizer failed, or could no longer go on, after or before words.
|
|
42
|
+
*
|
|
43
|
+
* Only `pause` may be sent; every other one's words go into the box for the visitor, never sent.
|
|
44
|
+
*/
|
|
45
|
+
type AvatarMicDoneBy = "tap" | "pause" | "limit" | "silence" | "failed";
|
|
91
46
|
/**
|
|
92
47
|
* One press of the mic, as the view shows it. `epoch` names the press: a new press has a new one, and what an older one
|
|
93
48
|
* reports is never shown for a newer one.
|
|
@@ -98,6 +53,11 @@ type AvatarMicFailure = "denied" | "unsupported" | "unavailable" | "limit" | "bu
|
|
|
98
53
|
* - `finishing`: the recording has stopped (a tap, or the limit: `limit`), and the last of the text is on its way;
|
|
99
54
|
* - `done`: the text, final (empty when no words were heard);
|
|
100
55
|
* - `failed`: no text, and why (with any final text heard before it failed, which the view does not use).
|
|
56
|
+
*
|
|
57
|
+
* A press with an {@link AvatarMicEndpoint} (the voice conversation, Stage C2) says more, and only it: `listening`
|
|
58
|
+
* says whether what is heard counts as speech (`genuine`) and whether the visitor has paused (`pausing`); `done` says
|
|
59
|
+
* why it ended (`by`) and whether its words are the avatar's own voice (`echo`). A press without one reports exactly
|
|
60
|
+
* the shapes it did before.
|
|
101
61
|
*/
|
|
102
62
|
type AvatarMicState = {
|
|
103
63
|
name: "idle";
|
|
@@ -109,6 +69,10 @@ type AvatarMicState = {
|
|
|
109
69
|
epoch: number;
|
|
110
70
|
startedAt: number;
|
|
111
71
|
text: string;
|
|
72
|
+
/** With an endpoint: the words heard are speech, not silence, sounds or a stall (`isGenuineSpeech`). */
|
|
73
|
+
genuine?: boolean;
|
|
74
|
+
/** With an endpoint: no new word for a moment, after speech ("Still listening — take your time"). */
|
|
75
|
+
pausing?: boolean;
|
|
112
76
|
} | {
|
|
113
77
|
name: "finishing";
|
|
114
78
|
epoch: number;
|
|
@@ -119,6 +83,10 @@ type AvatarMicState = {
|
|
|
119
83
|
epoch: number;
|
|
120
84
|
text: string;
|
|
121
85
|
limit: boolean;
|
|
86
|
+
/** With an endpoint: why the press ended. */
|
|
87
|
+
by?: AvatarMicDoneBy;
|
|
88
|
+
/** With an endpoint: the words are the avatar's own answer, heard back through the microphone. */
|
|
89
|
+
echo?: boolean;
|
|
122
90
|
} | {
|
|
123
91
|
name: "failed";
|
|
124
92
|
epoch: number;
|
|
@@ -151,9 +119,159 @@ interface AvatarMic {
|
|
|
151
119
|
/** The press as it stands. */
|
|
152
120
|
state(): AvatarMicState;
|
|
153
121
|
}
|
|
122
|
+
/**
|
|
123
|
+
* The mic through the browser's own speech recognition (Stage C, `mic.mode: "browser"` in availability): the browser
|
|
124
|
+
* turns the voice into text, and nothing of the press reaches the server. Same press, same states, same failures as
|
|
125
|
+
* the stream's ({@link AvatarMic}, {@link AvatarMicState}); it never reports `unavailable`, `limit` or `busy`.
|
|
126
|
+
*/
|
|
127
|
+
interface BrowserAvatarMicOptions {
|
|
128
|
+
/** The recognizer's language, a BCP-47 tag (`en-IN`). */
|
|
129
|
+
lang: string;
|
|
130
|
+
/** Tried once, in the same press, when the browser does not recognise `lang` (`en-US`). */
|
|
131
|
+
fallbackLang: string;
|
|
132
|
+
/**
|
|
133
|
+
* The longest press, from the moment the browser starts listening: thirty seconds. With an `endpoint`, the longest
|
|
134
|
+
* wait for a question to start, and the longest question, from its first word.
|
|
135
|
+
*/
|
|
136
|
+
maxPressMs: number;
|
|
137
|
+
/** Every change of the press's state. */
|
|
138
|
+
onState: (state: AvatarMicState) => void;
|
|
139
|
+
/**
|
|
140
|
+
* The voice conversation (Stage C2): every press ends itself once the visitor has paused after a question (see
|
|
141
|
+
* {@link AvatarMicEndpoint}). Without it, a press is exactly the editable one: tap to start, tap to stop.
|
|
142
|
+
*/
|
|
143
|
+
endpoint?: AvatarMicEndpoint;
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* How a press of the voice conversation ends itself (Stage C2): `done` with `by: "pause"` once what is heard is speech
|
|
147
|
+
* and no word of it has changed for `pauseMs` (twice that for a sentence left open), never sooner than
|
|
148
|
+
* `BROWSER_PAUSE_AFTER_START_MS` after its recognizer (re)started, and never between two recognizers.
|
|
149
|
+
*/
|
|
150
|
+
interface AvatarMicEndpoint {
|
|
151
|
+
/**
|
|
152
|
+
* The pause that ends a question, in milliseconds: the server's `mic.pauseMs` (2000 to 2500, read strictly by the
|
|
153
|
+
* view's API). Read within those bounds whatever is passed.
|
|
154
|
+
*/
|
|
155
|
+
pauseMs: number;
|
|
156
|
+
/**
|
|
157
|
+
* The words of the answer the avatar has just read (its captions), for the echo check, read as the press ends; null
|
|
158
|
+
* when there is none, or when the check is not for this press (the view's hook checks only the presses the
|
|
159
|
+
* conversation starts by itself).
|
|
160
|
+
*/
|
|
161
|
+
echoOf: () => string | null;
|
|
162
|
+
}
|
|
154
163
|
/** What the mic's chunk (`dist/avatar-mic/`) exports. */
|
|
155
164
|
interface AvatarMicModule {
|
|
156
165
|
createAvatarMic: (options: AvatarMicOptions) => AvatarMic;
|
|
166
|
+
createBrowserAvatarMic: (options: BrowserAvatarMicOptions) => AvatarMic;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/** One alternative of a recognition result: what the browser heard. */
|
|
170
|
+
interface RecognitionAlternativeLike {
|
|
171
|
+
readonly transcript: string;
|
|
172
|
+
}
|
|
173
|
+
/** One recognition result: final, or still being recognised. */
|
|
174
|
+
interface RecognitionResultLike {
|
|
175
|
+
readonly isFinal: boolean;
|
|
176
|
+
readonly length: number;
|
|
177
|
+
readonly [index: number]: RecognitionAlternativeLike | undefined;
|
|
178
|
+
}
|
|
179
|
+
/** The whole result list of a recognizer, as each `result` event carries it. */
|
|
180
|
+
interface RecognitionResultListLike {
|
|
181
|
+
readonly length: number;
|
|
182
|
+
readonly [index: number]: RecognitionResultLike | undefined;
|
|
183
|
+
}
|
|
184
|
+
/** What this module uses of a browser's `SpeechRecognition` (declared here: TypeScript's DOM library has none). */
|
|
185
|
+
interface RecognitionLike {
|
|
186
|
+
lang: string;
|
|
187
|
+
continuous: boolean;
|
|
188
|
+
interimResults: boolean;
|
|
189
|
+
maxAlternatives: number;
|
|
190
|
+
onstart: (() => void) | null;
|
|
191
|
+
onresult: ((event: {
|
|
192
|
+
readonly results: RecognitionResultListLike;
|
|
193
|
+
}) => void) | null;
|
|
194
|
+
onerror: ((event: {
|
|
195
|
+
readonly error: string;
|
|
196
|
+
}) => void) | null;
|
|
197
|
+
onend: (() => void) | null;
|
|
198
|
+
start(): void;
|
|
199
|
+
stop(): void;
|
|
200
|
+
abort(): void;
|
|
201
|
+
}
|
|
202
|
+
/** What a press needs from the browser. The browser's own by default; a test hands in fakes. */
|
|
203
|
+
interface BrowserAvatarMicSeams {
|
|
204
|
+
/** A new recognizer, or null where the browser has none. */
|
|
205
|
+
createRecognition: () => RecognitionLike | null;
|
|
206
|
+
/** A monotonic clock, in milliseconds, for the cooldowns. */
|
|
207
|
+
now: () => number;
|
|
208
|
+
/** {@link primeBrowserMicrophone}: called inside the tap, where the microphone is not known to be granted. */
|
|
209
|
+
primeMicrophone: () => Promise<void> | null;
|
|
210
|
+
/** {@link watchBrowserMicPermission}: called once, as the mic is made. */
|
|
211
|
+
watchMicPermission: (onState: (state: PermissionState) => void) => void;
|
|
212
|
+
/** {@link hasCoarsePointer}: read at each tap. */
|
|
213
|
+
touch: () => boolean;
|
|
214
|
+
}
|
|
215
|
+
/** The Avatar tab's mic through the browser's own recognizer. See the file's comment for what a press does. */
|
|
216
|
+
declare function createBrowserAvatarMic(options: BrowserAvatarMicOptions, seams?: Partial<BrowserAvatarMicSeams>): AvatarMic;
|
|
217
|
+
|
|
218
|
+
/** What the worklet posts for each frame. */
|
|
219
|
+
interface CaptureFrameMessage {
|
|
220
|
+
/** s16le mono 16 kHz, `CAPTURE_FRAME_SAMPLES` samples. */
|
|
221
|
+
pcm: ArrayBuffer;
|
|
222
|
+
/** Microphone level over the frame, dBFS. */
|
|
223
|
+
rmsDb: number;
|
|
224
|
+
/** Reference-bus level over the same samples, dBFS (about -90 when silent, as it always is here). */
|
|
225
|
+
refDb: number;
|
|
226
|
+
/** Zero crossings per sample. */
|
|
227
|
+
zcr: number;
|
|
228
|
+
/** AudioContext time at the end of the frame, seconds. */
|
|
229
|
+
ctxTime: number;
|
|
230
|
+
seq: number;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* BOFF-9797 (Stage C). The Avatar tab's own microphone capture: `getUserMedia`, through the capture worklet
|
|
235
|
+
* (`capture-worklet.ts`), into 16 kHz PCM frames.
|
|
236
|
+
*
|
|
237
|
+
* **Ported, not shared.** From `src/shared/assistant/voice/handsfree/micCapture.ts` in Algoshred/fe-libs at the
|
|
238
|
+
* commit `capture-worklet.ts` names (`CAPTURE_WORKLET_PORTED_FROM`; blob `8486c630`), with one difference: fe-libs
|
|
239
|
+
* routes everything its assistant plays into the worklet's second input (its playback reference, for barge-in), and the
|
|
240
|
+
* website plays nothing through this context (the avatar's voice is the room's own audio element, and the mic is
|
|
241
|
+
* offered only while no session plays), so no reference bus is made and the worklet has the microphone as its only
|
|
242
|
+
* input. `looksLikeHeadset`, which only fe-libs' barge-in reads, is not ported. `capture-worklet.test.ts` holds the
|
|
243
|
+
* parity tests.
|
|
244
|
+
*
|
|
245
|
+
* It opens `getUserMedia` with echo cancellation, noise suppression and auto gain on. Constraints are plain `true`,
|
|
246
|
+
* never `{ exact }`: `exact` throws OverconstrainedError on handsets that cannot honour one. The settings the browser
|
|
247
|
+
* actually applied are read back (`track.getSettings()`), because some Android handsets report `true` and apply nothing.
|
|
248
|
+
*
|
|
249
|
+
* `stop()` stops the track (the browser's recording indicator goes), disconnects the graph and closes the context. The
|
|
250
|
+
* caller does that the moment a press ends, whichever way it ends; `pagehide` does it too, whatever the caller does.
|
|
251
|
+
*/
|
|
252
|
+
|
|
253
|
+
interface MicCaptureHandle {
|
|
254
|
+
context: AudioContext;
|
|
255
|
+
micSource: MediaStreamAudioSourceNode;
|
|
256
|
+
track: MediaStreamTrack;
|
|
257
|
+
/** What the browser reports it applied. */
|
|
258
|
+
settings: MediaTrackSettings;
|
|
259
|
+
/** Stop everything and release the microphone. Safe to call more than once. */
|
|
260
|
+
stop(): void;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/** What the mic needs of a WebSocket; {@link openBrowserSocket} adapts the browser's. (fe-libs' seam.) */
|
|
264
|
+
interface SocketLike {
|
|
265
|
+
readonly readyState: number;
|
|
266
|
+
send(data: string | ArrayBufferLike | ArrayBufferView): void;
|
|
267
|
+
close(code?: number): void;
|
|
268
|
+
listen(handlers: SocketHandlers): void;
|
|
269
|
+
}
|
|
270
|
+
interface SocketHandlers {
|
|
271
|
+
open(): void;
|
|
272
|
+
message(data: unknown): void;
|
|
273
|
+
close(code: number): void;
|
|
274
|
+
error(): void;
|
|
157
275
|
}
|
|
158
276
|
|
|
159
277
|
/**
|
|
@@ -176,7 +294,8 @@ interface AvatarMicModule {
|
|
|
176
294
|
*
|
|
177
295
|
* **Released on every way out.** The microphone's track is stopped the moment the recording stops, and also when the
|
|
178
296
|
* press fails, is cancelled (`cancel()`: the view going, the page hidden, the Chat tab) or superseded (a new press while
|
|
179
|
-
* one is still finishing: `cancel` for the old one, so it stops
|
|
297
|
+
* one is still finishing, a tap the view lets through for this stream mic only: `cancel` for the old one, so it stops
|
|
298
|
+
* being paid for, and its text is never reported). A
|
|
180
299
|
* microphone that opens after its press has gone is closed as it opens. The stream is closed whenever no press needs
|
|
181
300
|
* it: one connection serves a press and the press that supersedes it, and no other.
|
|
182
301
|
*/
|
|
@@ -190,4 +309,4 @@ interface AvatarMicSeams {
|
|
|
190
309
|
/** The mic of one Avatar tab. See the file's comment for what a press does. */
|
|
191
310
|
declare function createAvatarMic(options: AvatarMicOptions, seams?: Partial<AvatarMicSeams>): AvatarMic;
|
|
192
311
|
|
|
193
|
-
export { type AvatarMic, type AvatarMicFailure, type AvatarMicModule, type AvatarMicOffer, type AvatarMicOptions, type AvatarMicState, createAvatarMic };
|
|
312
|
+
export { type AvatarMic, type AvatarMicDoneBy, type AvatarMicEndpoint, type AvatarMicFailure, type AvatarMicModule, type AvatarMicOffer, type AvatarMicOptions, type AvatarMicState, type BrowserAvatarMicOptions, createAvatarMic, createBrowserAvatarMic };
|
|
@@ -1,62 +1,3 @@
|
|
|
1
|
-
/** What the worklet posts for each frame. */
|
|
2
|
-
interface CaptureFrameMessage {
|
|
3
|
-
/** s16le mono 16 kHz, `CAPTURE_FRAME_SAMPLES` samples. */
|
|
4
|
-
pcm: ArrayBuffer;
|
|
5
|
-
/** Microphone level over the frame, dBFS. */
|
|
6
|
-
rmsDb: number;
|
|
7
|
-
/** Reference-bus level over the same samples, dBFS (about -90 when silent, as it always is here). */
|
|
8
|
-
refDb: number;
|
|
9
|
-
/** Zero crossings per sample. */
|
|
10
|
-
zcr: number;
|
|
11
|
-
/** AudioContext time at the end of the frame, seconds. */
|
|
12
|
-
ctxTime: number;
|
|
13
|
-
seq: number;
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
/**
|
|
17
|
-
* BOFF-9797 (Stage C). The Avatar tab's own microphone capture: `getUserMedia`, through the capture worklet
|
|
18
|
-
* (`capture-worklet.ts`), into 16 kHz PCM frames.
|
|
19
|
-
*
|
|
20
|
-
* **Ported, not shared.** From `src/shared/assistant/voice/handsfree/micCapture.ts` in Algoshred/fe-libs at the
|
|
21
|
-
* commit `capture-worklet.ts` names (`CAPTURE_WORKLET_PORTED_FROM`; blob `8486c630`), with one difference: fe-libs
|
|
22
|
-
* routes everything its assistant plays into the worklet's second input (its playback reference, for barge-in), and the
|
|
23
|
-
* website plays nothing through this context (the avatar's voice is the room's own audio element, and the mic is
|
|
24
|
-
* offered only while no session plays), so no reference bus is made and the worklet has the microphone as its only
|
|
25
|
-
* input. `looksLikeHeadset`, which only fe-libs' barge-in reads, is not ported. `capture-worklet.test.ts` holds the
|
|
26
|
-
* parity tests.
|
|
27
|
-
*
|
|
28
|
-
* It opens `getUserMedia` with echo cancellation, noise suppression and auto gain on. Constraints are plain `true`,
|
|
29
|
-
* never `{ exact }`: `exact` throws OverconstrainedError on handsets that cannot honour one. The settings the browser
|
|
30
|
-
* actually applied are read back (`track.getSettings()`), because some Android handsets report `true` and apply nothing.
|
|
31
|
-
*
|
|
32
|
-
* `stop()` stops the track (the browser's recording indicator goes), disconnects the graph and closes the context. The
|
|
33
|
-
* caller does that the moment a press ends, whichever way it ends; `pagehide` does it too, whatever the caller does.
|
|
34
|
-
*/
|
|
35
|
-
|
|
36
|
-
interface MicCaptureHandle {
|
|
37
|
-
context: AudioContext;
|
|
38
|
-
micSource: MediaStreamAudioSourceNode;
|
|
39
|
-
track: MediaStreamTrack;
|
|
40
|
-
/** What the browser reports it applied. */
|
|
41
|
-
settings: MediaTrackSettings;
|
|
42
|
-
/** Stop everything and release the microphone. Safe to call more than once. */
|
|
43
|
-
stop(): void;
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
/** What the mic needs of a WebSocket; {@link openBrowserSocket} adapts the browser's. (fe-libs' seam.) */
|
|
47
|
-
interface SocketLike {
|
|
48
|
-
readonly readyState: number;
|
|
49
|
-
send(data: string | ArrayBufferLike | ArrayBufferView): void;
|
|
50
|
-
close(code?: number): void;
|
|
51
|
-
listen(handlers: SocketHandlers): void;
|
|
52
|
-
}
|
|
53
|
-
interface SocketHandlers {
|
|
54
|
-
open(): void;
|
|
55
|
-
message(data: unknown): void;
|
|
56
|
-
close(code: number): void;
|
|
57
|
-
error(): void;
|
|
58
|
-
}
|
|
59
|
-
|
|
60
1
|
/**
|
|
61
2
|
* BOFF-9797 (Stage C). The shapes the Avatar tab's mic shares with the live avatar's view.
|
|
62
3
|
*
|
|
@@ -79,7 +20,8 @@ interface AvatarMicOffer {
|
|
|
79
20
|
* Why a press did not give text:
|
|
80
21
|
*
|
|
81
22
|
* - `denied`: the microphone was refused (the visitor, the browser or the page's permissions);
|
|
82
|
-
* - `unsupported`: this browser cannot capture here, or has no microphone to use
|
|
23
|
+
* - `unsupported`: this browser cannot capture here, or has no microphone to use (with the browser's recognizer: its
|
|
24
|
+
* speech service is off or unreachable here, or it has no language to offer);
|
|
83
25
|
* - `unavailable`: the server will not take a stream from this page now: no offer, an offer it refuses twice, its mic
|
|
84
26
|
* notice is another version than this SDK's, or the website avatar is paused;
|
|
85
27
|
* - `limit`: a limit on voice questions is reached (the visitor's, the network's, or every visitor's together);
|
|
@@ -88,6 +30,19 @@ interface AvatarMicOffer {
|
|
|
88
30
|
* time).
|
|
89
31
|
*/
|
|
90
32
|
type AvatarMicFailure = "denied" | "unsupported" | "unavailable" | "limit" | "busy" | "failed";
|
|
33
|
+
/**
|
|
34
|
+
* Why a press of the voice conversation ended with text, or with none (Stage C2, a press with an
|
|
35
|
+
* {@link AvatarMicEndpoint} only):
|
|
36
|
+
*
|
|
37
|
+
* - `pause`: the visitor stopped speaking for the pause, after a question: the one way a question is sent by itself;
|
|
38
|
+
* - `tap`: the visitor stopped it;
|
|
39
|
+
* - `limit`: the question reached the longest one, counted from its first word;
|
|
40
|
+
* - `silence`: nothing that counts as speech was heard in the longest wait, or before the recognizer ended;
|
|
41
|
+
* - `failed`: the recognizer failed, or could no longer go on, after or before words.
|
|
42
|
+
*
|
|
43
|
+
* Only `pause` may be sent; every other one's words go into the box for the visitor, never sent.
|
|
44
|
+
*/
|
|
45
|
+
type AvatarMicDoneBy = "tap" | "pause" | "limit" | "silence" | "failed";
|
|
91
46
|
/**
|
|
92
47
|
* One press of the mic, as the view shows it. `epoch` names the press: a new press has a new one, and what an older one
|
|
93
48
|
* reports is never shown for a newer one.
|
|
@@ -98,6 +53,11 @@ type AvatarMicFailure = "denied" | "unsupported" | "unavailable" | "limit" | "bu
|
|
|
98
53
|
* - `finishing`: the recording has stopped (a tap, or the limit: `limit`), and the last of the text is on its way;
|
|
99
54
|
* - `done`: the text, final (empty when no words were heard);
|
|
100
55
|
* - `failed`: no text, and why (with any final text heard before it failed, which the view does not use).
|
|
56
|
+
*
|
|
57
|
+
* A press with an {@link AvatarMicEndpoint} (the voice conversation, Stage C2) says more, and only it: `listening`
|
|
58
|
+
* says whether what is heard counts as speech (`genuine`) and whether the visitor has paused (`pausing`); `done` says
|
|
59
|
+
* why it ended (`by`) and whether its words are the avatar's own voice (`echo`). A press without one reports exactly
|
|
60
|
+
* the shapes it did before.
|
|
101
61
|
*/
|
|
102
62
|
type AvatarMicState = {
|
|
103
63
|
name: "idle";
|
|
@@ -109,6 +69,10 @@ type AvatarMicState = {
|
|
|
109
69
|
epoch: number;
|
|
110
70
|
startedAt: number;
|
|
111
71
|
text: string;
|
|
72
|
+
/** With an endpoint: the words heard are speech, not silence, sounds or a stall (`isGenuineSpeech`). */
|
|
73
|
+
genuine?: boolean;
|
|
74
|
+
/** With an endpoint: no new word for a moment, after speech ("Still listening — take your time"). */
|
|
75
|
+
pausing?: boolean;
|
|
112
76
|
} | {
|
|
113
77
|
name: "finishing";
|
|
114
78
|
epoch: number;
|
|
@@ -119,6 +83,10 @@ type AvatarMicState = {
|
|
|
119
83
|
epoch: number;
|
|
120
84
|
text: string;
|
|
121
85
|
limit: boolean;
|
|
86
|
+
/** With an endpoint: why the press ended. */
|
|
87
|
+
by?: AvatarMicDoneBy;
|
|
88
|
+
/** With an endpoint: the words are the avatar's own answer, heard back through the microphone. */
|
|
89
|
+
echo?: boolean;
|
|
122
90
|
} | {
|
|
123
91
|
name: "failed";
|
|
124
92
|
epoch: number;
|
|
@@ -151,9 +119,159 @@ interface AvatarMic {
|
|
|
151
119
|
/** The press as it stands. */
|
|
152
120
|
state(): AvatarMicState;
|
|
153
121
|
}
|
|
122
|
+
/**
|
|
123
|
+
* The mic through the browser's own speech recognition (Stage C, `mic.mode: "browser"` in availability): the browser
|
|
124
|
+
* turns the voice into text, and nothing of the press reaches the server. Same press, same states, same failures as
|
|
125
|
+
* the stream's ({@link AvatarMic}, {@link AvatarMicState}); it never reports `unavailable`, `limit` or `busy`.
|
|
126
|
+
*/
|
|
127
|
+
interface BrowserAvatarMicOptions {
|
|
128
|
+
/** The recognizer's language, a BCP-47 tag (`en-IN`). */
|
|
129
|
+
lang: string;
|
|
130
|
+
/** Tried once, in the same press, when the browser does not recognise `lang` (`en-US`). */
|
|
131
|
+
fallbackLang: string;
|
|
132
|
+
/**
|
|
133
|
+
* The longest press, from the moment the browser starts listening: thirty seconds. With an `endpoint`, the longest
|
|
134
|
+
* wait for a question to start, and the longest question, from its first word.
|
|
135
|
+
*/
|
|
136
|
+
maxPressMs: number;
|
|
137
|
+
/** Every change of the press's state. */
|
|
138
|
+
onState: (state: AvatarMicState) => void;
|
|
139
|
+
/**
|
|
140
|
+
* The voice conversation (Stage C2): every press ends itself once the visitor has paused after a question (see
|
|
141
|
+
* {@link AvatarMicEndpoint}). Without it, a press is exactly the editable one: tap to start, tap to stop.
|
|
142
|
+
*/
|
|
143
|
+
endpoint?: AvatarMicEndpoint;
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* How a press of the voice conversation ends itself (Stage C2): `done` with `by: "pause"` once what is heard is speech
|
|
147
|
+
* and no word of it has changed for `pauseMs` (twice that for a sentence left open), never sooner than
|
|
148
|
+
* `BROWSER_PAUSE_AFTER_START_MS` after its recognizer (re)started, and never between two recognizers.
|
|
149
|
+
*/
|
|
150
|
+
interface AvatarMicEndpoint {
|
|
151
|
+
/**
|
|
152
|
+
* The pause that ends a question, in milliseconds: the server's `mic.pauseMs` (2000 to 2500, read strictly by the
|
|
153
|
+
* view's API). Read within those bounds whatever is passed.
|
|
154
|
+
*/
|
|
155
|
+
pauseMs: number;
|
|
156
|
+
/**
|
|
157
|
+
* The words of the answer the avatar has just read (its captions), for the echo check, read as the press ends; null
|
|
158
|
+
* when there is none, or when the check is not for this press (the view's hook checks only the presses the
|
|
159
|
+
* conversation starts by itself).
|
|
160
|
+
*/
|
|
161
|
+
echoOf: () => string | null;
|
|
162
|
+
}
|
|
154
163
|
/** What the mic's chunk (`dist/avatar-mic/`) exports. */
|
|
155
164
|
interface AvatarMicModule {
|
|
156
165
|
createAvatarMic: (options: AvatarMicOptions) => AvatarMic;
|
|
166
|
+
createBrowserAvatarMic: (options: BrowserAvatarMicOptions) => AvatarMic;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/** One alternative of a recognition result: what the browser heard. */
|
|
170
|
+
interface RecognitionAlternativeLike {
|
|
171
|
+
readonly transcript: string;
|
|
172
|
+
}
|
|
173
|
+
/** One recognition result: final, or still being recognised. */
|
|
174
|
+
interface RecognitionResultLike {
|
|
175
|
+
readonly isFinal: boolean;
|
|
176
|
+
readonly length: number;
|
|
177
|
+
readonly [index: number]: RecognitionAlternativeLike | undefined;
|
|
178
|
+
}
|
|
179
|
+
/** The whole result list of a recognizer, as each `result` event carries it. */
|
|
180
|
+
interface RecognitionResultListLike {
|
|
181
|
+
readonly length: number;
|
|
182
|
+
readonly [index: number]: RecognitionResultLike | undefined;
|
|
183
|
+
}
|
|
184
|
+
/** What this module uses of a browser's `SpeechRecognition` (declared here: TypeScript's DOM library has none). */
|
|
185
|
+
interface RecognitionLike {
|
|
186
|
+
lang: string;
|
|
187
|
+
continuous: boolean;
|
|
188
|
+
interimResults: boolean;
|
|
189
|
+
maxAlternatives: number;
|
|
190
|
+
onstart: (() => void) | null;
|
|
191
|
+
onresult: ((event: {
|
|
192
|
+
readonly results: RecognitionResultListLike;
|
|
193
|
+
}) => void) | null;
|
|
194
|
+
onerror: ((event: {
|
|
195
|
+
readonly error: string;
|
|
196
|
+
}) => void) | null;
|
|
197
|
+
onend: (() => void) | null;
|
|
198
|
+
start(): void;
|
|
199
|
+
stop(): void;
|
|
200
|
+
abort(): void;
|
|
201
|
+
}
|
|
202
|
+
/** What a press needs from the browser. The browser's own by default; a test hands in fakes. */
|
|
203
|
+
interface BrowserAvatarMicSeams {
|
|
204
|
+
/** A new recognizer, or null where the browser has none. */
|
|
205
|
+
createRecognition: () => RecognitionLike | null;
|
|
206
|
+
/** A monotonic clock, in milliseconds, for the cooldowns. */
|
|
207
|
+
now: () => number;
|
|
208
|
+
/** {@link primeBrowserMicrophone}: called inside the tap, where the microphone is not known to be granted. */
|
|
209
|
+
primeMicrophone: () => Promise<void> | null;
|
|
210
|
+
/** {@link watchBrowserMicPermission}: called once, as the mic is made. */
|
|
211
|
+
watchMicPermission: (onState: (state: PermissionState) => void) => void;
|
|
212
|
+
/** {@link hasCoarsePointer}: read at each tap. */
|
|
213
|
+
touch: () => boolean;
|
|
214
|
+
}
|
|
215
|
+
/** The Avatar tab's mic through the browser's own recognizer. See the file's comment for what a press does. */
|
|
216
|
+
declare function createBrowserAvatarMic(options: BrowserAvatarMicOptions, seams?: Partial<BrowserAvatarMicSeams>): AvatarMic;
|
|
217
|
+
|
|
218
|
+
/** What the worklet posts for each frame. */
|
|
219
|
+
interface CaptureFrameMessage {
|
|
220
|
+
/** s16le mono 16 kHz, `CAPTURE_FRAME_SAMPLES` samples. */
|
|
221
|
+
pcm: ArrayBuffer;
|
|
222
|
+
/** Microphone level over the frame, dBFS. */
|
|
223
|
+
rmsDb: number;
|
|
224
|
+
/** Reference-bus level over the same samples, dBFS (about -90 when silent, as it always is here). */
|
|
225
|
+
refDb: number;
|
|
226
|
+
/** Zero crossings per sample. */
|
|
227
|
+
zcr: number;
|
|
228
|
+
/** AudioContext time at the end of the frame, seconds. */
|
|
229
|
+
ctxTime: number;
|
|
230
|
+
seq: number;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* BOFF-9797 (Stage C). The Avatar tab's own microphone capture: `getUserMedia`, through the capture worklet
|
|
235
|
+
* (`capture-worklet.ts`), into 16 kHz PCM frames.
|
|
236
|
+
*
|
|
237
|
+
* **Ported, not shared.** From `src/shared/assistant/voice/handsfree/micCapture.ts` in Algoshred/fe-libs at the
|
|
238
|
+
* commit `capture-worklet.ts` names (`CAPTURE_WORKLET_PORTED_FROM`; blob `8486c630`), with one difference: fe-libs
|
|
239
|
+
* routes everything its assistant plays into the worklet's second input (its playback reference, for barge-in), and the
|
|
240
|
+
* website plays nothing through this context (the avatar's voice is the room's own audio element, and the mic is
|
|
241
|
+
* offered only while no session plays), so no reference bus is made and the worklet has the microphone as its only
|
|
242
|
+
* input. `looksLikeHeadset`, which only fe-libs' barge-in reads, is not ported. `capture-worklet.test.ts` holds the
|
|
243
|
+
* parity tests.
|
|
244
|
+
*
|
|
245
|
+
* It opens `getUserMedia` with echo cancellation, noise suppression and auto gain on. Constraints are plain `true`,
|
|
246
|
+
* never `{ exact }`: `exact` throws OverconstrainedError on handsets that cannot honour one. The settings the browser
|
|
247
|
+
* actually applied are read back (`track.getSettings()`), because some Android handsets report `true` and apply nothing.
|
|
248
|
+
*
|
|
249
|
+
* `stop()` stops the track (the browser's recording indicator goes), disconnects the graph and closes the context. The
|
|
250
|
+
* caller does that the moment a press ends, whichever way it ends; `pagehide` does it too, whatever the caller does.
|
|
251
|
+
*/
|
|
252
|
+
|
|
253
|
+
interface MicCaptureHandle {
|
|
254
|
+
context: AudioContext;
|
|
255
|
+
micSource: MediaStreamAudioSourceNode;
|
|
256
|
+
track: MediaStreamTrack;
|
|
257
|
+
/** What the browser reports it applied. */
|
|
258
|
+
settings: MediaTrackSettings;
|
|
259
|
+
/** Stop everything and release the microphone. Safe to call more than once. */
|
|
260
|
+
stop(): void;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/** What the mic needs of a WebSocket; {@link openBrowserSocket} adapts the browser's. (fe-libs' seam.) */
|
|
264
|
+
interface SocketLike {
|
|
265
|
+
readonly readyState: number;
|
|
266
|
+
send(data: string | ArrayBufferLike | ArrayBufferView): void;
|
|
267
|
+
close(code?: number): void;
|
|
268
|
+
listen(handlers: SocketHandlers): void;
|
|
269
|
+
}
|
|
270
|
+
interface SocketHandlers {
|
|
271
|
+
open(): void;
|
|
272
|
+
message(data: unknown): void;
|
|
273
|
+
close(code: number): void;
|
|
274
|
+
error(): void;
|
|
157
275
|
}
|
|
158
276
|
|
|
159
277
|
/**
|
|
@@ -176,7 +294,8 @@ interface AvatarMicModule {
|
|
|
176
294
|
*
|
|
177
295
|
* **Released on every way out.** The microphone's track is stopped the moment the recording stops, and also when the
|
|
178
296
|
* press fails, is cancelled (`cancel()`: the view going, the page hidden, the Chat tab) or superseded (a new press while
|
|
179
|
-
* one is still finishing: `cancel` for the old one, so it stops
|
|
297
|
+
* one is still finishing, a tap the view lets through for this stream mic only: `cancel` for the old one, so it stops
|
|
298
|
+
* being paid for, and its text is never reported). A
|
|
180
299
|
* microphone that opens after its press has gone is closed as it opens. The stream is closed whenever no press needs
|
|
181
300
|
* it: one connection serves a press and the press that supersedes it, and no other.
|
|
182
301
|
*/
|
|
@@ -190,4 +309,4 @@ interface AvatarMicSeams {
|
|
|
190
309
|
/** The mic of one Avatar tab. See the file's comment for what a press does. */
|
|
191
310
|
declare function createAvatarMic(options: AvatarMicOptions, seams?: Partial<AvatarMicSeams>): AvatarMic;
|
|
192
311
|
|
|
193
|
-
export { type AvatarMic, type AvatarMicFailure, type AvatarMicModule, type AvatarMicOffer, type AvatarMicOptions, type AvatarMicState, createAvatarMic };
|
|
312
|
+
export { type AvatarMic, type AvatarMicDoneBy, type AvatarMicEndpoint, type AvatarMicFailure, type AvatarMicModule, type AvatarMicOffer, type AvatarMicOptions, type AvatarMicState, type BrowserAvatarMicOptions, createAvatarMic, createBrowserAvatarMic };
|