@burdenoff/fe-libs 2026.1117.4 → 2026.1117.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/shared/assistant/ui/useVoiceSession.js +312 -301
- package/dist/shared/assistant/voice/handsFree.d.ts +16 -0
- package/dist/shared/assistant/voice/handsFree.js +2 -2
- package/dist/shared/assistant/voice/handsfree/audioTrackTransport.js +1 -0
- package/dist/shared/assistant/voice/handsfree/backendStreamTransport.d.ts +11 -0
- package/dist/shared/assistant/voice/handsfree/backendStreamTransport.js +118 -63
- package/dist/shared/assistant/voice/handsfree/bargeIn.d.ts +20 -1
- package/dist/shared/assistant/voice/handsfree/bargeIn.js +3 -3
- package/dist/shared/assistant/voice/handsfree/handsFreeController.d.ts +10 -1
- package/dist/shared/assistant/voice/handsfree/handsFreeController.js +103 -83
- package/dist/shared/assistant/voice/handsfree/sttTransport.d.ts +20 -0
- package/dist/shared/assistant/voice/handsfree/useHandsFreeInterrupt.d.ts +8 -0
- package/dist/shared/assistant/voice/handsfree/useHandsFreeInterrupt.js +31 -26
- package/dist/shared/assistant/voice/handsfree/webSpeechTransport.js +1 -0
- package/dist/shared-assistant.js +399 -399
- package/package.json +1 -1
|
@@ -70,6 +70,22 @@ export declare function planSubmitTimer(input: {
|
|
|
70
70
|
holding: boolean;
|
|
71
71
|
/** The preference, in ms. */
|
|
72
72
|
graceMs: number;
|
|
73
|
+
/**
|
|
74
|
+
* ★ BOFF-12936: words of this turn are still to come from a recogniser that
|
|
75
|
+
* sends them only once an utterance is finalised (our stream on chirp_3).
|
|
76
|
+
* The person is talking, or their last phrase is being finalised. Nothing is
|
|
77
|
+
* sent until the words are in, so a question with a pause in it is never cut
|
|
78
|
+
* after its first phrase.
|
|
79
|
+
*/
|
|
80
|
+
wordsPending: boolean;
|
|
81
|
+
/**
|
|
82
|
+
* On such a recogniser only, else null: how long ago the person's last voiced
|
|
83
|
+
* frame was. Its words arrive about 1.6 s after the person stops (alpha,
|
|
84
|
+
* 17 Nov), so the pause counts from when they stopped, not from when the
|
|
85
|
+
* words came. Otherwise every turn there would wait that much longer than on
|
|
86
|
+
* Chrome's recogniser, whose words arrive as they are spoken.
|
|
87
|
+
*/
|
|
88
|
+
speechEndedAgoMs: number | null;
|
|
73
89
|
}): {
|
|
74
90
|
arm: boolean;
|
|
75
91
|
delayMs: number;
|
|
@@ -11,8 +11,8 @@ function i(e) {
|
|
|
11
11
|
function a(e) {
|
|
12
12
|
let t = i(e.graceMs);
|
|
13
13
|
return {
|
|
14
|
-
arm: e.phase === "listening" && !e.muted && !e.holding && e.spoken.trim().length > 0,
|
|
15
|
-
delayMs: t
|
|
14
|
+
arm: e.phase === "listening" && !e.muted && !e.holding && !e.wordsPending && e.spoken.trim().length > 0,
|
|
15
|
+
delayMs: t - (e.speechEndedAgoMs === null ? 0 : Math.min(t, Math.max(0, e.speechEndedAgoMs)))
|
|
16
16
|
};
|
|
17
17
|
}
|
|
18
18
|
function o(e) {
|
|
@@ -19,6 +19,7 @@ export interface StreamConnection {
|
|
|
19
19
|
token: string;
|
|
20
20
|
}
|
|
21
21
|
type IntervalHandle = ReturnType<typeof setInterval>;
|
|
22
|
+
type TimeoutHandle = ReturnType<typeof setTimeout>;
|
|
22
23
|
export interface BackendStreamDeps {
|
|
23
24
|
/**
|
|
24
25
|
* The URL and the access token for one connection. `null`: there is no
|
|
@@ -33,6 +34,8 @@ export interface BackendStreamDeps {
|
|
|
33
34
|
keepAliveMs?: number;
|
|
34
35
|
setInterval?: (fn: () => void, ms: number) => IntervalHandle;
|
|
35
36
|
clearInterval?: (handle: IntervalHandle) => void;
|
|
37
|
+
setTimeout?: (fn: () => void, ms: number) => TimeoutHandle;
|
|
38
|
+
clearTimeout?: (handle: TimeoutHandle) => void;
|
|
36
39
|
now?: () => number;
|
|
37
40
|
}
|
|
38
41
|
/** The gateway's frame cap is 8 KB; stay well under it. */
|
|
@@ -50,6 +53,14 @@ export declare const CLOSE_CODE_ERRORS: Readonly<Record<number, {
|
|
|
50
53
|
export declare const RECONNECT_CLOSE_CODES: ReadonlySet<number>;
|
|
51
54
|
export declare const MIN_STABLE_MS = 5000;
|
|
52
55
|
export declare const KEEP_ALIVE_MS = 20000;
|
|
56
|
+
/**
|
|
57
|
+
* How long an utterance waits for the previous one to be done. chirp_3 took
|
|
58
|
+
* 1.0 s from `end` to `done` on alpha (17 Nov); past this the previous one is
|
|
59
|
+
* cancelled, so the person's new words are never held back for long.
|
|
60
|
+
*/
|
|
61
|
+
export declare const HOLD_MAX_MS = 3000;
|
|
62
|
+
/** Frames a waiting utterance may buffer (4 s), whatever the clock says. */
|
|
63
|
+
export declare const MAX_HELD_FRAMES = 200;
|
|
53
64
|
/** Concatenate frames into chunks of at most `MAX_SEND_BYTES`. */
|
|
54
65
|
export declare function chunkFrames(frames: readonly Int16Array[]): Int16Array[];
|
|
55
66
|
export declare function createBackendStreamTransport(deps: BackendStreamDeps): SttTransport;
|
|
@@ -38,69 +38,105 @@ var t = 6400, n = 1, r = {
|
|
|
38
38
|
1001,
|
|
39
39
|
4408,
|
|
40
40
|
4503
|
|
41
|
-
]), a = 5e3, o = 2e4, s = (e) => typeof e == "object" && e && !Array.isArray(e) ? e : null;
|
|
42
|
-
function
|
|
41
|
+
]), a = 5e3, o = 2e4, s = 3e3, c = 200, l = (e) => typeof e == "object" && e && !Array.isArray(e) ? e : null;
|
|
42
|
+
function u(e) {
|
|
43
43
|
let n = Math.floor(t / 2), r = [], i = [], a = () => {
|
|
44
44
|
i.length > 0 && r.push(Int16Array.from(i)), i = [];
|
|
45
45
|
};
|
|
46
46
|
for (let t of e) for (let e of t) i.push(e), i.length === n && a();
|
|
47
47
|
return a(), r;
|
|
48
48
|
}
|
|
49
|
-
function
|
|
50
|
-
let a = t.now ?? (() => Date.now()), o = t.setInterval ?? ((e, t) => setInterval(e, t)),
|
|
51
|
-
|
|
49
|
+
function d(t) {
|
|
50
|
+
let a = t.now ?? (() => Date.now()), o = t.setInterval ?? ((e, t) => setInterval(e, t)), c = t.clearInterval ?? ((e) => clearInterval(e)), d = t.keepAliveMs ?? 2e4, f = t.setTimeout ?? ((e, t) => setTimeout(e, t)), p = t.clearTimeout ?? ((e) => clearTimeout(e)), m = null, h = null, g = "en-IN", _ = null, v = null, y = null, b = null, x = null, S = null, C = null, w = !1, T = !1, E = (e, t) => {
|
|
51
|
+
T || w || (T = !0, h?.onError({
|
|
52
52
|
code: e,
|
|
53
53
|
retryable: t
|
|
54
54
|
}));
|
|
55
|
-
},
|
|
56
|
-
|
|
57
|
-
},
|
|
58
|
-
|
|
59
|
-
},
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
},
|
|
63
|
-
},
|
|
64
|
-
|
|
65
|
-
},
|
|
55
|
+
}, D = (e) => {
|
|
56
|
+
m && m.readyState === n && m.send(e);
|
|
57
|
+
}, O = () => {
|
|
58
|
+
C !== null && c(C), C = null;
|
|
59
|
+
}, k = () => {
|
|
60
|
+
O(), C = o(() => {
|
|
61
|
+
D(JSON.stringify({ t: "ping" }));
|
|
62
|
+
}, d);
|
|
63
|
+
}, A = () => {
|
|
64
|
+
S !== null && p(S), S = null;
|
|
65
|
+
}, j = (e) => {
|
|
66
|
+
w || h?.onUtteranceDone(e);
|
|
67
|
+
}, M = () => {
|
|
68
|
+
A();
|
|
69
|
+
let e = x;
|
|
70
|
+
if (x = null, !e) return;
|
|
71
|
+
let { utterance: n } = e, r = t.languages(g);
|
|
72
|
+
if (r === null) j(n.utteranceId);
|
|
73
|
+
else {
|
|
74
|
+
D(JSON.stringify({
|
|
75
|
+
t: "config",
|
|
76
|
+
v: 1,
|
|
77
|
+
utteranceId: n.utteranceId,
|
|
78
|
+
epoch: n.epoch,
|
|
79
|
+
langs: r,
|
|
80
|
+
sampleRate: 16e3,
|
|
81
|
+
encoding: "s16le"
|
|
82
|
+
}));
|
|
83
|
+
for (let t of u([...n.preRoll, ...e.frames])) D(t);
|
|
84
|
+
e.ended && (D(JSON.stringify({
|
|
85
|
+
t: "end",
|
|
86
|
+
utteranceId: n.utteranceId
|
|
87
|
+
})), y = n.utteranceId, b = n.utteranceId);
|
|
88
|
+
}
|
|
89
|
+
}, N = (e) => {
|
|
90
|
+
b === e && (b = null, M());
|
|
91
|
+
}, P = () => {
|
|
92
|
+
let e = b;
|
|
93
|
+
e !== null && (D(JSON.stringify({
|
|
94
|
+
t: "cancel",
|
|
95
|
+
utteranceId: e
|
|
96
|
+
})), y === e && (y = null), b = null, j(e)), M();
|
|
97
|
+
}, F = (e) => {
|
|
98
|
+
v?.utteranceId === e && (v = null), y === e && (y = null);
|
|
99
|
+
}, I = (e) => {
|
|
66
100
|
if (typeof e != "string") return;
|
|
67
101
|
let t;
|
|
68
102
|
try {
|
|
69
|
-
t =
|
|
103
|
+
t = l(JSON.parse(e));
|
|
70
104
|
} catch {
|
|
71
105
|
return;
|
|
72
106
|
}
|
|
73
107
|
if (t) {
|
|
74
108
|
if (t.t === "partial" || t.t === "final") {
|
|
75
109
|
if (typeof t.text != "string" || typeof t.utteranceId != "string" || typeof t.epoch != "number") return;
|
|
76
|
-
t.t === "final" &&
|
|
110
|
+
t.t === "final" && y === t.utteranceId && (y = null), h?.onTranscript({
|
|
77
111
|
text: t.text,
|
|
78
112
|
final: t.t === "final",
|
|
79
113
|
utteranceId: t.utteranceId,
|
|
80
114
|
epoch: t.epoch,
|
|
81
|
-
language: typeof t.lang == "string" ? t.lang :
|
|
115
|
+
language: typeof t.lang == "string" ? t.lang : g
|
|
82
116
|
});
|
|
83
|
-
} else if (t.t === "done")
|
|
84
|
-
|
|
117
|
+
} else if (t.t === "done") {
|
|
118
|
+
if (typeof t.utteranceId != "string") return;
|
|
119
|
+
y === t.utteranceId && (y = null), h?.onUtteranceDone(t.utteranceId), N(t.utteranceId);
|
|
120
|
+
} else if (t.t === "error") {
|
|
85
121
|
let e = t.code === "quota" ? "quota" : t.code === "auth" ? "auth" : "upstream", n = t.retryable === !0;
|
|
86
122
|
if (e === "upstream" && n && typeof t.utteranceId == "string") {
|
|
87
|
-
|
|
123
|
+
F(t.utteranceId), h?.onUtteranceDone(t.utteranceId), N(t.utteranceId);
|
|
88
124
|
return;
|
|
89
125
|
}
|
|
90
|
-
|
|
126
|
+
E(e, n);
|
|
91
127
|
}
|
|
92
128
|
}
|
|
93
|
-
},
|
|
129
|
+
}, L = () => new Promise((n) => {
|
|
94
130
|
let o = !1, s = (e) => {
|
|
95
131
|
o || (o = !0, n(e));
|
|
96
132
|
};
|
|
97
133
|
t.resolveConnection().then((n) => {
|
|
98
|
-
if (
|
|
134
|
+
if (w) {
|
|
99
135
|
s(!1);
|
|
100
136
|
return;
|
|
101
137
|
}
|
|
102
138
|
if (n === null) {
|
|
103
|
-
|
|
139
|
+
E("auth", !1), s(!1);
|
|
104
140
|
return;
|
|
105
141
|
}
|
|
106
142
|
let o;
|
|
@@ -110,58 +146,74 @@ function l(t) {
|
|
|
110
146
|
s(!1);
|
|
111
147
|
return;
|
|
112
148
|
}
|
|
113
|
-
|
|
149
|
+
m = o;
|
|
114
150
|
let c = null;
|
|
115
151
|
o.listen({
|
|
116
152
|
open: () => {
|
|
117
|
-
|
|
153
|
+
m === o && (c = a(), k(), s(!0));
|
|
118
154
|
},
|
|
119
155
|
error: () => s(!1),
|
|
120
156
|
close: (e) => {
|
|
121
|
-
let t =
|
|
122
|
-
if (t && (
|
|
157
|
+
let t = m === o;
|
|
158
|
+
if (t && (m = null, O()), s(!1), !t || w || c === null) return;
|
|
123
159
|
if (i.has(e) && a() - c >= 5e3) {
|
|
124
|
-
|
|
160
|
+
R();
|
|
125
161
|
return;
|
|
126
162
|
}
|
|
127
163
|
let n = r[e];
|
|
128
|
-
n ?
|
|
164
|
+
n ? E(n.code, n.retryable) : E("closed", !0);
|
|
129
165
|
},
|
|
130
|
-
message:
|
|
166
|
+
message: I
|
|
131
167
|
});
|
|
132
168
|
}).catch(() => {
|
|
133
|
-
|
|
169
|
+
E("unavailable", !1), s(!1);
|
|
134
170
|
});
|
|
135
|
-
}),
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
171
|
+
}), R = () => {
|
|
172
|
+
let e = [
|
|
173
|
+
b,
|
|
174
|
+
x?.utterance.utteranceId ?? null,
|
|
175
|
+
v?.utteranceId ?? null
|
|
176
|
+
];
|
|
177
|
+
A(), x = null, b = null, v = null, y = null;
|
|
178
|
+
for (let t of new Set(e)) t !== null && j(t);
|
|
179
|
+
let t = L();
|
|
180
|
+
_ = t, t.then((e) => {
|
|
181
|
+
!e && _ === t && E("unavailable", !1);
|
|
140
182
|
});
|
|
141
183
|
};
|
|
142
184
|
return {
|
|
143
185
|
kind: "backend-stream",
|
|
144
186
|
recognitionOwner: "transport",
|
|
145
187
|
keepsOpeningWords: !0,
|
|
188
|
+
wordsArrive: "after-utterance",
|
|
146
189
|
async start(e, n, r) {
|
|
147
|
-
if (
|
|
148
|
-
|
|
190
|
+
if (h = r, g = n, w = !1, T = !1, t.languages(n) === null) {
|
|
191
|
+
E("unavailable", !1);
|
|
149
192
|
return;
|
|
150
193
|
}
|
|
151
|
-
let i =
|
|
152
|
-
|
|
194
|
+
let i = L();
|
|
195
|
+
_ = i, !await i && _ === i && E("unavailable", !1);
|
|
153
196
|
},
|
|
154
197
|
openUtterance(e) {
|
|
155
|
-
|
|
198
|
+
if (x && P(), v = {
|
|
156
199
|
utteranceId: e.utteranceId,
|
|
157
200
|
epoch: e.epoch
|
|
158
|
-
}
|
|
201
|
+
}, b !== null) {
|
|
202
|
+
x = {
|
|
203
|
+
utterance: e,
|
|
204
|
+
frames: [],
|
|
205
|
+
ended: !1
|
|
206
|
+
}, S = f(() => {
|
|
207
|
+
S = null, P();
|
|
208
|
+
}, s);
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
159
211
|
let n = e.preRoll;
|
|
160
|
-
(
|
|
161
|
-
if (!r ||
|
|
162
|
-
let i = t.languages(
|
|
212
|
+
(_ ?? Promise.resolve(!1)).then((r) => {
|
|
213
|
+
if (!r || v?.utteranceId !== e.utteranceId) return;
|
|
214
|
+
let i = t.languages(g);
|
|
163
215
|
if (i !== null) {
|
|
164
|
-
|
|
216
|
+
D(JSON.stringify({
|
|
165
217
|
t: "config",
|
|
166
218
|
v: 1,
|
|
167
219
|
utteranceId: e.utteranceId,
|
|
@@ -170,35 +222,38 @@ function l(t) {
|
|
|
170
222
|
sampleRate: 16e3,
|
|
171
223
|
encoding: "s16le"
|
|
172
224
|
}));
|
|
173
|
-
for (let e of
|
|
225
|
+
for (let e of u(n)) D(e);
|
|
174
226
|
}
|
|
175
227
|
});
|
|
176
228
|
},
|
|
177
229
|
pushFrame(e) {
|
|
178
|
-
|
|
230
|
+
v && (x && x.utterance.utteranceId === v.utteranceId ? (x.frames.push(e), x.frames.length > 200 && P()) : D(e));
|
|
179
231
|
},
|
|
180
232
|
closeUtterance(e) {
|
|
181
|
-
|
|
233
|
+
v?.utteranceId === e && (v = null, x && x.utterance.utteranceId === e ? x.ended = !0 : (D(JSON.stringify({
|
|
182
234
|
t: "end",
|
|
183
235
|
utteranceId: e
|
|
184
|
-
})),
|
|
236
|
+
})), y = e, b = e));
|
|
185
237
|
},
|
|
186
238
|
cancelUtterance(e) {
|
|
187
|
-
if (
|
|
188
|
-
else
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
239
|
+
if (x && x.utterance.utteranceId === e) A(), x = null, v?.utteranceId === e && (v = null);
|
|
240
|
+
else {
|
|
241
|
+
if (v?.utteranceId === e) v = null;
|
|
242
|
+
else if (y !== e) return;
|
|
243
|
+
y = null, D(JSON.stringify({
|
|
244
|
+
t: "cancel",
|
|
245
|
+
utteranceId: e
|
|
246
|
+
})), b === e && (b = null, M());
|
|
247
|
+
}
|
|
193
248
|
},
|
|
194
249
|
stop() {
|
|
195
|
-
|
|
196
|
-
let e =
|
|
197
|
-
|
|
250
|
+
w = !0, v = null, y = null, b = null, A(), x = null, O();
|
|
251
|
+
let e = m;
|
|
252
|
+
m = null, _ = null;
|
|
198
253
|
try {
|
|
199
254
|
e?.close(1e3);
|
|
200
255
|
} catch {}
|
|
201
256
|
}
|
|
202
257
|
};
|
|
203
258
|
}
|
|
204
|
-
export { r as CLOSE_CODE_ERRORS, o as KEEP_ALIVE_MS, t as MAX_SEND_BYTES, a as MIN_STABLE_MS, i as RECONNECT_CLOSE_CODES,
|
|
259
|
+
export { r as CLOSE_CODE_ERRORS, s as HOLD_MAX_MS, o as KEEP_ALIVE_MS, c as MAX_HELD_FRAMES, t as MAX_SEND_BYTES, a as MIN_STABLE_MS, i as RECONNECT_CLOSE_CODES, u as chunkFrames, d as createBackendStreamTransport };
|
|
@@ -34,8 +34,27 @@ export interface EpochGuard {
|
|
|
34
34
|
invalidate(): void;
|
|
35
35
|
}
|
|
36
36
|
export declare function createEpochGuard(): EpochGuard;
|
|
37
|
-
/**
|
|
37
|
+
/**
|
|
38
|
+
* Nothing heard this long after a barge-in: show "say that again, or tap to
|
|
39
|
+
* replay". Only for transports whose words arrive while the person speaks; the
|
|
40
|
+
* stream (`after-utterance`) is judged when its utterance is done instead.
|
|
41
|
+
*/
|
|
38
42
|
export declare const NOTHING_HEARD_MS = 1500;
|
|
43
|
+
/**
|
|
44
|
+
* The longest an ended utterance's words are waited for on a transport whose
|
|
45
|
+
* words arrive after the utterance (BOFF-12936): the backstop for a `done` or
|
|
46
|
+
* an error that never comes. It counts from the utterance's LOCAL end, so it
|
|
47
|
+
* covers the latest a valid final can come:
|
|
48
|
+
* - the stream may hold the utterance's `end` up to `HOLD_MAX_MS` (3 s) behind
|
|
49
|
+
* the one before it (backendStreamTransport);
|
|
50
|
+
* - the speech service then waits up to `SPEECH_STT_FINAL_TIMEOUT_MS` (8 s, its
|
|
51
|
+
* default) from the `end` it RECEIVES before it gives up and says so;
|
|
52
|
+
* - and 1 s for the network.
|
|
53
|
+
* At 9 s a held utterance's valid final could be dropped, and a real
|
|
54
|
+
* interruption called "nothing heard" (the independent review of #534).
|
|
55
|
+
* chirp_3 took 1.0 s from `end` on alpha (17 Nov).
|
|
56
|
+
*/
|
|
57
|
+
export declare const FINISHING_MAX_MS = 12000;
|
|
39
58
|
/** Phantom barge-ins (nothing heard afterwards) in one session before falling back to tap-first. */
|
|
40
59
|
export declare const PHANTOM_BARGE_INS_BEFORE_FALLBACK = 3;
|
|
41
60
|
/** Whether a barge-in that produced no words counts against the session. */
|
|
@@ -24,8 +24,8 @@ function t() {
|
|
|
24
24
|
}
|
|
25
25
|
};
|
|
26
26
|
}
|
|
27
|
-
var n = 1500, r = 3;
|
|
28
|
-
function
|
|
27
|
+
var n = 1500, r = 12e3, i = 3;
|
|
28
|
+
function a(e) {
|
|
29
29
|
return e >= 3;
|
|
30
30
|
}
|
|
31
|
-
export { n as NOTHING_HEARD_MS,
|
|
31
|
+
export { r as FINISHING_MAX_MS, n as NOTHING_HEARD_MS, i as PHANTOM_BARGE_INS_BEFORE_FALLBACK, t as createEpochGuard, e as planOnset, a as shouldFallBackToTap };
|
|
@@ -31,8 +31,17 @@ export interface ControllerDeps {
|
|
|
31
31
|
onTranscript: (text: string, final: boolean, language: string | undefined) => void;
|
|
32
32
|
/** Hands-free gave up: the session falls back to the tap-first Web Speech path. */
|
|
33
33
|
onFallback: (reason: FallbackReason) => void;
|
|
34
|
-
/**
|
|
34
|
+
/**
|
|
35
|
+
* A barge-in that produced no words: within `NOTHING_HEARD_MS`, or by the end
|
|
36
|
+
* of its utterance when the transport's words arrive `after-utterance`.
|
|
37
|
+
*/
|
|
35
38
|
onNothingHeard?: () => void;
|
|
39
|
+
/**
|
|
40
|
+
* Whether an utterance's words are still to come: one is open, or ended and
|
|
41
|
+
* not yet done, on a transport whose words arrive `after-utterance`. The
|
|
42
|
+
* session must not send the turn meanwhile (BOFF-12936). Called on change.
|
|
43
|
+
*/
|
|
44
|
+
onWordsPending?: (pending: boolean) => void;
|
|
36
45
|
/** Timing only: end of the latest echo-approved voiced capture frame. */
|
|
37
46
|
onSpeechFrame?: (at: number) => void;
|
|
38
47
|
now?: () => number;
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import "./captureWorklet.js";
|
|
2
|
-
import {
|
|
3
|
-
import { DEFAULT_ECHO_GATE_PARAMS as
|
|
4
|
-
import { MicCaptureError as
|
|
5
|
-
import { PreRollRing as
|
|
6
|
-
import { DEFAULT_VAD_PARAMS as
|
|
2
|
+
import { FINISHING_MAX_MS as e, NOTHING_HEARD_MS as t, createEpochGuard as n, planOnset as r, shouldFallBackToTap as i } from "./bargeIn.js";
|
|
3
|
+
import { DEFAULT_ECHO_GATE_PARAMS as a, EchoGate as o } from "./echoGate.js";
|
|
4
|
+
import { MicCaptureError as s } from "./micCapture.js";
|
|
5
|
+
import { PreRollRing as c } from "./preRollRing.js";
|
|
6
|
+
import { DEFAULT_VAD_PARAMS as l, createVad as u } from "./vad.js";
|
|
7
7
|
// src/shared/assistant/voice/handsfree/handsFreeController.ts
|
|
8
|
-
var
|
|
8
|
+
var d = a.referenceSilentDb, f = a.tailFrames, p = {
|
|
9
9
|
unavailable: "transport-unavailable",
|
|
10
10
|
auth: "transport-auth",
|
|
11
11
|
quota: "transport-quota",
|
|
@@ -13,113 +13,133 @@ var u = i.referenceSilentDb, d = i.tailFrames, f = {
|
|
|
13
13
|
closed: "transport-upstream",
|
|
14
14
|
"not-allowed": "mic-denied"
|
|
15
15
|
};
|
|
16
|
-
function
|
|
17
|
-
let
|
|
18
|
-
|
|
16
|
+
function m(a) {
|
|
17
|
+
let m = a.setTimer ?? ((e, t) => setTimeout(e, t)), h = a.clearTimer ?? ((e) => clearTimeout(e)), g = new o(), _ = u(), v = new c(a.prerollMs ?? 700, 20), y = n(), b = null, x = !1, S = 0, C = null, w = null, T = 0, E = null, D = -1, O = 0, k = null, A = null, j = /* @__PURE__ */ new Map(), M = !1, N = !1, P = (e) => a.onTelemetry?.(e), F = (e) => {
|
|
18
|
+
N || (N = !0, P({
|
|
19
19
|
type: "fallback",
|
|
20
20
|
reason: e
|
|
21
|
-
}),
|
|
22
|
-
},
|
|
23
|
-
|
|
24
|
-
},
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
let
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
21
|
+
}), K(), a.onFallback(e));
|
|
22
|
+
}, I = () => {
|
|
23
|
+
k !== null && h(k), k = null, A = null;
|
|
24
|
+
}, L = () => a.transport.recognitionOwner === "transport" && a.transport.wordsArrive === "after-utterance", R = () => {
|
|
25
|
+
let e = x && L() && (C !== null || j.size > 0);
|
|
26
|
+
e !== M && (M = e, a.onWordsPending?.(e));
|
|
27
|
+
}, z = () => {
|
|
28
|
+
O += 1, P({ type: "nothing-heard" }), a.onNothingHeard?.(), i(O) && F("phantom-barge-ins");
|
|
29
|
+
}, B = (e, t) => e === "avatar" ? (t > d && (D = S + f), S <= D ? "avatar" : null) : (D = -1, e === "avatar-unrouted" ? a.getSession().phase === "speaking" ? "device" : null : e), V = (e) => {
|
|
30
|
+
if (e === E) return;
|
|
31
|
+
let t = E;
|
|
32
|
+
E = e, e === null ? g.playbackStopped() : (t === null || t !== e) && g.playbackStarted(e === "hosted" || e === "avatar"), _.setOnsetFrames(e === "device" ? 6 : l.onsetFrames);
|
|
33
|
+
}, H = (e) => {
|
|
34
|
+
let t = j.get(e);
|
|
35
|
+
t && (h(t.timer), j.delete(e)), C?.id === e && (C.done = !0), A !== null && e === A && (A = null, z()), R();
|
|
36
|
+
}, U = () => {
|
|
37
|
+
if (!C) return;
|
|
38
|
+
let t = C;
|
|
39
|
+
a.transport.closeUtterance(t.id), C = null, L() && !t.done && j.set(t.id, {
|
|
40
|
+
epoch: t.epoch,
|
|
41
|
+
timer: m(() => H(t.id), e)
|
|
42
|
+
}), R();
|
|
43
|
+
}, W = (e) => {
|
|
44
|
+
let n = a.getSession(), i = r({
|
|
45
|
+
enabled: a.isEnabled(),
|
|
46
|
+
muted: n.muted,
|
|
47
|
+
phase: n.phase,
|
|
48
|
+
playbackAudible: E !== null
|
|
36
49
|
});
|
|
37
|
-
if (
|
|
50
|
+
if (P({
|
|
38
51
|
type: "onset",
|
|
39
|
-
plan:
|
|
40
|
-
atFrame:
|
|
41
|
-
playbackSource:
|
|
42
|
-
gate:
|
|
43
|
-
calibration:
|
|
44
|
-
}),
|
|
45
|
-
|
|
52
|
+
plan: i.kind,
|
|
53
|
+
atFrame: S,
|
|
54
|
+
playbackSource: E,
|
|
55
|
+
gate: e,
|
|
56
|
+
calibration: g.calibration
|
|
57
|
+
}), i.kind === "ignore") return;
|
|
58
|
+
i.kind === "barge-in" && (a.onBargeIn(), P({
|
|
46
59
|
type: "stopped-playback",
|
|
47
|
-
frame:
|
|
48
|
-
}),
|
|
49
|
-
|
|
50
|
-
},
|
|
51
|
-
let
|
|
52
|
-
|
|
53
|
-
id: `u${
|
|
54
|
-
epoch:
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
60
|
+
frame: S
|
|
61
|
+
}), g.playbackStopped(), E = null, I(), a.transport.recognitionOwner === "transport" && a.transport.wordsArrive === "while-speaking" && (k = m(() => {
|
|
62
|
+
k = null, z();
|
|
63
|
+
}, t))), w && !j.has(w) && a.transport.cancelUtterance(w);
|
|
64
|
+
let o = y.open();
|
|
65
|
+
T += 1, C = {
|
|
66
|
+
id: `u${T}`,
|
|
67
|
+
epoch: o,
|
|
68
|
+
done: !1
|
|
69
|
+
}, w = C.id, (A !== null || i.kind === "barge-in" && L()) && (A = C.id), a.transport.openUtterance({
|
|
70
|
+
utteranceId: C.id,
|
|
71
|
+
epoch: o,
|
|
72
|
+
preRoll: v.snapshot()
|
|
73
|
+
}), R();
|
|
74
|
+
}, G = (e) => {
|
|
75
|
+
if (!x) return;
|
|
76
|
+
S += 1, V(B(a.getPlaybackSource(), e.refDb)), a.getHeadsetHint && g.setHeadset(a.getHeadsetHint());
|
|
77
|
+
let t = g.pushFrame(e.rmsDb, e.refDb), n = new Int16Array(e.pcm);
|
|
78
|
+
v.push(n);
|
|
79
|
+
let r = _.push({
|
|
66
80
|
rmsDb: e.rmsDb,
|
|
67
81
|
zcr: e.zcr
|
|
68
82
|
}, t.pass);
|
|
69
|
-
if (r.event === "onset" &&
|
|
70
|
-
let t =
|
|
71
|
-
|
|
83
|
+
if (r.event === "onset" && W(t.reason), C && r.voiced && t.pass) {
|
|
84
|
+
let t = a.now?.() ?? performance.now(), n = b?.context ? Math.max(0, (b.context.currentTime - e.ctxTime) * 1e3) : 0;
|
|
85
|
+
a.onSpeechFrame?.(t - n);
|
|
72
86
|
}
|
|
73
|
-
r.event !== "onset" &&
|
|
74
|
-
},
|
|
75
|
-
if (!
|
|
76
|
-
|
|
77
|
-
let e
|
|
78
|
-
|
|
87
|
+
r.event !== "onset" && C && r.inSegment && a.transport.pushFrame(n), r.event === "end" && U();
|
|
88
|
+
}, K = () => {
|
|
89
|
+
if (!x && !b) return;
|
|
90
|
+
x = !1, I();
|
|
91
|
+
for (let { timer: e } of j.values()) h(e);
|
|
92
|
+
j.clear(), R(), C && a.transport.cancelUtterance(C.id), C = null, y.invalidate(), a.transport.stop(), v.clear(), g.reset(), _.reset();
|
|
93
|
+
let e = b;
|
|
94
|
+
b = null, e?.stop();
|
|
79
95
|
};
|
|
80
96
|
return {
|
|
81
97
|
async start() {
|
|
82
|
-
if (
|
|
83
|
-
let e =
|
|
84
|
-
if (!
|
|
85
|
-
|
|
98
|
+
if (x || N) return;
|
|
99
|
+
let e = a.getSession();
|
|
100
|
+
if (!a.isEnabled() || e.muted || e.phase === "ended" || e.phase === "idle") return;
|
|
101
|
+
x = !0;
|
|
86
102
|
try {
|
|
87
|
-
|
|
103
|
+
b = await a.startCapture(G);
|
|
88
104
|
} catch (e) {
|
|
89
|
-
|
|
105
|
+
x = !1, F(e instanceof s && e.reason === "not-allowed" ? "mic-denied" : "mic-unavailable");
|
|
90
106
|
return;
|
|
91
107
|
}
|
|
92
|
-
if (!
|
|
93
|
-
|
|
108
|
+
if (!x) {
|
|
109
|
+
b.stop(), b = null;
|
|
94
110
|
return;
|
|
95
111
|
}
|
|
96
|
-
let t =
|
|
97
|
-
context:
|
|
98
|
-
micSource:
|
|
99
|
-
track:
|
|
112
|
+
let t = b && "context" in b && b.context && b.micSource && b.track ? {
|
|
113
|
+
context: b.context,
|
|
114
|
+
micSource: b.micSource,
|
|
115
|
+
track: b.track
|
|
100
116
|
} : null;
|
|
101
|
-
await
|
|
117
|
+
await a.transport.start(t, a.language(), {
|
|
102
118
|
onTranscript(e) {
|
|
103
|
-
|
|
119
|
+
let t = j.get(e.utteranceId), n = e.final && t !== void 0 && t.epoch === e.epoch;
|
|
120
|
+
!y.accepts(e.epoch) && !n ? P({
|
|
104
121
|
type: "stale-transcript-dropped",
|
|
105
122
|
epoch: e.epoch,
|
|
106
|
-
current:
|
|
107
|
-
});
|
|
123
|
+
current: y.current()
|
|
124
|
+
}) : (I(), a.onTranscript(e.text, e.final, e.language));
|
|
125
|
+
},
|
|
126
|
+
onUtteranceDone(e) {
|
|
127
|
+
H(e);
|
|
108
128
|
},
|
|
109
129
|
onError(e) {
|
|
110
|
-
|
|
130
|
+
F(p[e.code]);
|
|
111
131
|
}
|
|
112
132
|
});
|
|
113
133
|
},
|
|
114
|
-
stop:
|
|
134
|
+
stop: K,
|
|
115
135
|
get frames() {
|
|
116
|
-
return
|
|
136
|
+
return S;
|
|
117
137
|
},
|
|
118
|
-
feed:
|
|
119
|
-
epoch:
|
|
138
|
+
feed: G,
|
|
139
|
+
epoch: y,
|
|
120
140
|
get running() {
|
|
121
|
-
return
|
|
141
|
+
return x;
|
|
122
142
|
}
|
|
123
143
|
};
|
|
124
144
|
}
|
|
125
|
-
export {
|
|
145
|
+
export { m as createHandsFreeController };
|