@burdenoff/fe-libs 2026.1118.8 → 2026.1118.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/shared/assistant/index.d.ts +2 -0
- package/dist/shared/assistant/ui/useVoiceSession.js +188 -142
- package/dist/shared/assistant/voice/handsFree.d.ts +3 -1
- package/dist/shared/assistant/voice/handsfree/audioTrackTransport.d.ts +8 -0
- package/dist/shared/assistant/voice/handsfree/audioTrackTransport.js +36 -31
- package/dist/shared/assistant/voice/handsfree/backendStreamTransport.js +1 -0
- package/dist/shared/assistant/voice/handsfree/echoGate.d.ts +21 -0
- package/dist/shared/assistant/voice/handsfree/echoGate.js +11 -2
- package/dist/shared/assistant/voice/handsfree/handsFreeController.d.ts +7 -0
- package/dist/shared/assistant/voice/handsfree/handsFreeController.js +46 -42
- package/dist/shared/assistant/voice/handsfree/playbackReference.d.ts +63 -5
- package/dist/shared/assistant/voice/handsfree/playbackReference.js +56 -29
- package/dist/shared/assistant/voice/handsfree/preRollRing.d.ts +7 -0
- package/dist/shared/assistant/voice/handsfree/preRollRing.js +3 -0
- package/dist/shared/assistant/voice/handsfree/sttTransport.d.ts +16 -0
- package/dist/shared/assistant/voice/handsfree/useHandsFreeInterrupt.js +55 -54
- package/dist/shared/assistant/voice/handsfree/vad.d.ts +9 -0
- package/dist/shared/assistant/voice/handsfree/vad.js +10 -7
- package/dist/shared/assistant/voice/handsfree/webSpeechTransport.js +1 -0
- package/dist/shared/assistant/voice/thinkingTone.d.ts +213 -0
- package/dist/shared/assistant/voice/thinkingTone.js +182 -0
- package/dist/shared/assistant/voice/thinkingToneConfig.d.ts +4 -0
- package/dist/shared/assistant/voice/thinkingToneConfig.js +9 -0
- package/dist/shared/assistant/voice/voiceSession.d.ts +24 -0
- package/dist/shared/assistant/voice/voiceSession.js +18 -11
- package/dist/shared-assistant.js +404 -402
- package/dist/ui/gallery-category-rail.d.ts +49 -0
- package/dist/ui/gallery-category-rail.js +110 -0
- package/dist/ui/gallery-pager-labels.d.ts +22 -0
- package/dist/ui/gallery-pager-labels.js +12 -0
- package/dist/ui/gallery-pager.d.ts +19 -0
- package/dist/ui/gallery-pager.js +89 -0
- package/dist/ui/gallery-paging.d.ts +45 -0
- package/dist/ui/gallery-paging.js +66 -0
- package/dist/ui/index.d.ts +4 -0
- package/dist/ui.js +49 -45
- package/package.json +1 -1
|
@@ -1,40 +1,40 @@
|
|
|
1
1
|
// src/shared/assistant/voice/handsfree/audioTrackTransport.ts
|
|
2
|
-
var e = .3, t = 1500;
|
|
3
|
-
function
|
|
4
|
-
let n = t.now ?? (() => Date.now()),
|
|
5
|
-
|
|
6
|
-
},
|
|
7
|
-
if (!
|
|
2
|
+
var e = .3, t = 1500, n = 400, r = .005;
|
|
3
|
+
function i(t) {
|
|
4
|
+
let n = t.now ?? (() => Date.now()), i = null, a = null, o = null, s = null, c = "en-US", l = null, u = !1, d = null, f = 0, p = null, m = (e) => {
|
|
5
|
+
a && o && (a.gain.cancelScheduledValues(o.currentTime), a.gain.setTargetAtTime(e, o.currentTime, r));
|
|
6
|
+
}, h = () => {
|
|
7
|
+
if (!u || !l) return;
|
|
8
8
|
let e = t.createRecognition();
|
|
9
|
-
if (!e)
|
|
9
|
+
if (!e) s?.onError({
|
|
10
10
|
code: "unavailable",
|
|
11
11
|
retryable: !1
|
|
12
12
|
});
|
|
13
13
|
else {
|
|
14
|
-
|
|
15
|
-
let t =
|
|
14
|
+
i = e, e.lang = c, e.continuous = !0, e.interimResults = !0, e.onresult = (e) => {
|
|
15
|
+
let t = d ?? (p && n() - f <= 1500 ? p : null);
|
|
16
16
|
if (t) for (let n = e.resultIndex; n < e.results.length; n += 1) {
|
|
17
17
|
let r = e.results[n], i = r?.[0]?.transcript?.trim() ?? "";
|
|
18
|
-
r && i &&
|
|
18
|
+
r && i && s?.onTranscript({
|
|
19
19
|
text: i,
|
|
20
20
|
final: r.isFinal,
|
|
21
21
|
utteranceId: t.utteranceId,
|
|
22
22
|
epoch: t.epoch,
|
|
23
|
-
language:
|
|
23
|
+
language: c
|
|
24
24
|
});
|
|
25
25
|
}
|
|
26
26
|
}, e.onerror = (e) => {
|
|
27
|
-
(e.error === "not-allowed" || e.error === "service-not-allowed") && (
|
|
27
|
+
(e.error === "not-allowed" || e.error === "service-not-allowed") && (u = !1, s?.onError({
|
|
28
28
|
code: "not-allowed",
|
|
29
29
|
retryable: !1
|
|
30
30
|
}));
|
|
31
31
|
}, e.onend = () => {
|
|
32
|
-
|
|
32
|
+
u && i === e && h();
|
|
33
33
|
};
|
|
34
34
|
try {
|
|
35
|
-
e.start(
|
|
35
|
+
e.start(l);
|
|
36
36
|
} catch {
|
|
37
|
-
|
|
37
|
+
s?.onError({
|
|
38
38
|
code: "unavailable",
|
|
39
39
|
retryable: !1
|
|
40
40
|
});
|
|
@@ -54,44 +54,49 @@ function n(t) {
|
|
|
54
54
|
});
|
|
55
55
|
return;
|
|
56
56
|
}
|
|
57
|
-
|
|
58
|
-
let
|
|
59
|
-
|
|
57
|
+
s = r, c = n, o = t.context;
|
|
58
|
+
let i = t.context.createDelay(1);
|
|
59
|
+
i.delayTime.value = e, a = t.context.createGain(), a.gain.value = 0;
|
|
60
60
|
let d = t.context.createMediaStreamDestination();
|
|
61
|
-
t.micSource.connect(
|
|
61
|
+
t.micSource.connect(i), i.connect(a), a.connect(d), l = d.stream.getAudioTracks()[0] ?? null, u = !0, h();
|
|
62
62
|
},
|
|
63
63
|
openUtterance(e) {
|
|
64
|
-
|
|
64
|
+
d = {
|
|
65
65
|
utteranceId: e.utteranceId,
|
|
66
66
|
epoch: e.epoch
|
|
67
|
-
},
|
|
67
|
+
}, p = d, m(1);
|
|
68
68
|
},
|
|
69
69
|
pushFrame() {},
|
|
70
70
|
closeUtterance(e) {
|
|
71
|
-
if (
|
|
72
|
-
|
|
71
|
+
if (d?.utteranceId !== e) return;
|
|
72
|
+
f = n(), d = null;
|
|
73
73
|
let t = e;
|
|
74
74
|
setTimeout(() => {
|
|
75
|
-
|
|
75
|
+
d === null && p?.utteranceId === t && m(0);
|
|
76
76
|
}, 400);
|
|
77
77
|
},
|
|
78
78
|
cancelUtterance(e) {
|
|
79
|
-
|
|
79
|
+
d?.utteranceId === e && (d = null), p?.utteranceId === e && (p = null), m(0);
|
|
80
|
+
},
|
|
81
|
+
cueStarted(t) {
|
|
82
|
+
if (!a || !o) return;
|
|
83
|
+
let n = t + e - 5 * r;
|
|
84
|
+
o.currentTime <= n ? a.gain.setTargetAtTime(0, n, r) : (d = null, p = null, m(0));
|
|
80
85
|
},
|
|
81
86
|
stop() {
|
|
82
|
-
|
|
83
|
-
let e =
|
|
84
|
-
if (
|
|
87
|
+
u = !1, d = null, p = null;
|
|
88
|
+
let e = i;
|
|
89
|
+
if (i = null, e) {
|
|
85
90
|
e.onresult = null, e.onend = null, e.onerror = null;
|
|
86
91
|
try {
|
|
87
92
|
e.abort();
|
|
88
93
|
} catch {}
|
|
89
94
|
}
|
|
90
95
|
try {
|
|
91
|
-
|
|
96
|
+
a?.disconnect();
|
|
92
97
|
} catch {}
|
|
93
|
-
|
|
98
|
+
l?.stop(), l = null;
|
|
94
99
|
}
|
|
95
100
|
};
|
|
96
101
|
}
|
|
97
|
-
export { e as BRANCH_DELAY_S, t as TRAILING_RESULT_MS,
|
|
102
|
+
export { e as BRANCH_DELAY_S, n as CLOSE_HOLD_MS, r as GAIN_TIME_CONSTANT_S, t as TRAILING_RESULT_MS, i as createAudioTrackTransport };
|
|
@@ -36,6 +36,9 @@
|
|
|
36
36
|
* 3. With NO reference (the device `speechSynthesis` voice does not pass
|
|
37
37
|
* through the browser's output graph) nothing can be attributed: refused
|
|
38
38
|
* while it speaks. Tap-to-interrupt is the fallback there, and it is said so.
|
|
39
|
+
* 4. ★ D3 (BOFF-12936), a CUE: the thinking tone, about 150 ms the moment a
|
|
40
|
+
* spoken turn is sent (`cueStarted`). Refused while it sounds and for the
|
|
41
|
+
* tail after it, whatever the reference says: see `cueStarted` for why.
|
|
39
42
|
*
|
|
40
43
|
* Known limit, measured in the browser test and the design spike: this is
|
|
41
44
|
* envelope evidence, not an adaptive filter. A person who speaks in lock-step
|
|
@@ -84,6 +87,8 @@ export type GateReason = 'no-playback' | 'calibrating'
|
|
|
84
87
|
| 'follows-reference'
|
|
85
88
|
/** Audible reference without enough shape to compare. */
|
|
86
89
|
| 'flat-reference'
|
|
90
|
+
/** D3: the assistant's own short cue (the thinking tone), or its tail: never the user. */
|
|
91
|
+
| 'cue'
|
|
87
92
|
/** The microphone does not follow the reference: this may be the user. */
|
|
88
93
|
| 'uncorrelated';
|
|
89
94
|
export interface GateVerdict {
|
|
@@ -109,6 +114,8 @@ export declare class EchoGate {
|
|
|
109
114
|
private prior;
|
|
110
115
|
/** The reference plays but no echo reaches the microphone (this playback only). */
|
|
111
116
|
private noEchoPath;
|
|
117
|
+
/** D3: this playback is a cue (`cueStarted`): nothing is attributed until its tail is over. */
|
|
118
|
+
private cue;
|
|
112
119
|
private lowRun;
|
|
113
120
|
private headset;
|
|
114
121
|
constructor(params?: Partial<EchoGateParams>);
|
|
@@ -123,6 +130,20 @@ export declare class EchoGate {
|
|
|
123
130
|
setHeadset(headset: boolean): void;
|
|
124
131
|
/** The assistant started making sound. `withReference`: it passes through the reference bus. */
|
|
125
132
|
playbackStarted(withReference: boolean): void;
|
|
133
|
+
/**
|
|
134
|
+
* ★ D3 (BOFF-12936): the assistant started a short CUE, the thinking tone. It
|
|
135
|
+
* is routed through the reference bus like a hosted piece, but it cannot be
|
|
136
|
+
* measured like one: about 150 ms is too short to calibrate on, and it is
|
|
137
|
+
* often the first sound of a session, before any reply has given a prior.
|
|
138
|
+
* Without a prior the gate looks only at the reference's CURRENT frame, so the
|
|
139
|
+
* tone's echo, arriving one room lag after the reference fell silent, would
|
|
140
|
+
* pass as the person. So nothing is attributed: every frame is refused while
|
|
141
|
+
* the cue sounds and for the gate's tail after it (`playbackStopped`), the
|
|
142
|
+
* same tail that covers a reply's lag and ringing room. The cost is a moment:
|
|
143
|
+
* a person who starts a correction inside it is heard once it is over, and the
|
|
144
|
+
* pre-roll keeps their opening words.
|
|
145
|
+
*/
|
|
146
|
+
cueStarted(): void;
|
|
126
147
|
playbackStopped(): void;
|
|
127
148
|
/** Feed one frame; returns whether THIS frame may be the user's. */
|
|
128
149
|
pushFrame(micDb: number, refDb: number): GateVerdict;
|
|
@@ -49,6 +49,7 @@ var o = class {
|
|
|
49
49
|
current = null;
|
|
50
50
|
prior = null;
|
|
51
51
|
noEchoPath = !1;
|
|
52
|
+
cue = !1;
|
|
52
53
|
lowRun = 0;
|
|
53
54
|
headset = !1;
|
|
54
55
|
constructor(t = {}) {
|
|
@@ -74,7 +75,10 @@ var o = class {
|
|
|
74
75
|
this.headset = e;
|
|
75
76
|
}
|
|
76
77
|
playbackStarted(e) {
|
|
77
|
-
this.playing = !0, this.hasReference = e, this.playStart = this.total, this.current = null, this.noEchoPath = !1, this.lowRun = 0;
|
|
78
|
+
this.playing = !0, this.hasReference = e, this.playStart = this.total, this.current = null, this.noEchoPath = !1, this.cue = !1, this.lowRun = 0;
|
|
79
|
+
}
|
|
80
|
+
cueStarted() {
|
|
81
|
+
this.playbackStarted(!0), this.cue = !0;
|
|
78
82
|
}
|
|
79
83
|
playbackStopped() {
|
|
80
84
|
this.playing && (this.playing = !1, this.armedUntil = this.total + this.params.tailFrames);
|
|
@@ -92,6 +96,11 @@ var o = class {
|
|
|
92
96
|
reason: "no-playback",
|
|
93
97
|
coherence: null
|
|
94
98
|
};
|
|
99
|
+
if (this.cue) return this.lowRun = 0, {
|
|
100
|
+
pass: !1,
|
|
101
|
+
reason: "cue",
|
|
102
|
+
coherence: null
|
|
103
|
+
};
|
|
95
104
|
if (!this.hasReference) return this.lowRun = 0, {
|
|
96
105
|
pass: !1,
|
|
97
106
|
reason: "no-reference",
|
|
@@ -187,7 +196,7 @@ var o = class {
|
|
|
187
196
|
this.mic.length <= t || (this.mic = this.mic.slice(r), this.ref = this.ref.slice(r), this.tail = this.tail.slice(r), this.base += r);
|
|
188
197
|
}
|
|
189
198
|
reset() {
|
|
190
|
-
this.mic = [], this.ref = [], this.tail = [], this.base = 0, this.total = 0, this.playing = !1, this.armedUntil = -1, this.current = null, this.prior = null, this.noEchoPath = !1, this.lowRun = 0;
|
|
199
|
+
this.mic = [], this.ref = [], this.tail = [], this.base = 0, this.total = 0, this.playing = !1, this.armedUntil = -1, this.current = null, this.prior = null, this.noEchoPath = !1, this.cue = !1, this.lowRun = 0;
|
|
191
200
|
}
|
|
192
201
|
};
|
|
193
202
|
export { e as DEFAULT_ECHO_GATE_PARAMS, o as EchoGate };
|
|
@@ -44,6 +44,13 @@ export interface ControllerDeps {
|
|
|
44
44
|
onWordsPending?: (pending: boolean) => void;
|
|
45
45
|
/** Timing only: end of the latest echo-approved voiced capture frame. */
|
|
46
46
|
onSpeechFrame?: (at: number) => void;
|
|
47
|
+
/**
|
|
48
|
+
* ★ D3 review round 2 (BOFF-12936): be told, synchronously, of every cue
|
|
49
|
+
* scheduled on the capture graph's context, with its start on that context
|
|
50
|
+
* (`subscribeCaptureCues`). Returns the unsubscribe. Without it the transport
|
|
51
|
+
* learns of a cue only from the first frame that shows it.
|
|
52
|
+
*/
|
|
53
|
+
subscribeCaptureCues?: (listener: (startsAt: number) => void) => () => void;
|
|
47
54
|
now?: () => number;
|
|
48
55
|
isEnabled: () => boolean;
|
|
49
56
|
prerollMs?: number;
|
|
@@ -14,40 +14,42 @@ var d = a.referenceSilentDb, f = a.tailFrames, p = {
|
|
|
14
14
|
"not-allowed": "mic-denied"
|
|
15
15
|
};
|
|
16
16
|
function m(a) {
|
|
17
|
-
let m = a.setTimer ?? ((e, t) => setTimeout(e, t)), h = a.clearTimer ?? ((e) => clearTimeout(e)), g = new o(), _ = u(), v = new c(a.prerollMs ?? 700, 20), y = n(), b = null, x = !1, S = 0, C = null, w = null, T = 0, E = null, D = -1, O = 0, k = null, A = null, j = /* @__PURE__ */ new Map(), M = !1, N = !1, P = (e) => a.onTelemetry?.(e),
|
|
18
|
-
N || (N = !0,
|
|
17
|
+
let m = a.setTimer ?? ((e, t) => setTimeout(e, t)), h = a.clearTimer ?? ((e) => clearTimeout(e)), g = new o(), _ = u(), v = new c(a.prerollMs ?? 700, 20), y = n(), b = null, x = !1, S = 0, C = null, w = null, T = 0, E = null, D = -1, O = 0, k = null, A = null, j = /* @__PURE__ */ new Map(), M = !1, N = !1, P = null, F = !1, I = (e) => a.onTelemetry?.(e), L = (e) => {
|
|
18
|
+
N || (N = !0, I({
|
|
19
19
|
type: "fallback",
|
|
20
20
|
reason: e
|
|
21
|
-
}),
|
|
22
|
-
},
|
|
21
|
+
}), Y(), a.onFallback(e));
|
|
22
|
+
}, R = () => {
|
|
23
23
|
k !== null && h(k), k = null, A = null;
|
|
24
|
-
},
|
|
25
|
-
let e = x &&
|
|
24
|
+
}, z = () => a.transport.recognitionOwner === "transport" && a.transport.wordsArrive === "after-utterance", B = () => {
|
|
25
|
+
let e = x && z() && (C !== null || j.size > 0);
|
|
26
26
|
e !== M && (M = e, a.onWordsPending?.(e));
|
|
27
|
-
},
|
|
28
|
-
O += 1,
|
|
29
|
-
},
|
|
27
|
+
}, V = () => {
|
|
28
|
+
O += 1, I({ type: "nothing-heard" }), a.onNothingHeard?.(), i(O) && L("phantom-barge-ins");
|
|
29
|
+
}, H = (e, t) => e === "avatar" ? (t > d && (D = S + f), S <= D ? "avatar" : null) : (D = -1, e === "avatar-unrouted" ? a.getSession().phase === "speaking" ? "device" : null : e), U = (e) => {
|
|
30
|
+
g.cueStarted(), _.endSegment(), F || a.transport.cueStarted(e), K();
|
|
31
|
+
}, W = (e, t) => {
|
|
30
32
|
if (e === E) return;
|
|
31
|
-
let
|
|
32
|
-
E = e, e === null ? g.playbackStopped() : (t === null ||
|
|
33
|
-
},
|
|
33
|
+
let n = E;
|
|
34
|
+
E = e, e === null ? g.playbackStopped() : e === "cue" ? U(t) : (n === null || n !== e) && g.playbackStarted(e === "hosted" || e === "avatar"), _.setOnsetFrames(e === "device" ? 6 : l.onsetFrames);
|
|
35
|
+
}, G = (e) => {
|
|
34
36
|
let t = j.get(e);
|
|
35
|
-
t && (h(t.timer), j.delete(e)), C?.id === e && (C.done = !0), A !== null && e === A && (A = null,
|
|
36
|
-
},
|
|
37
|
+
t && (h(t.timer), j.delete(e)), C?.id === e && (C.done = !0), A !== null && e === A && (A = null, V()), B();
|
|
38
|
+
}, K = () => {
|
|
37
39
|
if (!C) return;
|
|
38
40
|
let t = C;
|
|
39
|
-
a.transport.closeUtterance(t.id), C = null,
|
|
41
|
+
a.transport.closeUtterance(t.id), C = null, z() && !t.done && j.set(t.id, {
|
|
40
42
|
epoch: t.epoch,
|
|
41
|
-
timer: m(() =>
|
|
42
|
-
}),
|
|
43
|
-
},
|
|
43
|
+
timer: m(() => G(t.id), e)
|
|
44
|
+
}), B();
|
|
45
|
+
}, q = (e) => {
|
|
44
46
|
let n = a.getSession(), i = r({
|
|
45
47
|
enabled: a.isEnabled(),
|
|
46
48
|
muted: n.muted,
|
|
47
49
|
phase: n.phase,
|
|
48
50
|
playbackAudible: E !== null
|
|
49
51
|
});
|
|
50
|
-
if (
|
|
52
|
+
if (I({
|
|
51
53
|
type: "onset",
|
|
52
54
|
plan: i.kind,
|
|
53
55
|
atFrame: S,
|
|
@@ -55,41 +57,41 @@ function m(a) {
|
|
|
55
57
|
gate: e,
|
|
56
58
|
calibration: g.calibration
|
|
57
59
|
}), i.kind === "ignore") return;
|
|
58
|
-
i.kind === "barge-in" && (a.onBargeIn(),
|
|
60
|
+
i.kind === "barge-in" && (a.onBargeIn(), I({
|
|
59
61
|
type: "stopped-playback",
|
|
60
62
|
frame: S
|
|
61
|
-
}), g.playbackStopped(), E = null,
|
|
62
|
-
k = null,
|
|
63
|
+
}), g.playbackStopped(), E = null, R(), a.transport.recognitionOwner === "transport" && a.transport.wordsArrive === "while-speaking" && (k = m(() => {
|
|
64
|
+
k = null, V();
|
|
63
65
|
}, t))), w && !j.has(w) && a.transport.cancelUtterance(w);
|
|
64
66
|
let o = y.open();
|
|
65
67
|
T += 1, C = {
|
|
66
68
|
id: `u${T}`,
|
|
67
69
|
epoch: o,
|
|
68
70
|
done: !1
|
|
69
|
-
}, w = C.id, (A !== null || i.kind === "barge-in" &&
|
|
71
|
+
}, w = C.id, (A !== null || i.kind === "barge-in" && z()) && (A = C.id), a.transport.openUtterance({
|
|
70
72
|
utteranceId: C.id,
|
|
71
73
|
epoch: o,
|
|
72
74
|
preRoll: v.snapshot()
|
|
73
|
-
}),
|
|
74
|
-
},
|
|
75
|
+
}), B();
|
|
76
|
+
}, J = (e) => {
|
|
75
77
|
if (!x) return;
|
|
76
|
-
S += 1,
|
|
78
|
+
S += 1, W(H(a.getPlaybackSource(), e.refDb), e.ctxTime - 20 / 1e3), F = !1, a.getHeadsetHint && g.setHeadset(a.getHeadsetHint());
|
|
77
79
|
let t = g.pushFrame(e.rmsDb, e.refDb), n = new Int16Array(e.pcm);
|
|
78
|
-
v.push(n);
|
|
80
|
+
t.reason === "cue" ? v.discard() : v.push(n);
|
|
79
81
|
let r = _.push({
|
|
80
82
|
rmsDb: e.rmsDb,
|
|
81
83
|
zcr: e.zcr
|
|
82
84
|
}, t.pass);
|
|
83
|
-
if (r.event === "onset" &&
|
|
85
|
+
if (r.event === "onset" && q(t.reason), C && r.voiced && t.pass) {
|
|
84
86
|
let t = a.now?.() ?? performance.now(), n = b?.context ? Math.max(0, (b.context.currentTime - e.ctxTime) * 1e3) : 0;
|
|
85
87
|
a.onSpeechFrame?.(t - n);
|
|
86
88
|
}
|
|
87
|
-
r.event !== "onset" && C && r.inSegment && a.transport.pushFrame(n), r.event === "end" &&
|
|
88
|
-
},
|
|
89
|
-
if (!x && !b) return;
|
|
90
|
-
x = !1,
|
|
89
|
+
r.event !== "onset" && C && r.inSegment && a.transport.pushFrame(n), r.event === "end" && K();
|
|
90
|
+
}, Y = () => {
|
|
91
|
+
if (P?.(), P = null, F = !1, !x && !b) return;
|
|
92
|
+
x = !1, R();
|
|
91
93
|
for (let { timer: e } of j.values()) h(e);
|
|
92
|
-
j.clear(),
|
|
94
|
+
j.clear(), B(), C && a.transport.cancelUtterance(C.id), C = null, y.invalidate(), a.transport.stop(), v.clear(), g.reset(), _.reset();
|
|
93
95
|
let e = b;
|
|
94
96
|
b = null, e?.stop();
|
|
95
97
|
};
|
|
@@ -100,9 +102,9 @@ function m(a) {
|
|
|
100
102
|
if (!a.isEnabled() || e.muted || e.phase === "ended" || e.phase === "idle") return;
|
|
101
103
|
x = !0;
|
|
102
104
|
try {
|
|
103
|
-
b = await a.startCapture(
|
|
105
|
+
b = await a.startCapture(J);
|
|
104
106
|
} catch (e) {
|
|
105
|
-
x = !1,
|
|
107
|
+
x = !1, L(e instanceof s && e.reason === "not-allowed" ? "mic-denied" : "mic-unavailable");
|
|
106
108
|
return;
|
|
107
109
|
}
|
|
108
110
|
if (!x) {
|
|
@@ -117,25 +119,27 @@ function m(a) {
|
|
|
117
119
|
await a.transport.start(t, a.language(), {
|
|
118
120
|
onTranscript(e) {
|
|
119
121
|
let t = j.get(e.utteranceId), n = e.final && t !== void 0 && t.epoch === e.epoch;
|
|
120
|
-
!y.accepts(e.epoch) && !n ?
|
|
122
|
+
!y.accepts(e.epoch) && !n ? I({
|
|
121
123
|
type: "stale-transcript-dropped",
|
|
122
124
|
epoch: e.epoch,
|
|
123
125
|
current: y.current()
|
|
124
|
-
}) : (
|
|
126
|
+
}) : (R(), a.onTranscript(e.text, e.final, e.language));
|
|
125
127
|
},
|
|
126
128
|
onUtteranceDone(e) {
|
|
127
|
-
|
|
129
|
+
G(e);
|
|
128
130
|
},
|
|
129
131
|
onError(e) {
|
|
130
|
-
|
|
132
|
+
L(p[e.code]);
|
|
131
133
|
}
|
|
132
|
-
})
|
|
134
|
+
}), x && a.subscribeCaptureCues && P === null && (P = a.subscribeCaptureCues((e) => {
|
|
135
|
+
x && (F = !0, a.transport.cueStarted(e));
|
|
136
|
+
}));
|
|
133
137
|
},
|
|
134
|
-
stop:
|
|
138
|
+
stop: Y,
|
|
135
139
|
get frames() {
|
|
136
140
|
return S;
|
|
137
141
|
},
|
|
138
|
-
feed:
|
|
142
|
+
feed: J,
|
|
139
143
|
epoch: y,
|
|
140
144
|
get running() {
|
|
141
145
|
return x;
|
|
@@ -33,8 +33,31 @@
|
|
|
33
33
|
* is 'avatar-unrouted': there is no reference, so the controller treats the
|
|
34
34
|
* avatar like the device voice while a reply is spoken, and tap-to-interrupt
|
|
35
35
|
* stays the fallback.
|
|
36
|
+
*
|
|
37
|
+
* ★ D3, the thinking tone (BOFF-12936). A short cue the assistant plays itself
|
|
38
|
+
* the moment a spoken turn is sent (`../thinkingTone.ts`). While a capture graph
|
|
39
|
+
* is attached it is played on that graph's context and routed like a tapped
|
|
40
|
+
* element (`playbackReferenceRoute`): to the speakers AND the reference bus. It
|
|
41
|
+
* registers while it sounds (`beginCuePlayback`), so the source reads 'cue', and
|
|
42
|
+
* the controller tells the gate to attribute nothing until the cue and its tail
|
|
43
|
+
* are over (`EchoGate.cueStarted`): the tone is the assistant, never the person.
|
|
44
|
+
* ★ Review round 2: a cue scheduled on the attached graph's context is also
|
|
45
|
+
* announced, at once and with its start on that context, to whoever subscribed
|
|
46
|
+
* (`subscribeCaptureCues`: the controller, for its transport). Waiting for the
|
|
47
|
+
* next frame to show it was too late when the main thread was busy right after
|
|
48
|
+
* the send.
|
|
49
|
+
*/
|
|
50
|
+
export type PlaybackSource = 'hosted' | 'cue' | 'avatar' | 'avatar-unrouted' | 'device' | null;
|
|
51
|
+
/**
|
|
52
|
+
* ★ D3 review round 2 (BOFF-12936): a cue as it is scheduled. `context` is the
|
|
53
|
+
* audio context it plays on, compared by identity with the attached capture
|
|
54
|
+
* graph's; `startsAt` is that context's time (`currentTime`, seconds) at its
|
|
55
|
+
* first sample.
|
|
36
56
|
*/
|
|
37
|
-
export
|
|
57
|
+
export interface ScheduledCue {
|
|
58
|
+
context: object;
|
|
59
|
+
startsAt: number;
|
|
60
|
+
}
|
|
38
61
|
/** The avatar controller's handle on its voice reference (P5). */
|
|
39
62
|
export interface AvatarVoiceRegistration {
|
|
40
63
|
/** The avatar's received audio track, once the room has it; null when it has none to give. */
|
|
@@ -56,10 +79,11 @@ export declare function isAvatarVoiceReferenceSupported(): boolean;
|
|
|
56
79
|
export declare function registerAvatarVoice(): AvatarVoiceRegistration;
|
|
57
80
|
export declare function attachPlaybackReference(context: AudioContext, referenceBus: AudioNode): void;
|
|
58
81
|
/**
|
|
59
|
-
* Stop referencing. `closeContext` is run now, or - if a tapped element
|
|
60
|
-
* still playing through the context - when the last one ends, so muting
|
|
61
|
-
* microphone never cuts a reply off mid-word
|
|
62
|
-
*
|
|
82
|
+
* Stop referencing. `closeContext` is run now, or - if a tapped element or a
|
|
83
|
+
* cue is still playing through the context - when the last one ends, so muting
|
|
84
|
+
* the microphone never cuts a reply off mid-word, nor the thinking tone mid-way
|
|
85
|
+
* (a cut is a click). An avatar's voice route belongs to the closing context, so
|
|
86
|
+
* it is dropped now; the next attach routes it again.
|
|
63
87
|
*/
|
|
64
88
|
export declare function detachPlaybackReference(closeContext: () => void): void;
|
|
65
89
|
/**
|
|
@@ -74,6 +98,37 @@ export declare function detachPlaybackReference(closeContext: () => void): void;
|
|
|
74
98
|
export declare function isPlaybackReferenceAttached(): boolean;
|
|
75
99
|
/** Route a freshly created playback element through the reference bus. */
|
|
76
100
|
export declare function tapPlaybackElement(element: HTMLMediaElement): void;
|
|
101
|
+
/**
|
|
102
|
+
* ★ D3 (BOFF-12936): where a cue the assistant plays itself (the thinking tone)
|
|
103
|
+
* goes while a capture graph is attached: that graph's context, with the
|
|
104
|
+
* reference bus to feed beside its speakers, exactly as a tapped element is
|
|
105
|
+
* routed. Null with nothing attached; the cue then plays on a context of its own.
|
|
106
|
+
*/
|
|
107
|
+
export declare function playbackReferenceRoute(): {
|
|
108
|
+
context: AudioContext;
|
|
109
|
+
referenceBus: AudioNode;
|
|
110
|
+
} | null;
|
|
111
|
+
/**
|
|
112
|
+
* ★ D3 (BOFF-12936): a cue (the thinking tone) starts sounding. Until the
|
|
113
|
+
* returned end is called the playback source reads 'cue' (`activePlaybackSource`)
|
|
114
|
+
* and a capture close waits for it (`detachPlaybackReference`). The end is
|
|
115
|
+
* idempotent; the caller calls it when the cue is silent, whatever stopped it.
|
|
116
|
+
*
|
|
117
|
+
* ★ Review round 2: `scheduled` says where and from when it sounds. When that
|
|
118
|
+
* is the attached capture graph's context, every `subscribeCaptureCues`
|
|
119
|
+
* listener is told its start before this returns. A listener that throws stops
|
|
120
|
+
* neither the cue nor the others.
|
|
121
|
+
*/
|
|
122
|
+
export declare function beginCuePlayback(scheduled?: ScheduledCue): () => void;
|
|
123
|
+
/**
|
|
124
|
+
* ★ D3 review round 2 (BOFF-12936): be told, synchronously, of every cue
|
|
125
|
+
* scheduled on the attached capture graph's context, with the time on that
|
|
126
|
+
* context it starts at. The hands-free controller subscribes for its transport:
|
|
127
|
+
* the audio-track branch must be shut before the first of the tone leaves its
|
|
128
|
+
* delay line, and the next capture frame may be handled too late for that.
|
|
129
|
+
* Returns the unsubscribe.
|
|
130
|
+
*/
|
|
131
|
+
export declare function subscribeCaptureCues(listener: (startsAt: number) => void): () => void;
|
|
77
132
|
/** Cut every tapped element's output at once (a barge-in). */
|
|
78
133
|
export declare function silencePlaybackReference(): void;
|
|
79
134
|
export declare function setDeviceVoice(speaking: boolean): void;
|
|
@@ -84,6 +139,9 @@ export declare function setDeviceVoice(speaking: boolean): void;
|
|
|
84
139
|
* reference level when it is audible, and from the session when an unrouted
|
|
85
140
|
* one may be speaking. The device voice wins over a connected avatar: it has no
|
|
86
141
|
* reference, so while it speaks nothing on the reference bus can explain it.
|
|
142
|
+
* A cue (D3) wins over a connected avatar too: while it sounds nothing is
|
|
143
|
+
* attributed at all, which is stricter than anything the avatar's reference
|
|
144
|
+
* would allow.
|
|
87
145
|
*/
|
|
88
146
|
export declare function activePlaybackSource(): PlaybackSource;
|
|
89
147
|
/** Test seam: forget everything. */
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// src/shared/assistant/voice/handsfree/playbackReference.ts
|
|
2
|
-
var e = null, t = null, n = [], r = !1, i = [];
|
|
3
|
-
function
|
|
2
|
+
var e = null, t = null, n = [], r = !1, i = [], a = /* @__PURE__ */ new Set(), o = /* @__PURE__ */ new Set();
|
|
3
|
+
function s(t) {
|
|
4
4
|
let n = e;
|
|
5
5
|
if (!(n === null || t.track === null || t.node !== null || t.released)) try {
|
|
6
6
|
let e = n.context.createMediaStreamSource(new MediaStream([t.track]));
|
|
@@ -9,13 +9,13 @@ function a(t) {
|
|
|
9
9
|
t.node = null;
|
|
10
10
|
}
|
|
11
11
|
}
|
|
12
|
-
function
|
|
12
|
+
function c(e) {
|
|
13
13
|
let t = e.node;
|
|
14
14
|
if (e.node = null, t !== null) try {
|
|
15
15
|
t.disconnect();
|
|
16
16
|
} catch {}
|
|
17
17
|
}
|
|
18
|
-
function
|
|
18
|
+
function l() {
|
|
19
19
|
try {
|
|
20
20
|
let e = globalThis.navigator, t = e?.userAgent;
|
|
21
21
|
return typeof t != "string" || /\b(iPhone|iPad|iPod)\b/.test(t) || /\bMacintosh\b/.test(t) && (e?.maxTouchPoints ?? 0) > 1 ? !0 : /AppleWebKit/.test(t) && !/(Chrome|Chromium|Edg|OPR|Firefox|Android)/.test(t);
|
|
@@ -23,14 +23,14 @@ function s() {
|
|
|
23
23
|
return !0;
|
|
24
24
|
}
|
|
25
25
|
}
|
|
26
|
-
function
|
|
26
|
+
function u() {
|
|
27
27
|
try {
|
|
28
|
-
return typeof MediaStream == "function" && typeof AudioContext == "function" && typeof AudioContext.prototype.createMediaStreamSource == "function" && !
|
|
28
|
+
return typeof MediaStream == "function" && typeof AudioContext == "function" && typeof AudioContext.prototype.createMediaStreamSource == "function" && !l();
|
|
29
29
|
} catch {
|
|
30
30
|
return !1;
|
|
31
31
|
}
|
|
32
32
|
}
|
|
33
|
-
function
|
|
33
|
+
function d() {
|
|
34
34
|
let e = {
|
|
35
35
|
track: null,
|
|
36
36
|
node: null,
|
|
@@ -38,33 +38,39 @@ function l() {
|
|
|
38
38
|
};
|
|
39
39
|
return i.push(e), {
|
|
40
40
|
setTrack(t) {
|
|
41
|
-
e.released || e.track === t || (
|
|
41
|
+
e.released || e.track === t || (c(e), e.track = t, s(e));
|
|
42
42
|
},
|
|
43
43
|
release() {
|
|
44
|
-
e.released || (e.released = !0,
|
|
44
|
+
e.released || (e.released = !0, c(e), i = i.filter((t) => t !== e));
|
|
45
45
|
}
|
|
46
46
|
};
|
|
47
47
|
}
|
|
48
|
-
function
|
|
48
|
+
function f(t, n) {
|
|
49
49
|
e = {
|
|
50
50
|
context: t,
|
|
51
51
|
referenceBus: n
|
|
52
52
|
};
|
|
53
|
-
for (let e of i)
|
|
53
|
+
for (let e of i) s(e);
|
|
54
54
|
}
|
|
55
|
-
function
|
|
56
|
-
|
|
57
|
-
e = null, n.some((e) => e.active) ? t = r : (n = [], r());
|
|
55
|
+
function p() {
|
|
56
|
+
return a.size > 0 || n.some((e) => e.active);
|
|
58
57
|
}
|
|
59
|
-
function
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
t = null, n = [], i();
|
|
58
|
+
function m(r) {
|
|
59
|
+
for (let e of i) c(e);
|
|
60
|
+
e = null, p() ? t = r : (n = [], r());
|
|
63
61
|
}
|
|
64
|
-
function
|
|
62
|
+
function h() {
|
|
63
|
+
if (!t || p()) return;
|
|
64
|
+
let e = t;
|
|
65
|
+
t = null, n = [], e();
|
|
66
|
+
}
|
|
67
|
+
function g(e, t) {
|
|
68
|
+
e.active = t, t || h();
|
|
69
|
+
}
|
|
70
|
+
function _() {
|
|
65
71
|
return e !== null;
|
|
66
72
|
}
|
|
67
|
-
function
|
|
73
|
+
function v(t) {
|
|
68
74
|
let r = e;
|
|
69
75
|
if (r) try {
|
|
70
76
|
let e = r.context.createMediaElementSource(t), i = r.context.createGain();
|
|
@@ -74,27 +80,48 @@ function m(t) {
|
|
|
74
80
|
gain: i,
|
|
75
81
|
active: !1
|
|
76
82
|
};
|
|
77
|
-
n.push(a), t.addEventListener("playing", () =>
|
|
83
|
+
n.push(a), t.addEventListener("playing", () => g(a, !0));
|
|
78
84
|
for (let e of [
|
|
79
85
|
"pause",
|
|
80
86
|
"ended",
|
|
81
87
|
"error",
|
|
82
88
|
"emptied"
|
|
83
|
-
]) t.addEventListener(e, () =>
|
|
89
|
+
]) t.addEventListener(e, () => g(a, !1));
|
|
84
90
|
} catch {}
|
|
85
91
|
}
|
|
86
|
-
function
|
|
92
|
+
function y() {
|
|
93
|
+
let t = e;
|
|
94
|
+
return t === null ? null : {
|
|
95
|
+
context: t.context,
|
|
96
|
+
referenceBus: t.referenceBus
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
function b(t) {
|
|
100
|
+
let n = {};
|
|
101
|
+
if (a.add(n), t !== void 0 && e !== null && t.context === e.context) for (let e of [...o]) try {
|
|
102
|
+
e(t.startsAt);
|
|
103
|
+
} catch {}
|
|
104
|
+
return () => {
|
|
105
|
+
a.delete(n), h();
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
function x(e) {
|
|
109
|
+
return o.add(e), () => {
|
|
110
|
+
o.delete(e);
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
function S() {
|
|
87
114
|
for (let e of n) try {
|
|
88
115
|
e.gain.gain.value = 0;
|
|
89
116
|
} catch {}
|
|
90
117
|
}
|
|
91
|
-
function
|
|
118
|
+
function C(e) {
|
|
92
119
|
r = e;
|
|
93
120
|
}
|
|
94
|
-
function
|
|
95
|
-
return n.some((e) => e.active) ? "hosted" : r ? "device" : i.length > 0 ? i.every((e) => e.node !== null) ? "avatar" : "avatar-unrouted" : null;
|
|
121
|
+
function w() {
|
|
122
|
+
return n.some((e) => e.active) ? "hosted" : r ? "device" : a.size > 0 ? "cue" : i.length > 0 ? i.every((e) => e.node !== null) ? "avatar" : "avatar-unrouted" : null;
|
|
96
123
|
}
|
|
97
|
-
function
|
|
98
|
-
e = null, t = null, n = [], r = !1, i = [];
|
|
124
|
+
function T() {
|
|
125
|
+
e = null, t = null, n = [], r = !1, i = [], a = /* @__PURE__ */ new Set(), o = /* @__PURE__ */ new Set();
|
|
99
126
|
}
|
|
100
|
-
export {
|
|
127
|
+
export { w as activePlaybackSource, f as attachPlaybackReference, b as beginCuePlayback, m as detachPlaybackReference, u as isAvatarVoiceReferenceSupported, _ as isPlaybackReferenceAttached, y as playbackReferenceRoute, d as registerAvatarVoice, T as resetPlaybackReferenceForTest, C as setDeviceVoice, S as silencePlaybackReference, x as subscribeCaptureCues, v as tapPlaybackElement };
|