@burdenoff/fe-libs 2026.1118.7 → 2026.1118.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/shared/assistant/index.d.ts +2 -0
- package/dist/shared/assistant/ui/useVoiceSession.js +188 -142
- package/dist/shared/assistant/voice/handsFree.d.ts +3 -1
- package/dist/shared/assistant/voice/handsfree/audioTrackTransport.d.ts +8 -0
- package/dist/shared/assistant/voice/handsfree/audioTrackTransport.js +36 -31
- package/dist/shared/assistant/voice/handsfree/backendStreamTransport.js +1 -0
- package/dist/shared/assistant/voice/handsfree/echoGate.d.ts +21 -0
- package/dist/shared/assistant/voice/handsfree/echoGate.js +11 -2
- package/dist/shared/assistant/voice/handsfree/handsFreeController.d.ts +7 -0
- package/dist/shared/assistant/voice/handsfree/handsFreeController.js +46 -42
- package/dist/shared/assistant/voice/handsfree/playbackReference.d.ts +63 -5
- package/dist/shared/assistant/voice/handsfree/playbackReference.js +56 -29
- package/dist/shared/assistant/voice/handsfree/preRollRing.d.ts +7 -0
- package/dist/shared/assistant/voice/handsfree/preRollRing.js +3 -0
- package/dist/shared/assistant/voice/handsfree/sttTransport.d.ts +16 -0
- package/dist/shared/assistant/voice/handsfree/useHandsFreeInterrupt.js +55 -54
- package/dist/shared/assistant/voice/handsfree/vad.d.ts +9 -0
- package/dist/shared/assistant/voice/handsfree/vad.js +10 -7
- package/dist/shared/assistant/voice/handsfree/webSpeechTransport.js +1 -0
- package/dist/shared/assistant/voice/thinkingTone.d.ts +213 -0
- package/dist/shared/assistant/voice/thinkingTone.js +182 -0
- package/dist/shared/assistant/voice/thinkingToneConfig.d.ts +4 -0
- package/dist/shared/assistant/voice/thinkingToneConfig.js +9 -0
- package/dist/shared/assistant/voice/voiceSession.d.ts +24 -0
- package/dist/shared/assistant/voice/voiceSession.js +18 -11
- package/dist/shared/backgrounds/useBackgroundPreference.d.ts +10 -4
- package/dist/shared/backgrounds/useBackgroundPreference.js +69 -47
- package/dist/shared-assistant.js +404 -402
- package/package.json +1 -1
|
@@ -28,4 +28,11 @@ export declare class PreRollRing {
|
|
|
28
28
|
get size(): number;
|
|
29
29
|
/** Zero and drop everything. */
|
|
30
30
|
clear(): void;
|
|
31
|
+
/**
|
|
32
|
+
* Drop everything WITHOUT zeroing it, while the session runs: the frames are
|
|
33
|
+
* shared, not copied, so a transport may still hold some of them (a pre-roll
|
|
34
|
+
* it has yet to send, an utterance it holds back). `push` lets go of a frame
|
|
35
|
+
* the same way. D3 (BOFF-12936): the pre-roll never holds the thinking tone.
|
|
36
|
+
*/
|
|
37
|
+
discard(): void;
|
|
31
38
|
}
|
|
@@ -29,5 +29,8 @@ var i = class {
|
|
|
29
29
|
for (let e of this.frames) e?.fill(0);
|
|
30
30
|
this.frames.fill(void 0), this.head = 0, this.count = 0;
|
|
31
31
|
}
|
|
32
|
+
discard() {
|
|
33
|
+
this.count !== 0 && (this.frames.fill(void 0), this.head = 0, this.count = 0);
|
|
34
|
+
}
|
|
32
35
|
};
|
|
33
36
|
export { e as DEFAULT_PREROLL_MS, n as MAX_PREROLL_MS, t as MIN_PREROLL_MS, i as PreRollRing, r as clampPreRollMs };
|
|
@@ -86,6 +86,22 @@ export interface SttTransport {
|
|
|
86
86
|
closeUtterance(utteranceId: string): void;
|
|
87
87
|
/** The utterance was superseded: stop it and stop being billed for it. */
|
|
88
88
|
cancelUtterance(utteranceId: string): void;
|
|
89
|
+
/**
|
|
90
|
+
* ★ D3 (BOFF-12936): the assistant's thinking tone has started. `at` is the
|
|
91
|
+
* capture graph's AudioContext time from which the microphone may hear it.
|
|
92
|
+
* Nothing the microphone hears from then on may reach this transport's
|
|
93
|
+
* recogniser, whether or not an utterance is open. What it heard before still
|
|
94
|
+
* does, and a final still owed for it is delivered as usual: the controller
|
|
95
|
+
* CLOSES an utterance open at the cue, it does not cancel it.
|
|
96
|
+
*
|
|
97
|
+
* ★ Review round 2: told as the tone is scheduled, with its exact start, when
|
|
98
|
+
* it plays on the capture graph's context; otherwise by the first frame that
|
|
99
|
+
* shows it, with that frame's start (no later than the tone's). A transport
|
|
100
|
+
* told too late to keep the tone from its recogniser delivers nothing more of
|
|
101
|
+
* the utterance the tone could have reached, rather than deliver the tone as
|
|
102
|
+
* the person.
|
|
103
|
+
*/
|
|
104
|
+
cueStarted(at: number): void;
|
|
89
105
|
stop(): void;
|
|
90
106
|
}
|
|
91
107
|
export interface TransportEnvironment {
|
|
@@ -1,88 +1,89 @@
|
|
|
1
1
|
import { createHandsFreeController as e } from "./handsFreeController.js";
|
|
2
2
|
import { chooseTransport as t, createRuntimeTransport as n, isMobileUserAgent as r, readTransportEnvironment as i } from "./handsFreeRuntime.js";
|
|
3
3
|
import { looksLikeHeadset as a, startMicCapture as o } from "./micCapture.js";
|
|
4
|
-
import { activePlaybackSource as s } from "./playbackReference.js";
|
|
5
|
-
import { HANDS_FREE_STREAM_NOTICE_VERSION as
|
|
6
|
-
import { useCallback as
|
|
4
|
+
import { activePlaybackSource as s, subscribeCaptureCues as c } from "./playbackReference.js";
|
|
5
|
+
import { HANDS_FREE_STREAM_NOTICE_VERSION as l, readHandsFreeStreaming as u, subscribeHandsFreeStreaming as d } from "./streamingConfig.js";
|
|
6
|
+
import { useCallback as f, useEffect as p, useMemo as m, useRef as h, useState as g, useSyncExternalStore as _ } from "react";
|
|
7
7
|
// src/shared/assistant/voice/handsfree/useHandsFreeInterrupt.ts
|
|
8
|
-
var
|
|
9
|
-
function
|
|
8
|
+
var v = 3e4;
|
|
9
|
+
function y(e, n) {
|
|
10
10
|
let a = i(e, n), o = t(a), s = o.kind === "web-speech" && r(a.userAgent);
|
|
11
11
|
return {
|
|
12
12
|
kind: o.kind,
|
|
13
13
|
supported: a.hasAudioWorklet && !s
|
|
14
14
|
};
|
|
15
15
|
}
|
|
16
|
-
var
|
|
17
|
-
function
|
|
18
|
-
let { enabled: r, phase: i, muted:
|
|
19
|
-
|
|
20
|
-
let [
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
}, [
|
|
24
|
-
(i === "idle" || i === "ended") && (
|
|
25
|
-
}, [i]),
|
|
16
|
+
var b = (e) => e === "listening" || e === "thinking" || e === "speaking";
|
|
17
|
+
function x(t) {
|
|
18
|
+
let { enabled: r, phase: i, muted: x, language: S, streamNoticeAccepted: C } = t, w = h(t);
|
|
19
|
+
w.current = t;
|
|
20
|
+
let [T, E] = g(null), [D, O] = g(!1), [k, A] = g(!1), [j, M] = g(!1), [N, P] = g(!1), [F, I] = g(!1), L = h(!1), [R, z] = g(!1), B = _(d, u, u), V = m(() => y(S, B), [S, B]), H = V.kind;
|
|
21
|
+
p(() => {
|
|
22
|
+
C !== "google-stt-eu-1" && z(!1);
|
|
23
|
+
}, [C]), p(() => {
|
|
24
|
+
(i === "idle" || i === "ended") && (E(null), O(!1));
|
|
25
|
+
}, [i]), p(() => {
|
|
26
26
|
if (!r || typeof document > "u") return;
|
|
27
27
|
let e = null, t = () => {
|
|
28
|
-
e !== null && clearTimeout(e), e = null, document.visibilityState === "hidden" ? e = setTimeout(() =>
|
|
28
|
+
e !== null && clearTimeout(e), e = null, document.visibilityState === "hidden" ? e = setTimeout(() => P(!0), v) : P(!1);
|
|
29
29
|
};
|
|
30
30
|
return document.addEventListener("visibilitychange", t), () => {
|
|
31
31
|
document.removeEventListener("visibilitychange", t), e !== null && clearTimeout(e);
|
|
32
32
|
};
|
|
33
33
|
}, [r]);
|
|
34
|
-
let
|
|
35
|
-
|
|
36
|
-
}, []),
|
|
37
|
-
|
|
38
|
-
}, []),
|
|
39
|
-
accept:
|
|
40
|
-
decline:
|
|
41
|
-
}), [
|
|
42
|
-
|
|
43
|
-
if (!
|
|
44
|
-
let t = n(
|
|
45
|
-
|
|
34
|
+
let U = r && V.supported && b(i) && !x && T === null && !N, W = H === "backend-stream" && !R && C !== "google-stt-eu-1", G = U && W, K = U && !W, q = f(() => {
|
|
35
|
+
z(!0), w.current.onAcceptStreamNotice?.(l);
|
|
36
|
+
}, []), J = f(() => {
|
|
37
|
+
E("notice-declined");
|
|
38
|
+
}, []), Y = m(() => ({
|
|
39
|
+
accept: q,
|
|
40
|
+
decline: J
|
|
41
|
+
}), [q, J]);
|
|
42
|
+
p(() => {
|
|
43
|
+
if (!K) return;
|
|
44
|
+
let t = n(H);
|
|
45
|
+
A(t.recognitionOwner === "transport" && t.wordsArrive === "after-utterance");
|
|
46
46
|
let r = e({
|
|
47
47
|
startCapture: o,
|
|
48
48
|
transport: t,
|
|
49
49
|
getSession: () => ({
|
|
50
|
-
phase:
|
|
51
|
-
muted:
|
|
50
|
+
phase: w.current.phase,
|
|
51
|
+
muted: w.current.muted
|
|
52
52
|
}),
|
|
53
53
|
getPlaybackSource: s,
|
|
54
|
-
|
|
55
|
-
|
|
54
|
+
subscribeCaptureCues: c,
|
|
55
|
+
getHeadsetHint: () => L.current,
|
|
56
|
+
language: () => w.current.language,
|
|
56
57
|
onBargeIn: () => {
|
|
57
|
-
|
|
58
|
+
O(!1), w.current.onBargeIn();
|
|
58
59
|
},
|
|
59
60
|
onTranscript: (e, t, n) => {
|
|
60
|
-
|
|
61
|
+
O(!1), w.current.onTranscript(e, t, n);
|
|
61
62
|
},
|
|
62
|
-
onSpeechFrame: (e) =>
|
|
63
|
-
onFallback:
|
|
64
|
-
onNothingHeard: () =>
|
|
65
|
-
onWordsPending:
|
|
66
|
-
isEnabled: () =>
|
|
63
|
+
onSpeechFrame: (e) => w.current.onSpeechFrame?.(e),
|
|
64
|
+
onFallback: E,
|
|
65
|
+
onNothingHeard: () => O(!0),
|
|
66
|
+
onWordsPending: M,
|
|
67
|
+
isEnabled: () => w.current.enabled
|
|
67
68
|
}), i = !1;
|
|
68
69
|
return r.start().then(() => {
|
|
69
|
-
i ||
|
|
70
|
+
i || I(!0);
|
|
70
71
|
}), navigator.mediaDevices?.enumerateDevices().then((e) => {
|
|
71
|
-
|
|
72
|
+
L.current = a(e.filter((e) => e.kind === "audiooutput").map((e) => e.label));
|
|
72
73
|
}).catch(() => void 0), () => {
|
|
73
|
-
i = !0, r.stop(),
|
|
74
|
+
i = !0, r.stop(), I(!1), A(!1), M(!1);
|
|
74
75
|
};
|
|
75
|
-
}, [
|
|
76
|
-
let
|
|
77
|
-
return r && !
|
|
78
|
-
mode:
|
|
79
|
-
transportKind:
|
|
80
|
-
suppressWebSpeech: r &&
|
|
81
|
-
fallbackReason:
|
|
82
|
-
nothingHeard:
|
|
83
|
-
privacyNotice:
|
|
84
|
-
wordsAfterUtterance:
|
|
85
|
-
wordsPending:
|
|
76
|
+
}, [K, H]);
|
|
77
|
+
let X = "off";
|
|
78
|
+
return r && !V.supported ? X = "unavailable" : T === null ? G ? X = "notice" : K && (X = F ? "active" : "starting") : X = "fallback", {
|
|
79
|
+
mode: X,
|
|
80
|
+
transportKind: V.supported ? H : null,
|
|
81
|
+
suppressWebSpeech: r && V.supported && H !== "web-speech" && T === null && b(i),
|
|
82
|
+
fallbackReason: T,
|
|
83
|
+
nothingHeard: D,
|
|
84
|
+
privacyNotice: X === "notice" ? Y : null,
|
|
85
|
+
wordsAfterUtterance: k,
|
|
86
|
+
wordsPending: j
|
|
86
87
|
};
|
|
87
88
|
}
|
|
88
|
-
export {
|
|
89
|
+
export { v as HIDDEN_RELEASE_MS, x as useHandsFreeInterrupt };
|
|
@@ -48,6 +48,15 @@ export interface Vad {
|
|
|
48
48
|
/** `gatePass`: the echo gate accepts this frame as someone other than the assistant. */
|
|
49
49
|
push(frame: VadFrame, gatePass: boolean): VadStep;
|
|
50
50
|
reset(): void;
|
|
51
|
+
/**
|
|
52
|
+
* End the segment in progress now, with no `end` event. The noise floor stays
|
|
53
|
+
* the room's, as the segment froze it, until the segment's own frames have
|
|
54
|
+
* left the floor's history (`reset` forgets the floor). The next onset is
|
|
55
|
+
* confirmed afresh, from frames the echo gate passes. D3 (BOFF-12936): the
|
|
56
|
+
* thinking tone starts, and a segment that began before it must not carry on
|
|
57
|
+
* through it.
|
|
58
|
+
*/
|
|
59
|
+
endSegment(): void;
|
|
51
60
|
/** Change the number of frames that confirm an onset (strict mode). */
|
|
52
61
|
setOnsetFrames(frames: number): void;
|
|
53
62
|
}
|
|
@@ -17,20 +17,23 @@ function r(t = {}) {
|
|
|
17
17
|
let r = {
|
|
18
18
|
...e,
|
|
19
19
|
...t
|
|
20
|
-
}, i = r.onsetFrames, a = [], o = -90, s = 0, c = 0, l = !1;
|
|
20
|
+
}, i = r.onsetFrames, a = [], o = -90, s = 0, c = 0, l = !1, u = 0;
|
|
21
21
|
return {
|
|
22
22
|
push(e, t) {
|
|
23
|
-
a.push(e.rmsDb), a.length > r.floorWindowFrames && a.shift(), l || (o = Math.max(-90, n(a, r.floorPercentile)));
|
|
24
|
-
let
|
|
25
|
-
return l ? (c =
|
|
26
|
-
voiced:
|
|
27
|
-
event:
|
|
23
|
+
a.push(e.rmsDb), a.length > r.floorWindowFrames && a.shift(), u > 0 ? --u : l || (o = Math.max(-90, n(a, r.floorPercentile)));
|
|
24
|
+
let d = Math.max(o + r.voicedAboveFloorDb, r.absFloorDb), f = e.rmsDb > d && e.zcr > r.zcrMin && e.zcr < r.zcrMax, p = null;
|
|
25
|
+
return l ? (c = f ? 0 : c + r.frameMs, c >= r.hangoverMs && (l = !1, s = 0, p = "end")) : (s = f && t ? s + 1 : 0, s >= i && (l = !0, c = 0, p = "onset")), {
|
|
26
|
+
voiced: f,
|
|
27
|
+
event: p,
|
|
28
28
|
inSegment: l,
|
|
29
29
|
floorDb: o
|
|
30
30
|
};
|
|
31
31
|
},
|
|
32
32
|
reset() {
|
|
33
|
-
a = [], o = -90, s = 0, c = 0, l = !1;
|
|
33
|
+
a = [], o = -90, s = 0, c = 0, l = !1, u = 0;
|
|
34
|
+
},
|
|
35
|
+
endSegment() {
|
|
36
|
+
l && (u = r.floorWindowFrames), s = 0, c = 0, l = !1;
|
|
34
37
|
},
|
|
35
38
|
setOnsetFrames(e) {
|
|
36
39
|
i = Math.max(1, Math.round(e));
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
import { ScheduledCue } from './handsfree/playbackReference';
|
|
2
|
+
import { VoiceSessionPhase } from './voiceSession';
|
|
3
|
+
/**
|
|
4
|
+
* Does this commit of the voice session play the tone?
|
|
5
|
+
*
|
|
6
|
+
* The send of a spoken turn is the session leaving `listening` for `thinking`:
|
|
7
|
+
* the reducer's only way there is the `silence` that submits (the grace timer,
|
|
8
|
+
* or "I'm done"). The other way into `thinking`, a reply pausing between its
|
|
9
|
+
* phrases (`reply-audio-paused`), comes from `speaking`; a typed turn never moves
|
|
10
|
+
* the voice session at all. So `from`/`to` alone keep both out.
|
|
11
|
+
*
|
|
12
|
+
* Never while muted (the person has turned their microphone off), and never
|
|
13
|
+
* while a live avatar is ON: the avatar is the voice output then, with its own
|
|
14
|
+
* audio on its own track, and a tone from the page beside it would be neither
|
|
15
|
+
* its voice nor something its echo reference accounts for.
|
|
16
|
+
*/
|
|
17
|
+
export declare function shouldPlayThinkingTone(input: {
|
|
18
|
+
/** The app's switch (`thinkingToneEnabled`). */
|
|
19
|
+
enabled: boolean;
|
|
20
|
+
/** The phase of the session's previous commit. */
|
|
21
|
+
from: VoiceSessionPhase;
|
|
22
|
+
/** The phase now. */
|
|
23
|
+
to: VoiceSessionPhase;
|
|
24
|
+
muted: boolean;
|
|
25
|
+
/** A live avatar is ON (`useLiveAvatarOn`). */
|
|
26
|
+
avatarOn: boolean;
|
|
27
|
+
}): boolean;
|
|
28
|
+
/**
|
|
29
|
+
* How long after the tone has ended a recogniser's result of it may still
|
|
30
|
+
* arrive: the same 1.5 s H2 allows the turn just sent (a final arrives late).
|
|
31
|
+
*/
|
|
32
|
+
export declare const THINKING_TONE_RESULT_TAIL_MS = 1500;
|
|
33
|
+
/**
|
|
34
|
+
* ★ D3 review round 2 (BOFF-12936): how long a send's tone may wait for the
|
|
35
|
+
* phrase the browser's recogniser still has open, before that turn goes without
|
|
36
|
+
* its tone: as long as H2 allows a final of the turn just sent to take (1.5 s).
|
|
37
|
+
*/
|
|
38
|
+
export declare const THINKING_TONE_PHRASE_WAIT_MS = 1500;
|
|
39
|
+
/** What to do now about the tone of a send (`planThinkingTone`). */
|
|
40
|
+
export type ThinkingToneStep = 'play' | 'wait' | 'skip';
|
|
41
|
+
/**
|
|
42
|
+
* ★ D3 review round 2 (BOFF-12936): a send's tone, from the moment of the send
|
|
43
|
+
* (`shouldPlayThinkingTone`) until it plays or is dropped.
|
|
44
|
+
*
|
|
45
|
+
* The browser's own recogniser (tap-first, and hands-free on Web Speech) hears
|
|
46
|
+
* the room, the tone with it, and nothing vets its audio. A phrase it still has
|
|
47
|
+
* open at the send (a live guess, not yet final) is the one that carries the
|
|
48
|
+
* person's last words: "I'm done" is often tapped while the guess still lags
|
|
49
|
+
* them. A tone sounding then is written into that phrase's final, and no rule on
|
|
50
|
+
* the words can take it out again: "remind me to call mom at five" and "remind
|
|
51
|
+
* me to call mom ding dong" are alike to any count, and in Chinese or Japanese,
|
|
52
|
+
* written without spaces, there are no words to count.
|
|
53
|
+
*
|
|
54
|
+
* So the tone waits for that phrase to be final (`phraseOpen` false) and plays
|
|
55
|
+
* then, while the turn is still in flight: the final has been judged by then,
|
|
56
|
+
* exactly as with no tone, and the tone's window covers only what starts after
|
|
57
|
+
* it. It never plays into the phrase, nor late: from `until` on the turn goes
|
|
58
|
+
* without its tone, whenever the phrase ends. A turn superseded, answered or
|
|
59
|
+
* cut off meanwhile, a mute, a live avatar or the switch turned off drop it too.
|
|
60
|
+
* With no phrase open (always, on a transport that owns recognition: its own
|
|
61
|
+
* guards keep the tone out) it plays at once.
|
|
62
|
+
*/
|
|
63
|
+
export declare function planThinkingTone(input: {
|
|
64
|
+
/** The app's switch (`thinkingToneEnabled`). */
|
|
65
|
+
enabled: boolean;
|
|
66
|
+
phase: VoiceSessionPhase;
|
|
67
|
+
muted: boolean;
|
|
68
|
+
/** A live avatar is ON (`useLiveAvatarOn`). */
|
|
69
|
+
avatarOn: boolean;
|
|
70
|
+
/** The browser's recogniser has a phrase open: a live guess not yet final. */
|
|
71
|
+
phraseOpen: boolean;
|
|
72
|
+
/** Wall-clock ms. */
|
|
73
|
+
now: number;
|
|
74
|
+
/** Until when this send's tone may wait (wall-clock ms): the send plus `THINKING_TONE_PHRASE_WAIT_MS`. */
|
|
75
|
+
until: number;
|
|
76
|
+
}): ThinkingToneStep;
|
|
77
|
+
/**
|
|
78
|
+
* Until when (wall-clock ms) a result the tap-first recogniser reports may be
|
|
79
|
+
* the tone, for a tone that started at `startedAt` and is heard for
|
|
80
|
+
* `soundsForMs` (`ThinkingTonePlayer.play`: until it has left the speakers).
|
|
81
|
+
*/
|
|
82
|
+
export declare function thinkingToneHeardUntil(startedAt: number, soundsForMs: number): number;
|
|
83
|
+
export interface ThinkingTonePartial {
|
|
84
|
+
readonly hz: number;
|
|
85
|
+
/** Relative level; the partials are scaled so that together they peak at `peakGain`. */
|
|
86
|
+
readonly level: number;
|
|
87
|
+
}
|
|
88
|
+
export interface ThinkingToneSpec {
|
|
89
|
+
/** From the first sample to exact silence. */
|
|
90
|
+
readonly durationMs: number;
|
|
91
|
+
/** The linear rise from silence to the peak: long enough that the start is not a click. */
|
|
92
|
+
readonly attackMs: number;
|
|
93
|
+
/** The final linear fade to exact zero: long enough that the end is not a click either. */
|
|
94
|
+
readonly releaseMs: number;
|
|
95
|
+
/** Peak gain of the partials together (0..1 of full scale). */
|
|
96
|
+
readonly peakGain: number;
|
|
97
|
+
/** What share of the peak the bell-like decay has fallen to when the final fade starts. */
|
|
98
|
+
readonly decayTo: number;
|
|
99
|
+
readonly partials: readonly ThinkingTonePartial[];
|
|
100
|
+
}
|
|
101
|
+
/** A soft "ding": E5 with a quieter B5 above it, a fifth apart; peak about -26 dBFS. */
|
|
102
|
+
export declare const THINKING_TONE: ThinkingToneSpec;
|
|
103
|
+
/** One automation event, `at` seconds after the tone starts, as Web Audio applies it. */
|
|
104
|
+
export interface ThinkingToneAutomationEvent {
|
|
105
|
+
readonly at: number;
|
|
106
|
+
readonly value: number;
|
|
107
|
+
/** `set` holds from `at`; a ramp runs from the event before it to `at`. */
|
|
108
|
+
readonly ramp: 'set' | 'linear' | 'exponential';
|
|
109
|
+
}
|
|
110
|
+
/** The tone's envelope: silence, a linear attack, a bell-like decay, a linear fade to exact zero. */
|
|
111
|
+
export declare function thinkingToneEnvelope(spec?: ThinkingToneSpec): ThinkingToneAutomationEvent[];
|
|
112
|
+
/**
|
|
113
|
+
* The value automation `events` give at time `t`, as Web Audio computes it: a
|
|
114
|
+
* `set` holds from its time, a ramp runs from the event before it to its own
|
|
115
|
+
* time, and before the first event the param holds `initial`. An exponential
|
|
116
|
+
* ramp from or to zero holds its start value, as the specification says.
|
|
117
|
+
*/
|
|
118
|
+
export declare function thinkingToneAutomationAt(events: readonly ThinkingToneAutomationEvent[], t: number, initial?: number): number;
|
|
119
|
+
/** The slice of an `AudioParam` the tone automates. */
|
|
120
|
+
export interface ThinkingToneParam {
|
|
121
|
+
value: number;
|
|
122
|
+
setValueAtTime(value: number, startTime: number): unknown;
|
|
123
|
+
linearRampToValueAtTime(value: number, endTime: number): unknown;
|
|
124
|
+
exponentialRampToValueAtTime(value: number, endTime: number): unknown;
|
|
125
|
+
cancelScheduledValues(cancelTime: number): unknown;
|
|
126
|
+
}
|
|
127
|
+
/** The slice of an `AudioNode`. */
|
|
128
|
+
export interface ThinkingToneNode {
|
|
129
|
+
connect(destination: ThinkingToneNode): unknown;
|
|
130
|
+
disconnect(): void;
|
|
131
|
+
}
|
|
132
|
+
export interface ThinkingToneGainNode extends ThinkingToneNode {
|
|
133
|
+
readonly gain: ThinkingToneParam;
|
|
134
|
+
}
|
|
135
|
+
export interface ThinkingToneOscillatorNode extends ThinkingToneNode {
|
|
136
|
+
type: OscillatorType;
|
|
137
|
+
readonly frequency: ThinkingToneParam;
|
|
138
|
+
start(when?: number): void;
|
|
139
|
+
stop(when?: number): void;
|
|
140
|
+
}
|
|
141
|
+
/** The slice of an `AudioContext` the tone uses. */
|
|
142
|
+
export interface ThinkingToneContext {
|
|
143
|
+
readonly currentTime: number;
|
|
144
|
+
readonly state: string;
|
|
145
|
+
/** Seconds from the context's output to the audio system (absent where the browser lacks it). */
|
|
146
|
+
readonly baseLatency?: number;
|
|
147
|
+
/** Seconds from the audio system to the speakers: 150-300 ms over Bluetooth (absent where unknown). */
|
|
148
|
+
readonly outputLatency?: number;
|
|
149
|
+
readonly destination: ThinkingToneNode;
|
|
150
|
+
createGain(): ThinkingToneGainNode;
|
|
151
|
+
createOscillator(): ThinkingToneOscillatorNode;
|
|
152
|
+
resume(): Promise<void>;
|
|
153
|
+
close(): Promise<void>;
|
|
154
|
+
}
|
|
155
|
+
/** Stopping early: from wherever the envelope is to zero, linearly (never a cut). */
|
|
156
|
+
export declare const THINKING_TONE_STOP_FADE_MS = 15;
|
|
157
|
+
/** A tone scheduled on a context. */
|
|
158
|
+
export interface ScheduledThinkingTone {
|
|
159
|
+
/** Its context's time (`currentTime`, seconds) at its first sample: the envelope's start. */
|
|
160
|
+
readonly startsAt: number;
|
|
161
|
+
/** Milliseconds until it is silent: its own end, or the end of the fade after `fadeOut`. */
|
|
162
|
+
remainingMs(): number;
|
|
163
|
+
/**
|
|
164
|
+
* Fade from wherever the envelope is to zero over `THINKING_TONE_STOP_FADE_MS`,
|
|
165
|
+
* then stop. Idempotent; nothing at all once it is silent or its context closed.
|
|
166
|
+
*/
|
|
167
|
+
fadeOut(): void;
|
|
168
|
+
/** Disconnect every node (once it is silent). Idempotent. */
|
|
169
|
+
release(): void;
|
|
170
|
+
}
|
|
171
|
+
/** Schedule the tone on `context`, into every one of `destinations`. */
|
|
172
|
+
export declare function scheduleThinkingTone(context: ThinkingToneContext, destinations: readonly ThinkingToneNode[], spec?: ThinkingToneSpec): ScheduledThinkingTone;
|
|
173
|
+
/**
|
|
174
|
+
* How long a sample the context renders takes to reach the speakers, in ms: its
|
|
175
|
+
* own latency and the output device's. 0 where the browser does not say.
|
|
176
|
+
*/
|
|
177
|
+
export declare function thinkingToneOutputDelayMs(context: ThinkingToneContext): number;
|
|
178
|
+
export interface ThinkingTonePlayer {
|
|
179
|
+
/**
|
|
180
|
+
* Inside the tap that starts a voice session: make the player's own context
|
|
181
|
+
* ready, since the tone itself plays from a timer, not a gesture.
|
|
182
|
+
*/
|
|
183
|
+
prime(): void;
|
|
184
|
+
/**
|
|
185
|
+
* Play the tone now: how long it is heard (ms), until its last sample has left
|
|
186
|
+
* the speakers; or null when nothing could play it.
|
|
187
|
+
*/
|
|
188
|
+
play(): number | null;
|
|
189
|
+
/** Fade a tone still sounding to silence (never a click). */
|
|
190
|
+
stop(): void;
|
|
191
|
+
/** The session is over: stop, and close the player's own context once it is silent. */
|
|
192
|
+
dispose(): void;
|
|
193
|
+
}
|
|
194
|
+
export interface ThinkingTonePlayerDeps {
|
|
195
|
+
/** A context of the player's own (default: a new `AudioContext`, or null without Web Audio). */
|
|
196
|
+
createContext?: () => ThinkingToneContext | null;
|
|
197
|
+
/**
|
|
198
|
+
* The capture graph's context and reference bus while hands-free listens
|
|
199
|
+
* (default: `playbackReferenceRoute`).
|
|
200
|
+
*/
|
|
201
|
+
captureRoute?: () => {
|
|
202
|
+
context: ThinkingToneContext;
|
|
203
|
+
referenceBus: ThinkingToneNode;
|
|
204
|
+
} | null;
|
|
205
|
+
/**
|
|
206
|
+
* Register the tone as the assistant's playback, with the context it plays on
|
|
207
|
+
* and when on that context it starts; returns its end (default: `beginCuePlayback`).
|
|
208
|
+
*/
|
|
209
|
+
beginCue?: (scheduled: ScheduledCue) => () => void;
|
|
210
|
+
/** Run `fn` after `ms`; returns its cancel (default: `setTimeout`). */
|
|
211
|
+
setTimer?: (fn: () => void, ms: number) => () => void;
|
|
212
|
+
}
|
|
213
|
+
export declare function createThinkingTonePlayer(deps?: ThinkingTonePlayerDeps): ThinkingTonePlayer;
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
import { TRAILING_RESULT_WINDOW_MS as e } from "./handsFree.js";
|
|
2
|
+
import { beginCuePlayback as t, playbackReferenceRoute as n } from "./handsfree/playbackReference.js";
|
|
3
|
+
// src/shared/assistant/voice/thinkingTone.ts
|
|
4
|
+
function r(e) {
|
|
5
|
+
return e.enabled && e.from === "listening" && e.to === "thinking" && !e.muted && !e.avatarOn;
|
|
6
|
+
}
|
|
7
|
+
var i = e, a = e;
|
|
8
|
+
function o(e) {
|
|
9
|
+
return !e.enabled || e.phase !== "thinking" || e.muted || e.avatarOn || e.now >= e.until ? "skip" : e.phraseOpen ? "wait" : "play";
|
|
10
|
+
}
|
|
11
|
+
function s(e, t) {
|
|
12
|
+
return e + Math.max(0, t) + i;
|
|
13
|
+
}
|
|
14
|
+
var c = Object.freeze({
|
|
15
|
+
durationMs: 150,
|
|
16
|
+
attackMs: 10,
|
|
17
|
+
releaseMs: 30,
|
|
18
|
+
peakGain: .05,
|
|
19
|
+
decayTo: .1,
|
|
20
|
+
partials: Object.freeze([Object.freeze({
|
|
21
|
+
hz: 660,
|
|
22
|
+
level: 1
|
|
23
|
+
}), Object.freeze({
|
|
24
|
+
hz: 990,
|
|
25
|
+
level: .35
|
|
26
|
+
})])
|
|
27
|
+
});
|
|
28
|
+
function l(e = c) {
|
|
29
|
+
let t = e.durationMs / 1e3;
|
|
30
|
+
return [
|
|
31
|
+
{
|
|
32
|
+
at: 0,
|
|
33
|
+
value: 0,
|
|
34
|
+
ramp: "set"
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
at: e.attackMs / 1e3,
|
|
38
|
+
value: e.peakGain,
|
|
39
|
+
ramp: "linear"
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
at: t - e.releaseMs / 1e3,
|
|
43
|
+
value: e.peakGain * e.decayTo,
|
|
44
|
+
ramp: "exponential"
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
at: t,
|
|
48
|
+
value: 0,
|
|
49
|
+
ramp: "linear"
|
|
50
|
+
}
|
|
51
|
+
];
|
|
52
|
+
}
|
|
53
|
+
function u(e, t, n = 0) {
|
|
54
|
+
let r = null;
|
|
55
|
+
for (let i of e) {
|
|
56
|
+
if (t < i.at) {
|
|
57
|
+
if (r === null) return n;
|
|
58
|
+
if (i.ramp === "set") return r.value;
|
|
59
|
+
let e = (t - r.at) / (i.at - r.at);
|
|
60
|
+
return i.ramp === "linear" ? r.value + (i.value - r.value) * e : r.value === 0 || i.value === 0 || r.value * i.value < 0 ? r.value : r.value * (i.value / r.value) ** e;
|
|
61
|
+
}
|
|
62
|
+
r = i;
|
|
63
|
+
}
|
|
64
|
+
return r === null ? n : r.value;
|
|
65
|
+
}
|
|
66
|
+
var d = .01, f = .02, p = 15;
|
|
67
|
+
function m(e, t, n = c) {
|
|
68
|
+
let r = e.currentTime + d, i = r + n.durationMs / 1e3, a = l(n), o = e.createGain();
|
|
69
|
+
o.gain.value = 0;
|
|
70
|
+
for (let e of a) e.ramp === "set" ? o.gain.setValueAtTime(e.value, r + e.at) : e.ramp === "linear" ? o.gain.linearRampToValueAtTime(e.value, r + e.at) : o.gain.exponentialRampToValueAtTime(e.value, r + e.at);
|
|
71
|
+
for (let e of t) o.connect(e);
|
|
72
|
+
let s = n.partials.reduce((e, t) => e + t.level, 0), p = n.partials.map((t) => {
|
|
73
|
+
let n = e.createOscillator();
|
|
74
|
+
n.type = "sine", n.frequency.value = t.hz;
|
|
75
|
+
let r = e.createGain();
|
|
76
|
+
return r.gain.value = t.level / s, n.connect(r), r.connect(o), {
|
|
77
|
+
oscillator: n,
|
|
78
|
+
level: r
|
|
79
|
+
};
|
|
80
|
+
});
|
|
81
|
+
for (let { oscillator: e } of p) e.start(r), e.stop(i + f);
|
|
82
|
+
let m = i, h = !1, g = !1;
|
|
83
|
+
return {
|
|
84
|
+
startsAt: r,
|
|
85
|
+
remainingMs: () => Math.max(0, (m - e.currentTime) * 1e3),
|
|
86
|
+
fadeOut() {
|
|
87
|
+
if (h || g) return;
|
|
88
|
+
h = !0;
|
|
89
|
+
let t = e.currentTime;
|
|
90
|
+
if (e.state === "closed" || t >= m) return;
|
|
91
|
+
let n = t + 15 / 1e3;
|
|
92
|
+
try {
|
|
93
|
+
o.gain.cancelScheduledValues(t), o.gain.setValueAtTime(u(a, t - r), t), o.gain.linearRampToValueAtTime(0, n);
|
|
94
|
+
for (let { oscillator: e } of p) e.stop(n + f);
|
|
95
|
+
m = n;
|
|
96
|
+
} catch {
|
|
97
|
+
m = t;
|
|
98
|
+
}
|
|
99
|
+
},
|
|
100
|
+
release() {
|
|
101
|
+
if (!g) {
|
|
102
|
+
g = !0;
|
|
103
|
+
for (let e of [...p.flatMap(({ oscillator: e, level: t }) => [e, t]), o]) try {
|
|
104
|
+
e.disconnect();
|
|
105
|
+
} catch {}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
var h = 50;
|
|
111
|
+
function g(e) {
|
|
112
|
+
let t = (e.baseLatency ?? 0) + (e.outputLatency ?? 0);
|
|
113
|
+
return Number.isFinite(t) && t > 0 ? t * 1e3 : 0;
|
|
114
|
+
}
|
|
115
|
+
function _() {
|
|
116
|
+
if (typeof AudioContext != "function") return null;
|
|
117
|
+
try {
|
|
118
|
+
return new AudioContext({ latencyHint: "interactive" });
|
|
119
|
+
} catch {
|
|
120
|
+
return null;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
function v(e, t) {
|
|
124
|
+
let n = setTimeout(e, t);
|
|
125
|
+
return () => clearTimeout(n);
|
|
126
|
+
}
|
|
127
|
+
function y(e = {}) {
|
|
128
|
+
let r = e.createContext ?? _, i = e.captureRoute ?? n, a = e.beginCue ?? t, o = e.setTimer ?? v, s = null, c = null, l = (e) => {
|
|
129
|
+
e.cancelTimer(), e.tone.release(), e.endCue(), c === e && (c = null);
|
|
130
|
+
}, u = (e) => e.tone.remainingMs() + g(e.context), d = (e) => {
|
|
131
|
+
e.cancelTimer(), e.cancelTimer = o(() => l(e), u(e) + h);
|
|
132
|
+
}, f = () => {
|
|
133
|
+
let e = c;
|
|
134
|
+
e !== null && (e.tone.fadeOut(), d(e));
|
|
135
|
+
}, p = () => {
|
|
136
|
+
let e = i();
|
|
137
|
+
return e !== null && e.context.state === "running" ? {
|
|
138
|
+
context: e.context,
|
|
139
|
+
destinations: [e.context.destination, e.referenceBus]
|
|
140
|
+
} : s !== null && s.state === "running" ? {
|
|
141
|
+
context: s,
|
|
142
|
+
destinations: [s.destination]
|
|
143
|
+
} : null;
|
|
144
|
+
};
|
|
145
|
+
return {
|
|
146
|
+
prime() {
|
|
147
|
+
s === null && (s = r()), s !== null && s.state !== "running" && s.resume().catch(() => void 0);
|
|
148
|
+
},
|
|
149
|
+
play() {
|
|
150
|
+
f();
|
|
151
|
+
let e = p();
|
|
152
|
+
if (e === null) return null;
|
|
153
|
+
let t;
|
|
154
|
+
try {
|
|
155
|
+
t = m(e.context, e.destinations);
|
|
156
|
+
} catch {
|
|
157
|
+
return null;
|
|
158
|
+
}
|
|
159
|
+
let n = {
|
|
160
|
+
tone: t,
|
|
161
|
+
context: e.context,
|
|
162
|
+
endCue: a({
|
|
163
|
+
context: e.context,
|
|
164
|
+
startsAt: t.startsAt
|
|
165
|
+
}),
|
|
166
|
+
cancelTimer: () => void 0
|
|
167
|
+
};
|
|
168
|
+
return c = n, d(n), u(n);
|
|
169
|
+
},
|
|
170
|
+
stop: f,
|
|
171
|
+
dispose() {
|
|
172
|
+
f();
|
|
173
|
+
let e = s;
|
|
174
|
+
if (s = null, e === null) return;
|
|
175
|
+
let t = () => {
|
|
176
|
+
e.close().catch(() => void 0);
|
|
177
|
+
};
|
|
178
|
+
c?.context === e ? o(t, u(c) + h) : t();
|
|
179
|
+
}
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
export { c as THINKING_TONE, a as THINKING_TONE_PHRASE_WAIT_MS, i as THINKING_TONE_RESULT_TAIL_MS, p as THINKING_TONE_STOP_FADE_MS, y as createThinkingTonePlayer, o as planThinkingTone, m as scheduleThinkingTone, r as shouldPlayThinkingTone, u as thinkingToneAutomationAt, l as thinkingToneEnvelope, s as thinkingToneHeardUntil, g as thinkingToneOutputDelayMs };
|