@burdenoff/fe-libs 2026.1118.6 → 2026.1118.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/shared/assistant/avatar/live/createLiveAvatarSeam.js +8 -6
- package/dist/shared/assistant/speech/hostedTtsSpeechEngine.js +37 -27
- package/dist/shared/assistant/speech/index.d.ts +1 -1
- package/dist/shared/assistant/speech/notifyCancellationSpeechEngine.d.ts +14 -1
- package/dist/shared/assistant/speech/notifyCancellationSpeechEngine.js +3 -0
- package/dist/shared/assistant/speech/routingSpeechEngine.js +4 -4
- package/dist/shared/assistant/speech/speechFeed.d.ts +20 -3
- package/dist/shared/assistant/speech/speechFeed.js +25 -17
- package/dist/shared/assistant/speech/types.d.ts +68 -1
- package/dist/shared/assistant/speech/types.js +7 -2
- package/dist/shared/assistant/turnTimeline.d.ts +32 -0
- package/dist/shared/assistant/turnTimeline.js +3 -3
- package/dist/shared/assistant/ui/AssistantChatArea.js +1228 -1227
- package/dist/shared/assistant/ui/AssistantSpeechProvider.d.ts +5 -3
- package/dist/shared/assistant/ui/AssistantSpeechProvider.js +393 -346
- package/dist/shared/assistant/voice/firstClipPrefetchConfig.d.ts +19 -0
- package/dist/shared/assistant/voice/firstClipPrefetchConfig.js +8 -1
- package/dist/shared/assistant/voice/livePhraseSpeech.d.ts +23 -0
- package/dist/shared/assistant/voice/livePhraseSpeech.js +30 -30
- package/dist/shared/assistant/voice/phraseGate.d.ts +13 -0
- package/dist/shared/assistant/voice/phraseGate.js +6 -2
- package/dist/shared/assistant/voice/phrasePlayback.d.ts +59 -0
- package/dist/shared/assistant/voice/phrasePlayback.js +56 -34
- package/dist/shared/backgrounds/useBackgroundPreference.d.ts +10 -4
- package/dist/shared/backgrounds/useBackgroundPreference.js +69 -47
- package/dist/shared-assistant.js +418 -418
- package/package.json +1 -1
|
@@ -251,14 +251,16 @@ function W(e = {}) {
|
|
|
251
251
|
listVoices: () => e.listVoices(),
|
|
252
252
|
subscribeVoices: (t) => e.subscribeVoices(t),
|
|
253
253
|
wouldSpeakOnAvatar: (e) => t(e).target === "avatar",
|
|
254
|
-
prefetch: (n, r) => {
|
|
255
|
-
let {
|
|
256
|
-
|
|
254
|
+
...e.prefetch ? { prefetch: (n, r) => {
|
|
255
|
+
let { phase: i } = j.getSnapshot();
|
|
256
|
+
if (i === "starting" || i === "connecting") return;
|
|
257
|
+
let { voice: a, target: o } = t(r);
|
|
258
|
+
e.prefetch?.(n, o === "avatar" && a !== null ? {
|
|
257
259
|
...r,
|
|
258
|
-
voice:
|
|
260
|
+
voice: a
|
|
259
261
|
} : r);
|
|
260
|
-
},
|
|
261
|
-
cancelPrefetch: () => e.cancelPrefetch?.(),
|
|
262
|
+
} } : {},
|
|
263
|
+
...e.cancelPrefetch ? { cancelPrefetch: () => e.cancelPrefetch?.() } : {},
|
|
262
264
|
speak: (n, a, o) => {
|
|
263
265
|
r();
|
|
264
266
|
let { voice: s, target: c } = t(a);
|
|
@@ -24,17 +24,17 @@ var x = 6e4, S = 2, C = 1e3;
|
|
|
24
24
|
function w(e) {
|
|
25
25
|
return x + e / C * 2 * 1e3;
|
|
26
26
|
}
|
|
27
|
-
var
|
|
27
|
+
var ee = class extends Error {
|
|
28
28
|
reason = "unknown";
|
|
29
29
|
constructor() {
|
|
30
30
|
super("audio output made no progress"), this.name = "OutputStalledError";
|
|
31
31
|
}
|
|
32
32
|
};
|
|
33
|
-
function
|
|
33
|
+
function T(e) {
|
|
34
34
|
let t = 0, n = !1, r = null, i = () => {
|
|
35
35
|
let n = r;
|
|
36
36
|
n !== null && (n.cancel(), n.cancel = e(() => {
|
|
37
|
-
r === n && (r = null, n.fail(new
|
|
37
|
+
r === n && (r = null, n.fail(new ee()));
|
|
38
38
|
}, w(t)));
|
|
39
39
|
};
|
|
40
40
|
return {
|
|
@@ -66,7 +66,7 @@ function E(e) {
|
|
|
66
66
|
}
|
|
67
67
|
};
|
|
68
68
|
}
|
|
69
|
-
var
|
|
69
|
+
var E = "data:audio/wav;base64,UklGRiQAAABXQVZFZm10IBAAAAABAAEARKwAAIhYAQACABAAZGF0YQAAAAA=", D = [], te = 4;
|
|
70
70
|
function O() {
|
|
71
71
|
for (let e of [...D]) e();
|
|
72
72
|
}
|
|
@@ -96,7 +96,7 @@ function A(v) {
|
|
|
96
96
|
if (!(k || typeof Audio != "function")) try {
|
|
97
97
|
let e = O ?? new Audio();
|
|
98
98
|
if (O = e, C === e) return;
|
|
99
|
-
e.muted = !0, e.src =
|
|
99
|
+
e.muted = !0, e.src = E, Promise.resolve(e.play()).then(() => {
|
|
100
100
|
O === e && C !== e && e.pause();
|
|
101
101
|
}, () => {});
|
|
102
102
|
} catch {}
|
|
@@ -139,32 +139,36 @@ function A(v) {
|
|
|
139
139
|
let Y = (e, t) => JSON.stringify(J(e, t)), X = () => {
|
|
140
140
|
I?.controller.abort(), I = null;
|
|
141
141
|
}, Z = (e, t, n) => {
|
|
142
|
-
let r =
|
|
143
|
-
|
|
142
|
+
let r = (e) => {
|
|
143
|
+
try {
|
|
144
|
+
t.onFirstClip?.(e);
|
|
145
|
+
} catch {}
|
|
146
|
+
}, i = I;
|
|
147
|
+
return I = null, i ? i.key === Y(e, t) ? i.failed ? (r("none"), null) : (n.aborted ? i.controller.abort() : n.addEventListener("abort", () => i.controller.abort(), { once: !0 }), t.onFirstClip && i.request.then(() => r("used"), () => r("none")), i.request) : (i.controller.abort(), r("mismatched"), null) : (r("none"), null);
|
|
144
148
|
};
|
|
145
|
-
async function Q(t, n, r, i = !1) {
|
|
146
|
-
let
|
|
147
|
-
for (let t = 1; e(r),
|
|
149
|
+
async function Q(t, n, r, i = !1, a = _) {
|
|
150
|
+
let o = J(t, n), l = v.getHeaders?.() ?? {}, u;
|
|
151
|
+
for (let t = 1; e(r), u = await y(v.endpoint, {
|
|
148
152
|
method: "POST",
|
|
149
153
|
signal: r,
|
|
150
154
|
headers: {
|
|
151
155
|
"content-type": "application/json",
|
|
152
|
-
...
|
|
156
|
+
...l
|
|
153
157
|
},
|
|
154
|
-
body: JSON.stringify(
|
|
155
|
-
}), !
|
|
156
|
-
if (!(g.has(
|
|
157
|
-
let e = /* @__PURE__ */ Error(`hosted tts ${
|
|
158
|
-
throw e.reason = i &&
|
|
158
|
+
body: JSON.stringify(o)
|
|
159
|
+
}), !u.ok; t += 1) {
|
|
160
|
+
if (!(g.has(u.status) && t < a && !r.aborted)) {
|
|
161
|
+
let e = /* @__PURE__ */ Error(`hosted tts ${u.status}`);
|
|
162
|
+
throw e.reason = i && u.status === 429 ? "rate-limited" : s(u.status), e.headers = h(l), e;
|
|
159
163
|
}
|
|
160
|
-
await ne(b(
|
|
164
|
+
await ne(b(u.headers.get("retry-after"), t), r);
|
|
161
165
|
}
|
|
162
|
-
let
|
|
163
|
-
if (!
|
|
166
|
+
let d = u.headers.get("content-type"), f = d?.includes("json") ? await u.json() : await u.arrayBuffer(), p = c(d, f);
|
|
167
|
+
if (!p) {
|
|
164
168
|
let e = /* @__PURE__ */ Error("hosted tts returned no audio");
|
|
165
169
|
throw e.reason = "unknown", e;
|
|
166
170
|
}
|
|
167
|
-
return
|
|
171
|
+
return p;
|
|
168
172
|
}
|
|
169
173
|
function $(e, r, i) {
|
|
170
174
|
return new Promise((a, o) => {
|
|
@@ -226,7 +230,7 @@ function A(v) {
|
|
|
226
230
|
}, c = (e) => {
|
|
227
231
|
let n = Q(e, t, a.signal);
|
|
228
232
|
return n.catch(() => {}), n;
|
|
229
|
-
}, f = t.audioOutput ?
|
|
233
|
+
}, f = t.audioOutput ? T(S) : null, h = t.audioOutput ? p(t.audioOutput, {
|
|
230
234
|
play: (e) => $(e, i, s),
|
|
231
235
|
isCurrent: () => i === j,
|
|
232
236
|
onChunkStart: (e) => {
|
|
@@ -258,7 +262,7 @@ function A(v) {
|
|
|
258
262
|
if (a.abort(), i !== j) return;
|
|
259
263
|
let t = null;
|
|
260
264
|
if (h) {
|
|
261
|
-
let n = e instanceof
|
|
265
|
+
let n = e instanceof ee ? null : await g(h.drain()).then(() => null, (e) => e);
|
|
262
266
|
if (i !== j) return;
|
|
263
267
|
h.cancel(), m(e) ? t = e : m(n) && (t = n);
|
|
264
268
|
}
|
|
@@ -275,7 +279,7 @@ function A(v) {
|
|
|
275
279
|
ie.cancel();
|
|
276
280
|
let n = ++j;
|
|
277
281
|
A();
|
|
278
|
-
let r = e.audioOutput ?
|
|
282
|
+
let r = e.audioOutput ? T(S) : null, i, a = -1, o = -1, s = -1, c = null, d = e.audioOutput ? p(e.audioOutput, {
|
|
279
283
|
play: (e) => $(e, n, () => i?.()),
|
|
280
284
|
isCurrent: () => n === j,
|
|
281
285
|
onChunkStart: () => {},
|
|
@@ -347,12 +351,18 @@ function A(v) {
|
|
|
347
351
|
let n = Y(e, t);
|
|
348
352
|
if (I?.key === n) return;
|
|
349
353
|
X();
|
|
350
|
-
|
|
351
|
-
|
|
354
|
+
try {
|
|
355
|
+
t.onEarlyFetch?.();
|
|
356
|
+
} catch {}
|
|
357
|
+
let r = new AbortController(), i = Q(e, t, r.signal, !1, 1), a = {
|
|
352
358
|
key: n,
|
|
353
359
|
controller: r,
|
|
354
|
-
request: i
|
|
360
|
+
request: i,
|
|
361
|
+
failed: !1
|
|
355
362
|
};
|
|
363
|
+
i.catch(() => {
|
|
364
|
+
a.failed = !0;
|
|
365
|
+
}), I = a;
|
|
356
366
|
},
|
|
357
367
|
cancelPrefetch() {
|
|
358
368
|
X();
|
|
@@ -364,4 +374,4 @@ function A(v) {
|
|
|
364
374
|
};
|
|
365
375
|
return ie;
|
|
366
376
|
}
|
|
367
|
-
export { C as HOSTED_OUTPUT_MIN_AUDIO_BYTES_PER_SECOND, S as HOSTED_OUTPUT_STALL_AUDIO_RATIO, x as HOSTED_OUTPUT_STALL_BASE_MS,
|
|
377
|
+
export { C as HOSTED_OUTPUT_MIN_AUDIO_BYTES_PER_SECOND, S as HOSTED_OUTPUT_STALL_AUDIO_RATIO, x as HOSTED_OUTPUT_STALL_BASE_MS, E as SILENT_WAV_DATA_URI, A as createHostedTtsEngine, w as hostedOutputStallMs, O as primeHostedPlayback, k as resetHostedPlaybackForTest, b as retryDelayMs };
|
|
@@ -4,4 +4,4 @@ export { configureSpeechEngine, createSpeechEngines, getSpeechEngine, readSpeech
|
|
|
4
4
|
export { createRoutingSpeechEngine } from './routingSpeechEngine';
|
|
5
5
|
export type { SpeechEngineConfig } from './createSpeechEngine';
|
|
6
6
|
export { HOSTED_VOICES, HOSTED_TTS_TIER, hostedSupportsLanguage } from './hostedVoiceCatalogue';
|
|
7
|
-
export type { SpeechEngine, SpeechQueue, SpeechErrorReason, SpeechSpeakHandlers, SpeechSpeakOptions, SpeechVoiceLike, } from './types';
|
|
7
|
+
export type { FirstClipOutcome, SpeechEngine, SpeechQueue, SpeechErrorReason, SpeechSpeakHandlers, SpeechSpeakOptions, SpeechVoiceLike, } from './types';
|
|
@@ -1,3 +1,16 @@
|
|
|
1
1
|
import { SpeechEngine } from './types';
|
|
2
|
-
/**
|
|
2
|
+
/**
|
|
3
|
+
* Notify the displaced reader even when a platform cancel suppresses its
|
|
4
|
+
* end/error event.
|
|
5
|
+
*
|
|
6
|
+
* ★★ P1 (BOFF-12936, fix 1): every optional member the engine below has is
|
|
7
|
+
* offered here too, and only then (`SPEECH_ENGINE_OPTIONAL_MEMBERS`;
|
|
8
|
+
* `speechEngineMembers.test.ts` holds every wrapper to that). `open` is wrapped
|
|
9
|
+
* by the cancellation guard; the others are pure pass-throughs. `prefetch` was
|
|
10
|
+
* once left out, which made the first clip's early fetch (P1) a no-op in every
|
|
11
|
+
* app: this layer sits over the routing engine in `getSpeechEngine()`. A
|
|
12
|
+
* prefetch plays nothing, so it neither cancels the reply in flight nor reports
|
|
13
|
+
* anything to it, and `cancel()` never drops the clip: the hosted engine keeps
|
|
14
|
+
* it across a cancel by design, because every reply cancels before it starts.
|
|
15
|
+
*/
|
|
3
16
|
export declare function notifyCancellationSpeechEngine(engine: SpeechEngine): SpeechEngine;
|
|
@@ -10,6 +10,9 @@ function e(e) {
|
|
|
10
10
|
listVoices: () => e.listVoices(),
|
|
11
11
|
subscribeVoices: (t) => e.subscribeVoices(t),
|
|
12
12
|
cancel: n,
|
|
13
|
+
...e.prefetch ? { prefetch: (t, n) => e.prefetch?.(t, n) } : {},
|
|
14
|
+
...e.cancelPrefetch ? { cancelPrefetch: () => e.cancelPrefetch?.() } : {},
|
|
15
|
+
...e.wouldSpeakOnAvatar ? { wouldSpeakOnAvatar: (t) => e.wouldSpeakOnAvatar?.(t) === !0 } : {},
|
|
13
16
|
...e.open ? { open(r, i = {}) {
|
|
14
17
|
n();
|
|
15
18
|
let a = { handlers: i };
|
|
@@ -48,12 +48,12 @@ function i({ device: i, hosted: a }) {
|
|
|
48
48
|
t(), n();
|
|
49
49
|
};
|
|
50
50
|
},
|
|
51
|
-
prefetch(e, t) {
|
|
51
|
+
...a.prefetch ? { prefetch(e, t) {
|
|
52
52
|
u(t) === a && a.prefetch?.(e, t);
|
|
53
|
-
},
|
|
54
|
-
cancelPrefetch() {
|
|
53
|
+
} } : {},
|
|
54
|
+
...a.cancelPrefetch ? { cancelPrefetch() {
|
|
55
55
|
a.cancelPrefetch?.();
|
|
56
|
-
},
|
|
56
|
+
} } : {},
|
|
57
57
|
speak(e, t, s) {
|
|
58
58
|
if (o?.cancel(), o = null, i.cancel(), a.cancel(), u(t) !== a) {
|
|
59
59
|
l(e, t, s);
|
|
@@ -20,7 +20,9 @@
|
|
|
20
20
|
* is written by the publisher first.
|
|
21
21
|
* - `abort` is called on EVERY exit that follows a publish without completing
|
|
22
22
|
* the turn (abandonment, an error after the answer) — the same rule as the
|
|
23
|
-
* existing "never read a cancelled answer aloud" check.
|
|
23
|
+
* existing "never read a cancelled answer aloud" check. ★ P1 fix 5
|
|
24
|
+
* (BOFF-12936): and on every exit that follows a `prefetch`, published or
|
|
25
|
+
* not, so the first clip fetched early is dropped with the turn.
|
|
24
26
|
* - A subscriber that throws never breaks the turn: speech is never worth an
|
|
25
27
|
* answer (BOFF-7283).
|
|
26
28
|
*/
|
|
@@ -36,6 +38,16 @@ export interface SpeechFeedItem {
|
|
|
36
38
|
lang?: string | undefined;
|
|
37
39
|
/** False only for opted-in S7 text already written to the live overlay. */
|
|
38
40
|
final: boolean;
|
|
41
|
+
/**
|
|
42
|
+
* ★★ P1 review (BOFF-12936), on a `prefetch` only: `false` when this turn's
|
|
43
|
+
* narrator will release nothing for the answer as it stands
|
|
44
|
+
* (`narratorReleasesNothing`: the turn has no narrator, it stopped before its
|
|
45
|
+
* first release, or the stream can carry no live text and carried none). The
|
|
46
|
+
* reply is then read whole at its publish, so the first clip's early fetch
|
|
47
|
+
* does not predict the phrase route for it. Absent: the phrase route may
|
|
48
|
+
* speak it.
|
|
49
|
+
*/
|
|
50
|
+
phraseRoute?: false;
|
|
39
51
|
}
|
|
40
52
|
export type SpeechFeedEvent = {
|
|
41
53
|
type: 'publish';
|
|
@@ -74,11 +86,16 @@ export declare const speechFeed: {
|
|
|
74
86
|
* while it is still being written (voice turns of an opted-in app only). The
|
|
75
87
|
* subscriber may fetch the clip the finished answer will start with; nothing
|
|
76
88
|
* is spoken, so the "never speak what is not on screen" rule is untouched.
|
|
77
|
-
* Not remembered
|
|
89
|
+
* Not remembered for `peek`, which stays about published answers; its id is
|
|
90
|
+
* remembered for `abort` (fix 5).
|
|
78
91
|
*/
|
|
79
92
|
prefetch(messageId: string, item: SpeechFeedItem): void;
|
|
80
93
|
audio(messageId: string, state: "queued" | "started" | "paused" | "ended" | "retrying"): void;
|
|
81
|
-
/**
|
|
94
|
+
/**
|
|
95
|
+
* Forget a published or prefetched message and tell the subscriber to stop
|
|
96
|
+
* speaking it and to drop its early clip. Delivered once, and only for a
|
|
97
|
+
* message published or prefetched since its last abort.
|
|
98
|
+
*/
|
|
82
99
|
abort(messageId: string, retrying?: boolean): void;
|
|
83
100
|
subscribe(listener: Listener): () => void;
|
|
84
101
|
/**
|
|
@@ -1,58 +1,66 @@
|
|
|
1
1
|
// src/shared/assistant/speech/speechFeed.ts
|
|
2
|
-
var e = /* @__PURE__ */ new Set(), t = /* @__PURE__ */ new Map();
|
|
3
|
-
function
|
|
2
|
+
var e = /* @__PURE__ */ new Set(), t = /* @__PURE__ */ new Map(), n = /* @__PURE__ */ new Set();
|
|
3
|
+
function r(t) {
|
|
4
4
|
for (let n of [...e]) try {
|
|
5
5
|
n(t);
|
|
6
6
|
} catch {}
|
|
7
7
|
}
|
|
8
|
-
function
|
|
8
|
+
function i(e, n) {
|
|
9
9
|
for (t.delete(e), t.set(e, n); t.size > 16;) {
|
|
10
10
|
let e = t.keys().next();
|
|
11
11
|
if (e.done) break;
|
|
12
12
|
t.delete(e.value);
|
|
13
13
|
}
|
|
14
14
|
}
|
|
15
|
-
|
|
15
|
+
function a(e) {
|
|
16
|
+
for (n.delete(e), n.add(e); n.size > 16;) {
|
|
17
|
+
let e = n.values().next();
|
|
18
|
+
if (e.done) break;
|
|
19
|
+
n.delete(e.value);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
var o = {
|
|
16
23
|
publish(e, t) {
|
|
17
|
-
|
|
24
|
+
i(e, t), r({
|
|
18
25
|
type: "publish",
|
|
19
26
|
messageId: e,
|
|
20
27
|
item: t
|
|
21
28
|
});
|
|
22
29
|
},
|
|
23
|
-
append(e, t,
|
|
24
|
-
|
|
30
|
+
append(e, t, n) {
|
|
31
|
+
i(e, t), r({
|
|
25
32
|
type: "append",
|
|
26
33
|
messageId: e,
|
|
27
34
|
item: t,
|
|
28
|
-
phrases:
|
|
35
|
+
phrases: n
|
|
29
36
|
});
|
|
30
37
|
},
|
|
31
38
|
retract(e) {
|
|
32
|
-
|
|
39
|
+
r({
|
|
33
40
|
type: "retract",
|
|
34
41
|
messageId: e
|
|
35
42
|
});
|
|
36
43
|
},
|
|
37
44
|
prefetch(e, t) {
|
|
38
|
-
|
|
45
|
+
a(e), r({
|
|
39
46
|
type: "prefetch",
|
|
40
47
|
messageId: e,
|
|
41
48
|
item: t
|
|
42
49
|
});
|
|
43
50
|
},
|
|
44
51
|
audio(e, t) {
|
|
45
|
-
|
|
52
|
+
r({
|
|
46
53
|
type: "audio",
|
|
47
54
|
messageId: e,
|
|
48
55
|
state: t
|
|
49
56
|
});
|
|
50
57
|
},
|
|
51
|
-
abort(e,
|
|
52
|
-
t.delete(e)
|
|
58
|
+
abort(e, i = !1) {
|
|
59
|
+
let a = t.delete(e), o = n.delete(e);
|
|
60
|
+
(a || o) && r({
|
|
53
61
|
type: "abort",
|
|
54
62
|
messageId: e,
|
|
55
|
-
...
|
|
63
|
+
...i ? { retrying: !0 } : {}
|
|
56
64
|
});
|
|
57
65
|
},
|
|
58
66
|
subscribe(t) {
|
|
@@ -64,7 +72,7 @@ var i = {
|
|
|
64
72
|
return t.get(e);
|
|
65
73
|
}
|
|
66
74
|
};
|
|
67
|
-
function
|
|
68
|
-
e.clear(), t.clear();
|
|
75
|
+
function s() {
|
|
76
|
+
e.clear(), t.clear(), n.clear();
|
|
69
77
|
}
|
|
70
|
-
export {
|
|
78
|
+
export { s as resetSpeechFeedForTest, o as speechFeed };
|
|
@@ -40,7 +40,43 @@ export interface SpeechSpeakOptions {
|
|
|
40
40
|
* for it too. A refusal (`output-not-cleared`) is reported as it always is.
|
|
41
41
|
*/
|
|
42
42
|
avatarOnly?: boolean;
|
|
43
|
+
/**
|
|
44
|
+
* ★ P1 measurement (BOFF-12936, fix 8): told what this reply's first piece
|
|
45
|
+
* found in the engine's early-clip slot (`FirstClipOutcome`), once. Attached
|
|
46
|
+
* per reply by the speech provider for a voice turn, like `audioOutput`, and
|
|
47
|
+
* carried with the rest of the options through every layer (each passes the
|
|
48
|
+
* caller's options on, or a copy of them), so no wrapper has a member to
|
|
49
|
+
* forward; `speechEngineMembers.test.ts` holds every layer to that. Only an
|
|
50
|
+
* engine that keeps an early clip (the hosted voice) calls it, and a reporter
|
|
51
|
+
* that throws is ignored there; every other engine ignores it. Not part of
|
|
52
|
+
* the request: two replies that differ only here ask for the same clip.
|
|
53
|
+
*/
|
|
54
|
+
onFirstClip?: (outcome: FirstClipOutcome) => void;
|
|
55
|
+
/**
|
|
56
|
+
* ★ P1 measurement (BOFF-12936, review): told, once, when a `prefetch` with
|
|
57
|
+
* these options SENDS its request: the turn that asked has paid for a clip,
|
|
58
|
+
* used or not (the proxy charges a request before its cache lookup, and an
|
|
59
|
+
* abort from the client stops the download, not the synthesis). Attached by
|
|
60
|
+
* the speech provider to a voice turn's early fetch, and carried with the
|
|
61
|
+
* options through every layer, like `onFirstClip`. Only the hosted engine
|
|
62
|
+
* calls it, right before its one early request; never for a repeat of the
|
|
63
|
+
* clip it already holds, never while it is switched off, and never when a
|
|
64
|
+
* layer above sends nothing (the device voice, a live avatar still
|
|
65
|
+
* starting). A callback that throws is ignored there. Not part of the
|
|
66
|
+
* request.
|
|
67
|
+
*/
|
|
68
|
+
onEarlyFetch?: () => void;
|
|
43
69
|
}
|
|
70
|
+
/**
|
|
71
|
+
* What a reply's first piece found in the early-clip slot (P1, fix 8):
|
|
72
|
+
* - `used`: its audio is the clip fetched early (it did not wait for a
|
|
73
|
+
* synthesis of its own);
|
|
74
|
+
* - `mismatched`: a clip fetched early for another request was there, and was
|
|
75
|
+
* dropped (paid for, never heard);
|
|
76
|
+
* - `none`: nothing usable: no clip was there, or the early fetch failed (or
|
|
77
|
+
* was stopped) and the reply fetched its own piece.
|
|
78
|
+
*/
|
|
79
|
+
export type FirstClipOutcome = 'used' | 'mismatched' | 'none';
|
|
44
80
|
/**
|
|
45
81
|
* Somewhere other than the speakers to play a reply's hosted audio
|
|
46
82
|
* (BOFF-8822).
|
|
@@ -238,12 +274,43 @@ export interface SpeechEngine {
|
|
|
238
274
|
* reply is released, so it can start the moment it is. Plays nothing; the
|
|
239
275
|
* clip is used only by the next reply whose first piece is exactly this
|
|
240
276
|
* request. Optional: engines that synthesise nothing (the device voice)
|
|
241
|
-
* omit it, and callers read that as "nothing to prefetch".
|
|
277
|
+
* omit it, and callers read that as "nothing to prefetch". An engine that
|
|
278
|
+
* sends a request for it says so (`options.onEarlyFetch`).
|
|
242
279
|
*/
|
|
243
280
|
prefetch?(text: string, options: SpeechSpeakOptions): void;
|
|
244
281
|
/** Drop a prefetched piece that will not be used (the reply was stopped or replaced). */
|
|
245
282
|
cancelPrefetch?(): void;
|
|
246
283
|
}
|
|
284
|
+
/**
|
|
285
|
+
* ★★ P1 (BOFF-12936, fix 1): every OPTIONAL member of `SpeechEngine`, by name.
|
|
286
|
+
*
|
|
287
|
+
* P1 shipped inert because one wrapper (`notifyCancellationSpeechEngine`)
|
|
288
|
+
* listed by hand the members it passed on, and `prefetch` was not among them:
|
|
289
|
+
* every layer above it read "no prefetch" and nothing was fetched early, with
|
|
290
|
+
* no test failing. A wrapper must decide, for each optional member, whether it
|
|
291
|
+
* forwards it, implements it or never offers it, and
|
|
292
|
+
* `speechEngineMembers.test.ts` asks every wrapper (notify, routing, the live
|
|
293
|
+
* avatar's decorator) that question for every name here. The list is held to
|
|
294
|
+
* the interface at compile time (`SpeechEngineOptionalMembersInSync`): an
|
|
295
|
+
* optional member added to `SpeechEngine` and not named here, or a name here
|
|
296
|
+
* that is not one, fails the type check.
|
|
297
|
+
*/
|
|
298
|
+
export declare const SPEECH_ENGINE_OPTIONAL_MEMBERS: readonly ["open", "wouldSpeakOnAvatar", "prefetch", "cancelPrefetch"];
|
|
299
|
+
export type SpeechEngineOptionalMember = (typeof SPEECH_ENGINE_OPTIONAL_MEMBERS)[number];
|
|
300
|
+
/** The keys of `T` that may be absent. */
|
|
301
|
+
type OptionalKeysOf<T> = {
|
|
302
|
+
[K in keyof T]-?: Pick<T, K> extends Required<Pick<T, K>> ? never : K;
|
|
303
|
+
}[keyof T];
|
|
304
|
+
/** True only when `A` and `B` are the same union. */
|
|
305
|
+
type SameUnion<A, B> = [A] extends [B] ? ([B] extends [A] ? true : false) : false;
|
|
306
|
+
/** Compiles only for `true`. */
|
|
307
|
+
type CompileTimeCheck<T extends true> = T;
|
|
308
|
+
/**
|
|
309
|
+
* ★★ The compile-time half of the guard (fix 1): fails the type check unless
|
|
310
|
+
* `SPEECH_ENGINE_OPTIONAL_MEMBERS` names exactly the optional members of
|
|
311
|
+
* `SpeechEngine`. Nothing reads it at run time.
|
|
312
|
+
*/
|
|
313
|
+
export type SpeechEngineOptionalMembersInSync = CompileTimeCheck<SameUnion<SpeechEngineOptionalMember, OptionalKeysOf<SpeechEngine>>>;
|
|
247
314
|
/** One reply. Appends never pre-empt the piece currently playing. */
|
|
248
315
|
export interface SpeechQueue {
|
|
249
316
|
/** False means full or terminal; a producer must retain its unaccepted text. */
|
|
@@ -1,3 +1,8 @@
|
|
|
1
1
|
// src/shared/assistant/speech/types.ts
|
|
2
|
-
var e = "left", t = "refuse-audible"
|
|
3
|
-
|
|
2
|
+
var e = "left", t = "refuse-audible", n = [
|
|
3
|
+
"open",
|
|
4
|
+
"wouldSpeakOnAvatar",
|
|
5
|
+
"prefetch",
|
|
6
|
+
"cancelPrefetch"
|
|
7
|
+
];
|
|
8
|
+
export { n as SPEECH_ENGINE_OPTIONAL_MEMBERS, e as SPEECH_OUTPUT_LEFT, t as SPEECH_OUTPUT_REFUSE_AUDIBLE };
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { FirstClipOutcome } from './speech/types';
|
|
1
2
|
import { StoppedTurnOutcome } from './turnState';
|
|
2
3
|
export declare const TURN_MILESTONES: readonly ["speechEnded", "lastSpeechFrame", "recognizerSpeechEnded", "graceStarted", "graceEnded", "handedOver", "accepted", "cleared", "attemptStarted", "prewarmJoined", "bootAdopted", "provisionStarted", "createRequested", "sandboxRunning", "serviceReady", "runtimeReady", "promptSent", "lastQuietPoll", "streamOpen", "streamRemint", "firstDelta", "firstOutput", "firstText", "streamIdle", "answerDone", "complete", "speechQueued", "ttsStarted", "firstAudio", "audioEnded"];
|
|
3
4
|
export type TurnMilestone = (typeof TURN_MILESTONES)[number];
|
|
@@ -54,6 +55,37 @@ export interface TurnAttemptFacts {
|
|
|
54
55
|
/** S7 local diagnostics only: no text and no new metric labels. */
|
|
55
56
|
spokeNarration?: boolean;
|
|
56
57
|
speechDivergence?: boolean;
|
|
58
|
+
/**
|
|
59
|
+
* ★ P1 measurement (BOFF-12936, fix 8): what a voice turn's reply found, at
|
|
60
|
+
* its first piece, in the engine's early-clip slot (`FirstClipOutcome`:
|
|
61
|
+
* `used`, `mismatched` or `none`), reported by the hosted engine itself and
|
|
62
|
+
* recorded by the speech provider for the live request. `none` too when no
|
|
63
|
+
* engine reported (the device voice), and for a mismatch with a clip another
|
|
64
|
+
* turn left behind (not this turn's waste). The first answer stands; a retry
|
|
65
|
+
* records its own (`restartAttempt` files the first attempt's).
|
|
66
|
+
*
|
|
67
|
+
* LOCAL ONLY, like the S7 flags above. The timing report is built from an
|
|
68
|
+
* allowlist (`turnTimingReport.ts`) and does not carry it, and
|
|
69
|
+
* wspace-sandbox-svc's `AssistantTurnTimingInput` has no such field: the
|
|
70
|
+
* gateway refuses an input field its type does not define, and the reporter
|
|
71
|
+
* reads that refusal as final and stops reporting for the page
|
|
72
|
+
* (`turnTimingReporter.ts`). Carrying it needs that field in the service
|
|
73
|
+
* first (docs/assistant-turn-latency.md). Read it from this timeline: the
|
|
74
|
+
* `burdenoff:assistant-turn-timeline` window event or `recentTurnTimelines()`.
|
|
75
|
+
*/
|
|
76
|
+
firstClip?: FirstClipOutcome;
|
|
77
|
+
/**
|
|
78
|
+
* ★ P1 measurement (BOFF-12936, review): this attempt's early fetch SENT its
|
|
79
|
+
* request (`SpeechSpeakOptions.onEarlyFetch`), so it paid for a clip, used
|
|
80
|
+
* or not: the proxy charges a request before its cache lookup and does not
|
|
81
|
+
* see the client abort it, so dropping the clip on an exit saves its
|
|
82
|
+
* download, not its cost. Recorded on the turn that paid, even when its
|
|
83
|
+
* reply never starts (a stop, a supersession, End, a failover's first
|
|
84
|
+
* attempt) or another turn's reply drops the clip it left behind, which
|
|
85
|
+
* `firstClip` alone cannot show. Absent: nothing was sent. Local only, for
|
|
86
|
+
* the same reason as `firstClip`; a retry records its own.
|
|
87
|
+
*/
|
|
88
|
+
firstClipFetched?: true;
|
|
57
89
|
runtime?: TurnRuntime;
|
|
58
90
|
/** The model called at least one tool on the way to its answer. */
|
|
59
91
|
usedTools?: boolean;
|
|
@@ -85,7 +85,7 @@ function m(e, t) {
|
|
|
85
85
|
if (!n) return;
|
|
86
86
|
let r = !1;
|
|
87
87
|
for (let e of ["spokeNarration", "speechDivergence"]) t[e] === !0 && n[e] !== !0 && (n[e] = !0, r = !0);
|
|
88
|
-
t.runtime !== void 0 && n.runtime === void 0 && (n.runtime = t.runtime, r = !0), t.usedTools !== void 0 && n.usedTools === void 0 && (n.usedTools = t.usedTools, r = !0), t.deadSandboxProbe !== void 0 && n.deadSandboxProbe === void 0 && (n.deadSandboxProbe = t.deadSandboxProbe, r = !0), t.cacheProofFailed !== void 0 && n.cacheProofFailed === void 0 && (n.cacheProofFailed = t.cacheProofFailed, r = !0), t.prewarm !== void 0 && n.prewarm === void 0 && (n.prewarm = t.prewarm, r = !0), t.transport !== void 0 && n.transport === void 0 && (n.transport = t.transport, r = !0), t.live !== void 0 && n.live === void 0 && (n.live = t.live, r = !0), t.session !== void 0 && n.session === void 0 && (n.session = t.session, r = !0), t.outcome !== void 0 && n.outcome === void 0 && (n.outcome = t.outcome, r = !0), r && d(n);
|
|
88
|
+
t.firstClip !== void 0 && n.firstClip === void 0 && (n.firstClip = t.firstClip, r = !0), t.firstClipFetched === !0 && n.firstClipFetched !== !0 && (n.firstClipFetched = !0, r = !0), t.runtime !== void 0 && n.runtime === void 0 && (n.runtime = t.runtime, r = !0), t.usedTools !== void 0 && n.usedTools === void 0 && (n.usedTools = t.usedTools, r = !0), t.deadSandboxProbe !== void 0 && n.deadSandboxProbe === void 0 && (n.deadSandboxProbe = t.deadSandboxProbe, r = !0), t.cacheProofFailed !== void 0 && n.cacheProofFailed === void 0 && (n.cacheProofFailed = t.cacheProofFailed, r = !0), t.prewarm !== void 0 && n.prewarm === void 0 && (n.prewarm = t.prewarm, r = !0), t.transport !== void 0 && n.transport === void 0 && (n.transport = t.transport, r = !0), t.live !== void 0 && n.live === void 0 && (n.live = t.live, r = !0), t.session !== void 0 && n.session === void 0 && (n.session = t.session, r = !0), t.outcome !== void 0 && n.outcome === void 0 && (n.outcome = t.outcome, r = !0), r && d(n);
|
|
89
89
|
}
|
|
90
90
|
function h(e, n = c()) {
|
|
91
91
|
let r = i.get(e);
|
|
@@ -98,9 +98,9 @@ function h(e, n = c()) {
|
|
|
98
98
|
let t = r.at[e];
|
|
99
99
|
t !== void 0 && (a.at[e] = t, delete r.at[e]);
|
|
100
100
|
}
|
|
101
|
-
r.runtime !== void 0 && (a.runtime = r.runtime), r.usedTools !== void 0 && (a.usedTools = r.usedTools), r.deadSandboxProbe !== void 0 && (a.deadSandboxProbe = r.deadSandboxProbe), r.cacheProofFailed !== void 0 && (a.cacheProofFailed = r.cacheProofFailed), r.prewarm !== void 0 && (a.prewarm = r.prewarm), r.transport !== void 0 && (a.transport = r.transport), r.live !== void 0 && (a.live = r.live), r.session !== void 0 && (a.session = r.session);
|
|
101
|
+
r.runtime !== void 0 && (a.runtime = r.runtime), r.usedTools !== void 0 && (a.usedTools = r.usedTools), r.deadSandboxProbe !== void 0 && (a.deadSandboxProbe = r.deadSandboxProbe), r.cacheProofFailed !== void 0 && (a.cacheProofFailed = r.cacheProofFailed), r.prewarm !== void 0 && (a.prewarm = r.prewarm), r.transport !== void 0 && (a.transport = r.transport), r.live !== void 0 && (a.live = r.live), r.session !== void 0 && (a.session = r.session), r.firstClip !== void 0 && (a.firstClip = r.firstClip), r.firstClipFetched !== void 0 && (a.firstClipFetched = r.firstClipFetched);
|
|
102
102
|
for (let e of ["spokeNarration", "speechDivergence"]) r[e] !== void 0 && (a[e] = r[e]), delete r[e];
|
|
103
|
-
delete r.transport, delete r.live, delete r.session, delete r.runtime, delete r.usedTools, delete r.deadSandboxProbe, delete r.cacheProofFailed, delete r.prewarm, r.attempts = [...r.attempts ?? [], a], r.retried = !0, d(r);
|
|
103
|
+
delete r.transport, delete r.live, delete r.session, delete r.firstClip, delete r.firstClipFetched, delete r.runtime, delete r.usedTools, delete r.deadSandboxProbe, delete r.cacheProofFailed, delete r.prewarm, r.attempts = [...r.attempts ?? [], a], r.retried = !0, d(r);
|
|
104
104
|
}
|
|
105
105
|
function g(e) {
|
|
106
106
|
let t = i.get(e);
|