@burdenoff/fe-libs 2026.1117.6 → 2026.1117.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/shared/assistant/avatar/live/createLiveAvatarSeam.js +8 -0
- package/dist/shared/assistant/index.d.ts +1 -0
- package/dist/shared/assistant/speech/hostedTtsSpeechEngine.js +132 -98
- package/dist/shared/assistant/speech/routingSpeechEngine.js +6 -0
- package/dist/shared/assistant/speech/speechFeed.d.ts +14 -0
- package/dist/shared/assistant/speech/speechFeed.js +7 -0
- package/dist/shared/assistant/speech/types.d.ts +10 -0
- package/dist/shared/assistant/turnState.d.ts +36 -0
- package/dist/shared/assistant/turnState.js +18 -1
- package/dist/shared/assistant/turnTimeline.d.ts +8 -88
- package/dist/shared/assistant/ui/AssistantChatArea.js +1212 -1201
- package/dist/shared/assistant/ui/AssistantSpeechProvider.js +281 -245
- package/dist/shared/assistant/voice/firstClipPrefetchConfig.d.ts +4 -0
- package/dist/shared/assistant/voice/firstClipPrefetchConfig.js +9 -0
- package/dist/shared/assistant/voice/speakableText.d.ts +18 -0
- package/dist/shared/assistant/voice/speakableText.js +113 -107
- package/dist/shared-assistant.js +381 -380
- package/package.json +1 -1
|
@@ -251,6 +251,14 @@ function W(e = {}) {
|
|
|
251
251
|
listVoices: () => e.listVoices(),
|
|
252
252
|
subscribeVoices: (t) => e.subscribeVoices(t),
|
|
253
253
|
wouldSpeakOnAvatar: (e) => t(e).target === "avatar",
|
|
254
|
+
prefetch: (n, r) => {
|
|
255
|
+
let { voice: i, target: a } = t(r);
|
|
256
|
+
e.prefetch?.(n, a === "avatar" && i !== null ? {
|
|
257
|
+
...r,
|
|
258
|
+
voice: i
|
|
259
|
+
} : r);
|
|
260
|
+
},
|
|
261
|
+
cancelPrefetch: () => e.cancelPrefetch?.(),
|
|
254
262
|
speak: (n, a, o) => {
|
|
255
263
|
r();
|
|
256
264
|
let { voice: s, target: c } = t(a);
|
|
@@ -422,6 +422,7 @@ export * from './avatar/live/avatarVoice';
|
|
|
422
422
|
export * from './promptDelivery';
|
|
423
423
|
export * from './speech/appendSpeechQueue';
|
|
424
424
|
export * from './voice/phraseSpeechConfig';
|
|
425
|
+
export * from './voice/firstClipPrefetchConfig';
|
|
425
426
|
export * from './voice/phraseGate';
|
|
426
427
|
export * from './voice/livePhraseSpeech';
|
|
427
428
|
export * from './voice/phrasePlayback';
|
|
@@ -66,14 +66,14 @@ function E(e) {
|
|
|
66
66
|
}
|
|
67
67
|
};
|
|
68
68
|
}
|
|
69
|
-
var
|
|
70
|
-
function
|
|
71
|
-
for (let e of [...
|
|
69
|
+
var ee = "data:audio/wav;base64,UklGRiQAAABXQVZFZm10IBAAAAABAAEARKwAAIhYAQACABAAZGF0YQAAAAA=", D = [], te = 4;
|
|
70
|
+
function O() {
|
|
71
|
+
for (let e of [...D]) e();
|
|
72
72
|
}
|
|
73
|
-
function
|
|
74
|
-
|
|
73
|
+
function k() {
|
|
74
|
+
D.length = 0;
|
|
75
75
|
}
|
|
76
|
-
function
|
|
76
|
+
function ne(e, t) {
|
|
77
77
|
return new Promise((n) => {
|
|
78
78
|
if (t.aborted) {
|
|
79
79
|
n();
|
|
@@ -88,41 +88,41 @@ function M(e, t) {
|
|
|
88
88
|
t.addEventListener("abort", i, { once: !0 });
|
|
89
89
|
});
|
|
90
90
|
}
|
|
91
|
-
function
|
|
91
|
+
function A(v) {
|
|
92
92
|
let y = v.fetchImpl ?? ((...e) => fetch(...e)), x = v.now ?? (() => Date.now()), S = v.setTimer ?? ((e, t) => {
|
|
93
93
|
let n = setTimeout(e, t);
|
|
94
94
|
return () => clearTimeout(n);
|
|
95
|
-
}), C = null, w = null,
|
|
96
|
-
if (!(
|
|
97
|
-
let e =
|
|
98
|
-
if (
|
|
99
|
-
e.muted = !0, e.src =
|
|
100
|
-
|
|
95
|
+
}), C = null, w = null, O = null, k = !1, A = () => {
|
|
96
|
+
if (!(k || typeof Audio != "function")) try {
|
|
97
|
+
let e = O ?? new Audio();
|
|
98
|
+
if (O = e, C === e) return;
|
|
99
|
+
e.muted = !0, e.src = ee, Promise.resolve(e.play()).then(() => {
|
|
100
|
+
O === e && C !== e && e.pause();
|
|
101
101
|
}, () => {});
|
|
102
102
|
} catch {}
|
|
103
103
|
};
|
|
104
|
-
|
|
105
|
-
let
|
|
104
|
+
D.push(A), D.length > te && D.shift();
|
|
105
|
+
let j = 0, M = null, N = null, P = null, F = null, I = null, L = 0, R = null, z = null, re = () => {
|
|
106
106
|
try {
|
|
107
107
|
return v.getHeaders?.() ?? {};
|
|
108
108
|
} catch {
|
|
109
109
|
return {};
|
|
110
110
|
}
|
|
111
|
-
},
|
|
112
|
-
for (let e of [...
|
|
113
|
-
},
|
|
114
|
-
|
|
115
|
-
},
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
let e =
|
|
119
|
-
e > 0 ?
|
|
111
|
+
}, B = () => x() >= L ? !1 : z === null || h(re()) === z, V = /* @__PURE__ */ new Set(), H = () => {
|
|
112
|
+
for (let e of [...V]) e();
|
|
113
|
+
}, U = () => {
|
|
114
|
+
R !== null && clearTimeout(R), R = null;
|
|
115
|
+
}, W = (e) => {
|
|
116
|
+
U(), R = setTimeout(() => {
|
|
117
|
+
R = null;
|
|
118
|
+
let e = L - x();
|
|
119
|
+
e > 0 ? W(e) : H();
|
|
120
120
|
}, e);
|
|
121
|
-
},
|
|
122
|
-
|
|
123
|
-
},
|
|
121
|
+
}, G = (e, t) => {
|
|
122
|
+
L = x() + e, z = t, W(e), H();
|
|
123
|
+
}, K = () => {
|
|
124
124
|
w &&= (URL.revokeObjectURL(w), null);
|
|
125
|
-
},
|
|
125
|
+
}, q = () => {
|
|
126
126
|
if (C) {
|
|
127
127
|
C.onended = null, C.onerror = null;
|
|
128
128
|
try {
|
|
@@ -130,54 +130,64 @@ function N(v) {
|
|
|
130
130
|
} catch {}
|
|
131
131
|
C = null;
|
|
132
132
|
}
|
|
133
|
-
|
|
133
|
+
K();
|
|
134
|
+
};
|
|
135
|
+
function J(e, t) {
|
|
136
|
+
let n = a(t.lang) ?? t.lang;
|
|
137
|
+
return o(e, n, r, t.voice?.name ?? null, t.rate);
|
|
138
|
+
}
|
|
139
|
+
let Y = (e, t) => JSON.stringify(J(e, t)), X = () => {
|
|
140
|
+
I?.controller.abort(), I = null;
|
|
141
|
+
}, Z = (e, t, n) => {
|
|
142
|
+
let r = I;
|
|
143
|
+
return I = null, r ? r.key === Y(e, t) ? (n.aborted ? r.controller.abort() : n.addEventListener("abort", () => r.controller.abort(), { once: !0 }), r.request) : (r.controller.abort(), null) : null;
|
|
134
144
|
};
|
|
135
|
-
async function
|
|
136
|
-
let
|
|
137
|
-
for (let t = 1; e(
|
|
145
|
+
async function Q(t, n, r, i = !1) {
|
|
146
|
+
let a = J(t, n), o = v.getHeaders?.() ?? {}, l;
|
|
147
|
+
for (let t = 1; e(r), l = await y(v.endpoint, {
|
|
138
148
|
method: "POST",
|
|
139
|
-
signal:
|
|
149
|
+
signal: r,
|
|
140
150
|
headers: {
|
|
141
151
|
"content-type": "application/json",
|
|
142
|
-
...
|
|
152
|
+
...o
|
|
143
153
|
},
|
|
144
|
-
body: JSON.stringify(
|
|
145
|
-
}), !
|
|
146
|
-
if (!(g.has(
|
|
147
|
-
let e = /* @__PURE__ */ Error(`hosted tts ${
|
|
148
|
-
throw e.reason =
|
|
154
|
+
body: JSON.stringify(a)
|
|
155
|
+
}), !l.ok; t += 1) {
|
|
156
|
+
if (!(g.has(l.status) && t < _ && !r.aborted)) {
|
|
157
|
+
let e = /* @__PURE__ */ Error(`hosted tts ${l.status}`);
|
|
158
|
+
throw e.reason = i && l.status === 429 ? "rate-limited" : s(l.status), e.headers = h(o), e;
|
|
149
159
|
}
|
|
150
|
-
await
|
|
160
|
+
await ne(b(l.headers.get("retry-after"), t), r);
|
|
151
161
|
}
|
|
152
|
-
let
|
|
153
|
-
if (!
|
|
162
|
+
let u = l.headers.get("content-type"), d = u?.includes("json") ? await l.json() : await l.arrayBuffer(), f = c(u, d);
|
|
163
|
+
if (!f) {
|
|
154
164
|
let e = /* @__PURE__ */ Error("hosted tts returned no audio");
|
|
155
165
|
throw e.reason = "unknown", e;
|
|
156
166
|
}
|
|
157
|
-
return
|
|
167
|
+
return f;
|
|
158
168
|
}
|
|
159
|
-
function
|
|
169
|
+
function $(e, r, i) {
|
|
160
170
|
return new Promise((a, o) => {
|
|
161
|
-
if (r !==
|
|
171
|
+
if (r !== j) {
|
|
162
172
|
a();
|
|
163
173
|
return;
|
|
164
174
|
}
|
|
165
175
|
let s = new Blob([e.slice().buffer], { type: "audio/mpeg" });
|
|
166
|
-
|
|
176
|
+
K(), w = URL.createObjectURL(s);
|
|
167
177
|
let c;
|
|
168
|
-
t() ? (c = new Audio(w), n(c)) : (c =
|
|
169
|
-
r ===
|
|
178
|
+
t() ? (c = new Audio(w), n(c)) : (c = O ?? new Audio(), O = c, c.muted = !1, c.src = w), C = c, c.onended = () => {
|
|
179
|
+
r === j && a();
|
|
170
180
|
}, c.onerror = () => {
|
|
171
|
-
if (r !==
|
|
181
|
+
if (r !== j) {
|
|
172
182
|
a();
|
|
173
183
|
return;
|
|
174
184
|
}
|
|
175
185
|
let e = /* @__PURE__ */ Error("audio playback failed");
|
|
176
186
|
e.reason = "unknown", o(e);
|
|
177
187
|
}, c.play().then(() => {
|
|
178
|
-
c ===
|
|
188
|
+
c === O && !c.muted && (k = !0), r === j && i?.();
|
|
179
189
|
}, (e) => {
|
|
180
|
-
if (r !==
|
|
190
|
+
if (r !== j) {
|
|
181
191
|
a();
|
|
182
192
|
return;
|
|
183
193
|
}
|
|
@@ -186,100 +196,109 @@ function N(v) {
|
|
|
186
196
|
});
|
|
187
197
|
});
|
|
188
198
|
}
|
|
189
|
-
let
|
|
199
|
+
let ie = {
|
|
190
200
|
id: "hosted-tts",
|
|
191
201
|
isSupported() {
|
|
192
|
-
return !
|
|
202
|
+
return !B() && typeof window < "u" && typeof window.Audio == "function" && typeof fetch == "function";
|
|
193
203
|
},
|
|
194
204
|
listVoices() {
|
|
195
|
-
return
|
|
205
|
+
return B() ? [] : [...i];
|
|
196
206
|
},
|
|
197
207
|
subscribeVoices(e) {
|
|
198
|
-
return
|
|
199
|
-
|
|
208
|
+
return V.add(e), () => {
|
|
209
|
+
V.delete(e);
|
|
200
210
|
};
|
|
201
211
|
},
|
|
202
212
|
speak(e, t, n = {}) {
|
|
203
|
-
|
|
213
|
+
N?.cancel(), N = null;
|
|
204
214
|
let r = d(e);
|
|
205
215
|
if (r.length === 0) {
|
|
206
216
|
n.onEnd?.();
|
|
207
217
|
return;
|
|
208
218
|
}
|
|
209
|
-
|
|
210
|
-
let i =
|
|
211
|
-
|
|
219
|
+
j += 1;
|
|
220
|
+
let i = j;
|
|
221
|
+
q(), P?.cancel(), F?.stop(), A();
|
|
212
222
|
let a = new AbortController();
|
|
213
|
-
|
|
223
|
+
M = a;
|
|
214
224
|
let o = !1, s = () => {
|
|
215
225
|
o || (o = !0, n.onStart?.());
|
|
216
226
|
}, c = (e) => {
|
|
217
|
-
let n =
|
|
227
|
+
let n = Q(e, t, a.signal);
|
|
218
228
|
return n.catch(() => {}), n;
|
|
219
229
|
}, f = t.audioOutput ? E(S) : null, h = t.audioOutput ? p(t.audioOutput, {
|
|
220
|
-
play: (e) =>
|
|
221
|
-
isCurrent: () => i ===
|
|
230
|
+
play: (e) => $(e, i, s),
|
|
231
|
+
isCurrent: () => i === j,
|
|
222
232
|
onChunkStart: (e) => {
|
|
223
233
|
f?.progress(), n.onChunkStart?.(e);
|
|
224
234
|
},
|
|
225
235
|
onStart: s
|
|
226
236
|
}) : null;
|
|
227
|
-
|
|
237
|
+
P = h, F = f;
|
|
228
238
|
let g = (e) => f ? f.guard(e) : e, _ = () => {
|
|
229
|
-
h && (f?.stop(),
|
|
239
|
+
h && (f?.stop(), F === f && (F = null), P === h && (P = null));
|
|
230
240
|
};
|
|
231
241
|
(async () => {
|
|
232
242
|
try {
|
|
233
|
-
let e =
|
|
234
|
-
for (let
|
|
235
|
-
if (i !==
|
|
236
|
-
let
|
|
237
|
-
if (i !==
|
|
238
|
-
let
|
|
239
|
-
|
|
243
|
+
let e = r[0], o = Z(e, t, a.signal), l = o ? o.catch(() => c(e)) : c(e);
|
|
244
|
+
for (let e = 0; e < r.length; e += 1) {
|
|
245
|
+
if (i !== j) return;
|
|
246
|
+
let t = await l;
|
|
247
|
+
if (i !== j) return;
|
|
248
|
+
let a = r[e + 1];
|
|
249
|
+
a !== void 0 && (l = c(a)), h ? (f?.handed(t.byteLength), await g(h.write(t))) : (n.onChunkStart?.(e), await $(t, i, s));
|
|
240
250
|
}
|
|
241
251
|
if (h) {
|
|
242
252
|
let e = await g(h.drain());
|
|
243
|
-
if (i !==
|
|
253
|
+
if (i !== j) return;
|
|
244
254
|
if (!e) throw Object.assign(/* @__PURE__ */ Error("audio output did not finish"), { reason: "unknown" });
|
|
245
255
|
}
|
|
246
|
-
i ===
|
|
256
|
+
i === j && (_(), n.onEnd?.());
|
|
247
257
|
} catch (e) {
|
|
248
|
-
if (a.abort(), i !==
|
|
258
|
+
if (a.abort(), i !== j) return;
|
|
249
259
|
let t = null;
|
|
250
260
|
if (h) {
|
|
251
261
|
let n = e instanceof T ? null : await g(h.drain()).then(() => null, (e) => e);
|
|
252
|
-
if (i !==
|
|
262
|
+
if (i !== j) return;
|
|
253
263
|
h.cancel(), m(e) ? t = e : m(n) && (t = n);
|
|
254
264
|
}
|
|
255
|
-
if (
|
|
265
|
+
if (j += 1, q(), _(), t) {
|
|
256
266
|
n.onError?.("output-not-cleared");
|
|
257
267
|
return;
|
|
258
268
|
}
|
|
259
269
|
let r = e?.reason;
|
|
260
|
-
u(r, e?.transient === !0) &&
|
|
270
|
+
u(r, e?.transient === !0) && G(l(r), r === "not-allowed" ? e.headers ?? null : null), n.onError?.(r === "not-allowed" || r === "unsupported" || r === "cancelled" ? r : "unknown");
|
|
261
271
|
}
|
|
262
272
|
})();
|
|
263
273
|
},
|
|
264
274
|
open(e, t = {}) {
|
|
265
|
-
|
|
266
|
-
let n = ++
|
|
267
|
-
|
|
275
|
+
ie.cancel();
|
|
276
|
+
let n = ++j;
|
|
277
|
+
A();
|
|
268
278
|
let r = e.audioOutput ? E(S) : null, i, a = -1, o = -1, s = -1, c = null, d = e.audioOutput ? p(e.audioOutput, {
|
|
269
|
-
play: (e) =>
|
|
270
|
-
isCurrent: () => n ===
|
|
279
|
+
play: (e) => $(e, n, () => i?.()),
|
|
280
|
+
isCurrent: () => n === j,
|
|
271
281
|
onChunkStart: () => {},
|
|
272
282
|
onStart: () => i?.()
|
|
273
283
|
}) : null;
|
|
274
|
-
|
|
284
|
+
P = d, F = r;
|
|
275
285
|
let h = (e) => r ? r.guard(e) : e, g = async () => {
|
|
276
286
|
if (d && !await h(d.drain())) throw Error("audio output did not finish");
|
|
277
287
|
}, _ = () => {
|
|
278
|
-
r?.stop(),
|
|
279
|
-
}, v = f({
|
|
288
|
+
r?.stop(), F === r && (F = null), P === d && (P = null), N === y && (N = null);
|
|
289
|
+
}, v = !0, y = f({
|
|
280
290
|
prepare: async (t, n) => {
|
|
281
291
|
try {
|
|
282
|
-
|
|
292
|
+
if (v) {
|
|
293
|
+
v = !1;
|
|
294
|
+
let r = Z(t, e, n);
|
|
295
|
+
if (r) try {
|
|
296
|
+
return await r;
|
|
297
|
+
} catch (e) {
|
|
298
|
+
if (n.aborted) throw e;
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
return await Q(t, e, n, !0);
|
|
283
302
|
} catch (e) {
|
|
284
303
|
if (n.aborted) throw e;
|
|
285
304
|
try {
|
|
@@ -297,20 +316,20 @@ function N(v) {
|
|
|
297
316
|
let r = new Promise((e) => {
|
|
298
317
|
c = e;
|
|
299
318
|
});
|
|
300
|
-
await Promise.race([
|
|
319
|
+
await Promise.race([$(e, n, l), r]), c = null;
|
|
301
320
|
}
|
|
302
321
|
},
|
|
303
322
|
drain: g,
|
|
304
323
|
retract: (e) => {
|
|
305
|
-
d && o >= e ? d.retract(a) : !d && s >= e && (
|
|
324
|
+
d && o >= e ? d.retract(a) : !d && s >= e && (q(), c?.());
|
|
306
325
|
},
|
|
307
326
|
cancel: () => {
|
|
308
|
-
|
|
327
|
+
j += 1, q(), c?.(), d?.cancel(), _();
|
|
309
328
|
},
|
|
310
329
|
failed: (e) => {
|
|
311
330
|
if (m(e)) return "output-not-cleared";
|
|
312
331
|
let t = e, n = t?.reason;
|
|
313
|
-
return u(n, t?.transient === !0) &&
|
|
332
|
+
return u(n, t?.transient === !0) && G(l(n), n === "not-allowed" ? t?.headers ?? null : null), n === "not-allowed" || n === "unsupported" || n === "cancelled" || n === "rate-limited" ? n : "unknown";
|
|
314
333
|
}
|
|
315
334
|
}, {
|
|
316
335
|
...t,
|
|
@@ -318,16 +337,31 @@ function N(v) {
|
|
|
318
337
|
_(), t.onEnd?.();
|
|
319
338
|
}
|
|
320
339
|
});
|
|
321
|
-
return
|
|
340
|
+
return N = y, y;
|
|
322
341
|
},
|
|
323
342
|
headersChanged() {
|
|
324
|
-
x() >=
|
|
343
|
+
x() >= L || z === null || B() || (L = 0, z = null, U(), H());
|
|
344
|
+
},
|
|
345
|
+
prefetch(e, t) {
|
|
346
|
+
if (B() || e.trim() === "") return;
|
|
347
|
+
let n = Y(e, t);
|
|
348
|
+
if (I?.key === n) return;
|
|
349
|
+
X();
|
|
350
|
+
let r = new AbortController(), i = Q(e, t, r.signal);
|
|
351
|
+
i.catch(() => {}), I = {
|
|
352
|
+
key: n,
|
|
353
|
+
controller: r,
|
|
354
|
+
request: i
|
|
355
|
+
};
|
|
356
|
+
},
|
|
357
|
+
cancelPrefetch() {
|
|
358
|
+
X();
|
|
325
359
|
},
|
|
326
360
|
cancel() {
|
|
327
|
-
let e =
|
|
328
|
-
|
|
361
|
+
let e = N;
|
|
362
|
+
N = null, e?.cancel(), j += 1, M?.abort(), M = null, q(), P?.cancel(), P = null, F?.stop(), F = null;
|
|
329
363
|
}
|
|
330
364
|
};
|
|
331
|
-
return
|
|
365
|
+
return ie;
|
|
332
366
|
}
|
|
333
|
-
export { C as HOSTED_OUTPUT_MIN_AUDIO_BYTES_PER_SECOND, S as HOSTED_OUTPUT_STALL_AUDIO_RATIO, x as HOSTED_OUTPUT_STALL_BASE_MS,
|
|
367
|
+
export { C as HOSTED_OUTPUT_MIN_AUDIO_BYTES_PER_SECOND, S as HOSTED_OUTPUT_STALL_AUDIO_RATIO, x as HOSTED_OUTPUT_STALL_BASE_MS, ee as SILENT_WAV_DATA_URI, A as createHostedTtsEngine, w as hostedOutputStallMs, O as primeHostedPlayback, k as resetHostedPlaybackForTest, b as retryDelayMs };
|
|
@@ -48,6 +48,12 @@ function i({ device: i, hosted: a }) {
|
|
|
48
48
|
t(), n();
|
|
49
49
|
};
|
|
50
50
|
},
|
|
51
|
+
prefetch(e, t) {
|
|
52
|
+
u(t) === a && a.prefetch?.(e, t);
|
|
53
|
+
},
|
|
54
|
+
cancelPrefetch() {
|
|
55
|
+
a.cancelPrefetch?.();
|
|
56
|
+
},
|
|
51
57
|
speak(e, t, s) {
|
|
52
58
|
if (o?.cancel(), o = null, i.cancel(), a.cancel(), u(t) !== a) {
|
|
53
59
|
l(e, t, s);
|
|
@@ -49,6 +49,12 @@ export type SpeechFeedEvent = {
|
|
|
49
49
|
} | {
|
|
50
50
|
type: 'retract';
|
|
51
51
|
messageId: string;
|
|
52
|
+
}
|
|
53
|
+
/** P1 (BOFF-12936): the answer so far, still being written; nothing is spoken. */
|
|
54
|
+
| {
|
|
55
|
+
type: 'prefetch';
|
|
56
|
+
messageId: string;
|
|
57
|
+
item: SpeechFeedItem;
|
|
52
58
|
} | {
|
|
53
59
|
type: 'audio';
|
|
54
60
|
messageId: string;
|
|
@@ -63,6 +69,14 @@ export declare const speechFeed: {
|
|
|
63
69
|
publish(messageId: string, item: SpeechFeedItem): void;
|
|
64
70
|
append(messageId: string, item: SpeechFeedItem, phrases: readonly string[]): void;
|
|
65
71
|
retract(messageId: string): void;
|
|
72
|
+
/**
|
|
73
|
+
* ★ P1 (BOFF-12936, decision D1): the answer as the person sees it so far,
|
|
74
|
+
* while it is still being written (voice turns of an opted-in app only). The
|
|
75
|
+
* subscriber may fetch the clip the finished answer will start with; nothing
|
|
76
|
+
* is spoken, so the "never speak what is not on screen" rule is untouched.
|
|
77
|
+
* Not remembered: `peek` stays about published answers.
|
|
78
|
+
*/
|
|
79
|
+
prefetch(messageId: string, item: SpeechFeedItem): void;
|
|
66
80
|
audio(messageId: string, state: "queued" | "started" | "paused" | "ended" | "retrying"): void;
|
|
67
81
|
/** Forget a published message and tell the subscriber to stop speaking it. */
|
|
68
82
|
abort(messageId: string, retrying?: boolean): void;
|
|
@@ -233,6 +233,16 @@ export interface SpeechEngine {
|
|
|
233
233
|
* and only then.
|
|
234
234
|
*/
|
|
235
235
|
wouldSpeakOnAvatar?(options: SpeechSpeakOptions): boolean;
|
|
236
|
+
/**
|
|
237
|
+
* ★ P1 (BOFF-12936, decision D1): fetch a reply's first piece before the
|
|
238
|
+
* reply is released, so it can start the moment it is. Plays nothing; the
|
|
239
|
+
* clip is used only by the next reply whose first piece is exactly this
|
|
240
|
+
* request. Optional: engines that synthesise nothing (the device voice)
|
|
241
|
+
* omit it, and callers read that as "nothing to prefetch".
|
|
242
|
+
*/
|
|
243
|
+
prefetch?(text: string, options: SpeechSpeakOptions): void;
|
|
244
|
+
/** Drop a prefetched piece that will not be used (the reply was stopped or replaced). */
|
|
245
|
+
cancelPrefetch?(): void;
|
|
236
246
|
}
|
|
237
247
|
/** One reply. Appends never pre-empt the piece currently playing. */
|
|
238
248
|
export interface SpeechQueue {
|
|
@@ -106,3 +106,39 @@ export declare function planTurnAbort(args: {
|
|
|
106
106
|
send: boolean;
|
|
107
107
|
reason: TurnAbortReason;
|
|
108
108
|
};
|
|
109
|
+
/**
|
|
110
|
+
* ★ BOFF-12936: why a turn was cancelled, as the exit that cancelled it says.
|
|
111
|
+
* The turn's timing report carries it as its outcome, in the spelling
|
|
112
|
+
* wspace-sandbox-svc's turn histograms allowlist (#178), so `stopped` means
|
|
113
|
+
* only that the person stopped it:
|
|
114
|
+
*
|
|
115
|
+
* - `stopped`: an explicit Stop (the transcript's or the composer's), or a tap
|
|
116
|
+
* on the voice orb;
|
|
117
|
+
* - `superseded`: newer speech replaced it, a correction heard while it was
|
|
118
|
+
* being prepared (`supersedeVoiceTurn`) or a spoken correction sent while it
|
|
119
|
+
* still held the conversation (`correctsInFlightTurn`);
|
|
120
|
+
* - `bargeIn`: the person talked over its answer (the hands-free interrupt);
|
|
121
|
+
* - `sandboxReleased`: the page stopped the sandbox it ran on, a reaped or
|
|
122
|
+
* replaced one (`trackAssistantSandboxTurn`).
|
|
123
|
+
*/
|
|
124
|
+
export type TurnStopReason = 'stopped' | 'superseded' | 'bargeIn' | 'sandboxReleased';
|
|
125
|
+
/** …and `conversationDeleted`: its conversation was deleted while it ran. */
|
|
126
|
+
export type StoppedTurnOutcome = TurnStopReason | 'conversationDeleted';
|
|
127
|
+
/** One page's record of why each cancelled turn was cancelled (BOFF-12936). */
|
|
128
|
+
export interface TurnStopReasons {
|
|
129
|
+
/** Record why `turnId` was cancelled. The FIRST reason a turn is given stands. */
|
|
130
|
+
note(turnId: string, reason: TurnStopReason): void;
|
|
131
|
+
/**
|
|
132
|
+
* The outcome of a turn that ended without one of its own, forgetting the
|
|
133
|
+
* turn: the reason it was cancelled with; without one, `conversationDeleted`
|
|
134
|
+
* when its conversation is gone, and `stopped` otherwise.
|
|
135
|
+
*/
|
|
136
|
+
settle(turnId: string, conversationExists: boolean): StoppedTurnOutcome;
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* More reasons than turns a page can have in flight. Every noted turn is in
|
|
140
|
+
* flight and settles in its own `finally`; the bound only keeps a turn that
|
|
141
|
+
* never settles from growing the record.
|
|
142
|
+
*/
|
|
143
|
+
export declare const MAX_TURN_STOP_REASONS = 64;
|
|
144
|
+
export declare function createTurnStopReasons(): TurnStopReasons;
|
|
@@ -30,4 +30,21 @@ function n(e) {
|
|
|
30
30
|
reason: "superseded"
|
|
31
31
|
};
|
|
32
32
|
}
|
|
33
|
-
|
|
33
|
+
var r = 64;
|
|
34
|
+
function i() {
|
|
35
|
+
let e = /* @__PURE__ */ new Map();
|
|
36
|
+
return {
|
|
37
|
+
note(t, n) {
|
|
38
|
+
if (!e.has(t)) for (e.set(t, n); e.size > 64;) {
|
|
39
|
+
let t = e.keys().next();
|
|
40
|
+
if (t.done) break;
|
|
41
|
+
e.delete(t.value);
|
|
42
|
+
}
|
|
43
|
+
},
|
|
44
|
+
settle(t, n) {
|
|
45
|
+
let r = e.get(t);
|
|
46
|
+
return e.delete(t), r ?? (n ? "stopped" : "conversationDeleted");
|
|
47
|
+
}
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
export { r as MAX_TURN_STOP_REASONS, i as createTurnStopReasons, e as isTurnInFlight, t as planAbandonedTurn, n as planTurnAbort };
|
|
@@ -1,90 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
* ★★ Where a turn's time goes — measured before anything is optimised
|
|
3
|
-
* (BOFF-7801).
|
|
4
|
-
*
|
|
5
|
-
* First answers took 55–150s+ in production, and nobody could say which part
|
|
6
|
-
* of that was the sandbox starting, a dead sandbox being probed, the model
|
|
7
|
-
* thinking, the poll noticing, our own bookkeeping, or the voice fetching
|
|
8
|
-
* audio. This records each milestone of a turn as it happens, so the slow part
|
|
9
|
-
* is measured instead of guessed.
|
|
10
|
-
*
|
|
11
|
-
* The chain, in order:
|
|
12
|
-
*
|
|
13
|
-
* speechEnded the last time the heard words CHANGED (voice only)
|
|
14
|
-
* graceStarted the correction grace began — the last result that re-armed
|
|
15
|
-
* it; after speechEnded when the recogniser re-commits late
|
|
16
|
-
* graceEnded the correction grace ran out; the turn was submitted
|
|
17
|
-
* handedOver its attachments settled and it went to the panel (voice)
|
|
18
|
-
* accepted the panel took the turn: the question is on screen
|
|
19
|
-
* cleared the send-time checks passed (mode access, write consent)
|
|
20
|
-
* attemptStarted this attempt at a runtime began (again, on a retry)
|
|
21
|
-
* prewarmJoined the send is waiting on a first-keystroke pre-warm in its
|
|
22
|
-
* mode that is BOOTING a sandbox — marked as it joins one
|
|
23
|
-
* already booting, or as the warm it joined begins the boot
|
|
24
|
-
* bootAdopted this attempt found a boot under way and is waiting it
|
|
25
|
-
* out: a sandbox listed PROVISIONING/PENDING, or a service
|
|
26
|
-
* that answered "not up yet" (the `adopted` runtime)
|
|
27
|
-
* provisionStarted no reusable sandbox — making room for a new one starts here
|
|
28
|
-
* createRequested the new sandbox was requested (after releasing old ones)
|
|
29
|
-
* sandboxRunning the created sandbox is running (cold)
|
|
30
|
-
* serviceReady the assistant service inside it answers (cold)
|
|
31
|
-
* runtimeReady a sandbox and a session are ready to take the prompt
|
|
32
|
-
* promptSent the sandbox accepted the prompt
|
|
33
|
-
* lastQuietPoll the last poll that saw nothing — with firstOutput, the
|
|
34
|
-
* UPPER bound of how late polling noticed the output
|
|
35
|
-
* streamOpen the turn stream said `open` (BOFF-7914 S3); it can precede
|
|
36
|
-
* `promptSent`, because the mint runs beside the prompt
|
|
37
|
-
* streamRemint the stream was minted again after it ended; the FIRST
|
|
38
|
-
* re-mint (the reasons are the `remintReasons` of the stream)
|
|
39
|
-
* firstDelta the first live text of this turn that could be shown
|
|
40
|
-
* firstOutput the first poll — or live render — that had anything to
|
|
41
|
-
* show (a tool line too)
|
|
42
|
-
* firstText the first poll that saw answer TEXT — prose not followed
|
|
43
|
-
* by a tool call, confirmed when the turn completes, so
|
|
44
|
-
* pre-tool narration does not count. In modes whose
|
|
45
|
-
* narration IS the answer (BigConsole's build log), the
|
|
46
|
-
* first narration: group reports by `mode`
|
|
47
|
-
* streamIdle the bridge said the turn is idle (stream turns only)
|
|
48
|
-
* answerDone the poll that saw the answer complete
|
|
49
|
-
* complete the turn ended (after artifacts and the credit report)
|
|
50
|
-
* speechQueued the answer was handed to the speech feed (BOFF-8062 C1):
|
|
51
|
-
* a voice turn publishes it at `answerDone`, before the
|
|
52
|
-
* artifacts and credit report — so `ttsStarted` and
|
|
53
|
-
* `firstAudio` can precede `complete`
|
|
54
|
-
* ttsStarted auto-speak handed the reply to the speech engine
|
|
55
|
-
* firstAudio the engine reported sound actually started
|
|
56
|
-
* audioEnded the auto-spoken reply finished playing
|
|
57
|
-
*
|
|
58
|
-
* ★ `graceStarted → graceEnded` is the 2s correction grace, and it is KEPT: it
|
|
59
|
-
* stops a pause mid-sentence from submitting half a question. It is a segment
|
|
60
|
-
* of its own so it can never be "optimised" away by accident — and the
|
|
61
|
-
* recogniser's own endpointing (`speechEnded → graceStarted`, 0.5–1.5s and
|
|
62
|
-
* longer on Android) is kept OUT of it, because that one can be optimised.
|
|
63
|
-
*
|
|
64
|
-
* ★ Filler is never output: the progress `content` is model output only
|
|
65
|
-
* (BOFF-7277) and the waiting copy is drawn by the bubble from the stage.
|
|
66
|
-
*
|
|
67
|
-
* ★ `prewarmJoined` and `bootAdopted` are marked the moment the wait is KNOWN,
|
|
68
|
-
* not once it pays off: the facts (`prewarm`, `runtime`) are only written
|
|
69
|
-
* when the wait ends, so a turn the person gives up on mid-wait would
|
|
70
|
-
* otherwise carry nothing to say it was waiting on a boot (BOFF-7801 review
|
|
71
|
-
* round 3). Once the facts are written, they are the answer.
|
|
72
|
-
*
|
|
73
|
-
* ★ A turn whose first attempt fails recoverably is retried under the SAME id.
|
|
74
|
-
* `restartAttempt` files the first attempt's marks under `attempts` so the
|
|
75
|
-
* second records its own — mixed attempts are not a timeline.
|
|
76
|
-
*
|
|
77
|
-
* Times are `performance.now()` — monotonic, comparable within the page. The
|
|
78
|
-
* event is visible to EVERY script on the page, so it carries timings and
|
|
79
|
-
* coarse dimensions only: never message text, never tenant, user, workspace or
|
|
80
|
-
* conversation ids. Delivery is asynchronous, so a listener never runs inside
|
|
81
|
-
* the step it measures.
|
|
82
|
-
*
|
|
83
|
-
* ★ Anything that ACTS on a timeline (the production latency reporter,
|
|
84
|
-
* `turnTimingReporter.ts`) subscribes with `subscribeTurnTimelines`, not the
|
|
85
|
-
* window event: any script on the page can dispatch that event, so a listener
|
|
86
|
-
* on it can be fed invented timings. Only this module calls subscribers.
|
|
87
|
-
*/
|
|
1
|
+
import { StoppedTurnOutcome } from './turnState';
|
|
88
2
|
export declare const TURN_MILESTONES: readonly ["speechEnded", "lastSpeechFrame", "recognizerSpeechEnded", "graceStarted", "graceEnded", "handedOver", "accepted", "cleared", "attemptStarted", "prewarmJoined", "bootAdopted", "provisionStarted", "createRequested", "sandboxRunning", "serviceReady", "runtimeReady", "promptSent", "lastQuietPoll", "streamOpen", "streamRemint", "firstDelta", "firstOutput", "firstText", "streamIdle", "answerDone", "complete", "speechQueued", "ttsStarted", "firstAudio", "audioEnded"];
|
|
89
3
|
export type TurnMilestone = (typeof TURN_MILESTONES)[number];
|
|
90
4
|
/**
|
|
@@ -117,8 +31,14 @@ export type TurnRuntime = 'warm' | 'preopened' | 'new-session' | 'probed' | 'ado
|
|
|
117
31
|
* - `timedOut`: the client's stream budget ran out before the answer finished.
|
|
118
32
|
* Only `answered` turns carry `firstText`; a timed-out turn's unconfirmed text
|
|
119
33
|
* is not recorded.
|
|
34
|
+
*
|
|
35
|
+
* ★ A turn that ended without an answer because it was cancelled says why
|
|
36
|
+
* (BOFF-12936, `TurnStopReason`): `stopped` only when the person stopped it,
|
|
37
|
+
* else `superseded`, `bargeIn`, `conversationDeleted` or `sandboxReleased`.
|
|
38
|
+
* wspace-sandbox-svc counts these four from #178 on; a service without them
|
|
39
|
+
* folds them into `other`.
|
|
120
40
|
*/
|
|
121
|
-
export type TurnTimelineOutcome = 'answered' | 'timedOut' | 'refused' | 'failed' |
|
|
41
|
+
export type TurnTimelineOutcome = 'answered' | 'timedOut' | 'refused' | 'failed' | StoppedTurnOutcome;
|
|
122
42
|
export interface TurnDimensions {
|
|
123
43
|
channel: 'chat' | 'voice' | 'listen';
|
|
124
44
|
mode: string;
|