@bevyra/avatar 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/bevyra-core-Bskcl7l_.js +195 -0
- package/dist/assets/clientOnlyAvatarWorker-C6wTIDwp.js +2119 -0
- package/dist/assets/clientOnlySourceWorker-CNVdGzNy.js +7347 -0
- package/dist/assets/ort.wasm.min-BcBtKW3v.js +1400 -0
- package/dist/assets/ort.webgpu.min-BqBLsdhU.js +1856 -0
- package/dist/assets/rustAvatarWorker-vggdUXxI.js +1702 -0
- package/dist/assets/teacherCorrectorWorker-Deu0CCPs.js +1406 -0
- package/dist/assets/workerHelpers-BkWezgrb.js +31 -0
- package/dist/bevyra-avatar-sdk.js +3367 -0
- package/dist/ort/ort-wasm-simd-threaded.jsep.mjs +136 -0
- package/dist/ort/ort-wasm-simd-threaded.jsep.wasm +0 -0
- package/dist/ort/ort-wasm-simd-threaded.mjs +68 -0
- package/dist/ort/ort-wasm-simd-threaded.wasm +0 -0
- package/dist/runtime/bevyra-core.js +1515 -0
- package/dist/runtime/bevyra-core.wasm +0 -0
- package/dist/runtime/closure_router.onnx +0 -0
- package/dist/runtime/runtime-manifest.json +27 -0
- package/dist/sdk-manifest.json +70 -0
- package/index.d.ts +25 -0
- package/index.js +108 -0
- package/package.json +10 -4
- package/README.md +0 -3
|
@@ -0,0 +1,3367 @@
|
|
|
1
|
+
class _e {
|
|
2
|
+
constructor() {
|
|
3
|
+
this.ends = /* @__PURE__ */ new Map();
|
|
4
|
+
}
|
|
5
|
+
append(e, t) {
|
|
6
|
+
if (!Number.isInteger(t) || t < 0)
|
|
7
|
+
throw new Error("Rendered frame count must be a non-negative integer.");
|
|
8
|
+
const s = this.ends.get(e) ?? 0;
|
|
9
|
+
return this.ends.set(e, s + t), s;
|
|
10
|
+
}
|
|
11
|
+
/** Frames rendered so far on this timeline. */
|
|
12
|
+
end(e) {
|
|
13
|
+
return this.ends.get(e) ?? 0;
|
|
14
|
+
}
|
|
15
|
+
reset(e) {
|
|
16
|
+
this.ends.delete(e);
|
|
17
|
+
}
|
|
18
|
+
clear() {
|
|
19
|
+
this.ends.clear();
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
function we(l) {
|
|
23
|
+
for (let e = 0; e < l.length; e += 4) {
|
|
24
|
+
const t = l[e] / 255, s = l[e + 1] / 255, i = l[e + 2] / 255, r = Math.max(t, i), n = Math.max(0, s - r), a = n * (0.75 + Math.max(t, s, i)), c = Math.max(0, Math.min(1, (a - 0.025) / 0.32)), o = c * c * (3 - 2 * c), h = c * (1 - c) * 0.35, d = Math.max(0, Math.min(1, (n - 2e-3) / 0.14));
|
|
25
|
+
l[e + 3] = Math.round(255 * (1 - Math.min(1, o + h))), l[e + 1] = Math.round(255 * (s * (1 - d) + r * d));
|
|
26
|
+
}
|
|
27
|
+
return l;
|
|
28
|
+
}
|
|
29
|
+
let Q = new Uint8Array(0);
|
|
30
|
+
function ee(l, e, t) {
|
|
31
|
+
const s = e * t;
|
|
32
|
+
if (!s || l.length < s * 4) return l;
|
|
33
|
+
for (let i = 0; i + s * 4 <= l.length; i += s * 4) {
|
|
34
|
+
Q.length < s && (Q = new Uint8Array(s));
|
|
35
|
+
const r = Q;
|
|
36
|
+
for (let n = 0; n < s; n++) r[n] = l[i + n * 4 + 3];
|
|
37
|
+
for (let n = 0; n < t; n++) {
|
|
38
|
+
const a = n > 0 ? -e : 0, c = n < t - 1 ? e : 0;
|
|
39
|
+
for (let o = 0; o < e; o++) {
|
|
40
|
+
const h = n * e + o, d = r[h];
|
|
41
|
+
if (!d) continue;
|
|
42
|
+
const u = o > 0 ? -1 : 0, p = o < e - 1 ? 1 : 0, f = r[h + a], b = r[h + c], S = r[h + u], g = r[h + p], y = d !== f || d !== b || d !== S || d !== g, v = y || d === 255 && (r[n * e + Math.max(0, o - 2)] < 255 || r[n * e + Math.min(e - 1, o + 2)] < 255 || r[Math.max(0, n - 2) * e + o] < 255 || r[Math.min(t - 1, n + 2) * e + o] < 255);
|
|
43
|
+
if (!v && d === 255) continue;
|
|
44
|
+
const w = i + h * 4;
|
|
45
|
+
if (y) {
|
|
46
|
+
const A = 4 * d + 2 * (f + b + S + g) + r[h + a + u] + r[h + a + p] + r[h + c + u] + r[h + c + p] + 8 >> 4;
|
|
47
|
+
A < d && (l[w + 3] = A);
|
|
48
|
+
}
|
|
49
|
+
const m = Math.max(l[w], l[w + 2]) + (v ? 0 : 10);
|
|
50
|
+
l[w + 1] > m && (l[w + 1] = m);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return l;
|
|
55
|
+
}
|
|
56
|
+
const ye = "precision mediump float;uniform sampler2D frame;uniform vec2 texel;varying vec2 v;float key(vec3 c){float n=max(c.r,c.b);float d=max(0.,c.g-n);float s=d*(.75+max(c.r,max(c.g,c.b)));float a=clamp((s-.025)/.32,0.,1.);return 1.-clamp(a*a*(3.-2.*a)+a*(1.-a)*.35,0.,1.);}float at(float x,float y){return key(texture2D(frame,v+vec2(x,y)*texel).rgb);}void main(){vec4 c=texture2D(frame,v);float center=key(c.rgb);float m=(4.*center+2.*(at(1.,0.)+at(-1.,0.)+at(0.,1.)+at(0.,-1.))+at(1.,1.)+at(-1.,1.)+at(1.,-1.)+at(-1.,-1.))/16.;float a=texel.x>0.?smoothstep(.38,.82,m):center;float edge=m<.999?1.:0.;float neutral=mix(max(c.r,c.b),(c.r+c.b)*.5,edge);float spill=(a<.999||edge>0.)?1.:clamp((max(0.,c.g-neutral)-.002)/.14,0.,1.);c.g=min(c.g,mix(c.g,neutral,spill));c.a=a;gl_FragColor=c;}";
|
|
57
|
+
class be {
|
|
58
|
+
constructor(e) {
|
|
59
|
+
this.target = e, this.scale = 1, this.measuredAt = -1 / 0;
|
|
60
|
+
}
|
|
61
|
+
measure(e, t) {
|
|
62
|
+
const s = performance.now();
|
|
63
|
+
if (s - this.measuredAt > 500) {
|
|
64
|
+
this.measuredAt = s;
|
|
65
|
+
const i = window.devicePixelRatio || 1, r = Math.max(this.target.clientWidth * i / e, this.target.clientHeight * i / t);
|
|
66
|
+
this.scale = Number.isFinite(r) ? Math.min(3, Math.max(1, Math.ceil(r * 4) / 4)) : 1;
|
|
67
|
+
}
|
|
68
|
+
return { width: Math.round(e * this.scale), height: Math.round(t * this.scale) };
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
const xe = 12, ne = 6e3;
|
|
72
|
+
function Me(l) {
|
|
73
|
+
const e = new DataView(l);
|
|
74
|
+
if (l.byteLength < 44 || e.getUint32(0, !1) !== 1380533830 || e.getUint32(8, !1) !== 1463899717) return;
|
|
75
|
+
let t = 0, s = 0, i = 0, r = 0;
|
|
76
|
+
for (let n = 12; n + 8 <= l.byteLength; ) {
|
|
77
|
+
const a = e.getUint32(n + 4, !0), c = n + 8;
|
|
78
|
+
if (c + a > l.byteLength) break;
|
|
79
|
+
const o = e.getUint32(n, !1);
|
|
80
|
+
if (o === 1718449184 && a >= 16 && (i = e.getUint16(c, !0), t = e.getUint16(c + 2, !0), s = e.getUint32(c + 4, !0), r = e.getUint16(c + 14, !0)), o === 1684108385 && i === 1 && t === 1 && s === 16e3 && r === 16) {
|
|
81
|
+
const h = new Float32Array(a / 2);
|
|
82
|
+
for (let d = 0; d < h.length; d++) h[d] = e.getInt16(c + d * 2, !0) / 32768;
|
|
83
|
+
return h;
|
|
84
|
+
}
|
|
85
|
+
n = c + a + (a & 1);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
function ae(l) {
|
|
89
|
+
const e = new Uint8Array(l.buffer, l.byteOffset, l.byteLength);
|
|
90
|
+
let t = 2166136261;
|
|
91
|
+
for (let s = 0; s < e.length; s++)
|
|
92
|
+
t ^= e[s], t = Math.imul(t, 16777619);
|
|
93
|
+
return (t >>> 0).toString(16).padStart(8, "0") + ":" + e.length;
|
|
94
|
+
}
|
|
95
|
+
const Pe = 25;
|
|
96
|
+
class Fe extends Error {
|
|
97
|
+
constructor(e, t) {
|
|
98
|
+
super(`On-device avatar throughput ${e.toFixed(1)} FPS is below the required ${t.toFixed(1)} FPS.`), this.measuredFps = e, this.requiredFps = t, this.code = "AVATAR_DEVICE_TOO_SLOW", this.name = "AvatarDevicePerformanceError";
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
class oe {
|
|
102
|
+
constructor(e, t, s, i) {
|
|
103
|
+
this.source = e, this.width = t, this.height = s, this.fps = i, this.worker = new Worker(
|
|
104
|
+
new URL(
|
|
105
|
+
/* @vite-ignore */
|
|
106
|
+
"/assets/clientOnlySourceWorker-CNVdGzNy.js",
|
|
107
|
+
import.meta.url
|
|
108
|
+
),
|
|
109
|
+
{ type: "module", name: "bevyra-client-source" }
|
|
110
|
+
), this.id = 0, this.pending = /* @__PURE__ */ new Map(), this.worker.onmessage = ({ data: r }) => {
|
|
111
|
+
const n = this.pending.get(r.id);
|
|
112
|
+
n && (this.pending.delete(r.id), r.error ? n.reject(new Error(r.error)) : n.resolve(r.result));
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
call(e, t, s = []) {
|
|
116
|
+
const i = ++this.id;
|
|
117
|
+
return new Promise((r, n) => {
|
|
118
|
+
this.pending.set(i, { resolve: r, reject: n }), this.worker.postMessage({ id: i, type: e, ...t }, s);
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
async ready() {
|
|
122
|
+
if (typeof VideoDecoder > "u")
|
|
123
|
+
throw new Error("Client-only avatars require WebCodecs.");
|
|
124
|
+
const e = this.source;
|
|
125
|
+
(await this.call("init", { source: e, width: this.width, height: this.height, fps: this.fps }, [e])).fallback && console.info("[BeVyra SDK] hardware source decode unavailable; using software WebCodecs");
|
|
126
|
+
}
|
|
127
|
+
async frames(e, t = !0) {
|
|
128
|
+
const s = e.slice();
|
|
129
|
+
return this.call("frames", { indices: s.buffer, retain: t }, [s.buffer]);
|
|
130
|
+
}
|
|
131
|
+
close() {
|
|
132
|
+
this.worker.postMessage({ id: ++this.id, type: "close" }), this.worker.terminate(), this.pending.forEach((e) => e.reject(new Error("Source decoder closed."))), this.pending.clear();
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
class j {
|
|
136
|
+
constructor(e, t, s, i, r, n, a = "reference", c = !1) {
|
|
137
|
+
this.canvas = e, this.width = t, this.height = s, this.fps = i, this.onStage = r, this.mode = n, this.workerKind = a, this.diagnostics = c, this.runtimeId = crypto.randomUUID().slice(0, 8), this.sourceFrameTrace = [], this.canvasWrites = [], this.id = 0, this.generation = 0, this.pending = /* @__PURE__ */ new Map(), this.stopped = !1, this.presented = [], this.displayProbe = 0, this.inferenceStride = 1, this.silenceStart = Number.POSITIVE_INFINITY, this.sourceFrameCount = 0, this.fullPlanActive = !1, this.liveBatch = 4, this.visualChain = Promise.resolve(), this.presentationChain = Promise.resolve(), this.playbackOrigins = /* @__PURE__ */ new Map(), this.renderedTimeline = new _e(), this.audioNodes = /* @__PURE__ */ new Set(), this.pendingPlayback = [], this.nextAudioTime = 0, this.audioStarted = !1, this.preparedAudioSeconds = 0, this.scheduledAudioSeconds = 0, this.currentTurnId = 0, this.pendingPresentation = [], this.audioQueue = [], this.queuedChunks = 0, this.idleCursor = 0, this.idleHandoffPending = !1, this.idleTransitionRemaining = 0, this.lastWasIdle = !1, this.speechTransitionRemaining = 0, this.settleTimer = 0, this.assistantAudioActive = !1, this.resetVisualOnNextAssistantAudio = !1, this.resetCoreOnNextAssistantAudio = !1, this.idleEpoch = 0, this.idleRendering = !1, this.idleHolds = 0, this.idleLookahead = 2, this.idleIterationDone = Promise.resolve(), this.avatarCanvas = document.createElement("canvas"), this.body = "full", this.speechStartTimer = 0, this.speechAudible = !1, this.turnClauses = 0, this.unannouncedClauses = [], this.worker = a === "rust-v2" ? new Worker(new URL(
|
|
138
|
+
/* @vite-ignore */
|
|
139
|
+
"/assets/rustAvatarWorker-vggdUXxI.js",
|
|
140
|
+
import.meta.url
|
|
141
|
+
), {
|
|
142
|
+
type: "module",
|
|
143
|
+
name: "bevyra-rust-avatar"
|
|
144
|
+
}) : new Worker(new URL(
|
|
145
|
+
/* @vite-ignore */
|
|
146
|
+
"/assets/clientOnlyAvatarWorker-C6wTIDwp.js",
|
|
147
|
+
import.meta.url
|
|
148
|
+
), {
|
|
149
|
+
type: "module",
|
|
150
|
+
name: "bevyra-client-avatar-reference"
|
|
151
|
+
}), n === "stream" && this.measureDisplayLead();
|
|
152
|
+
const o = crossOriginIsolated && typeof SharedArrayBuffer < "u", h = navigator.hardwareConcurrency || 1, d = o ? h >= 8 ? 4 : h >= 6 ? 3 : 2 : 1;
|
|
153
|
+
this.metrics = {
|
|
154
|
+
requestedThreads: d,
|
|
155
|
+
effectiveThreads: 1,
|
|
156
|
+
crossOriginIsolated: o,
|
|
157
|
+
hardwareConcurrency: h,
|
|
158
|
+
ortBackend: "wasm",
|
|
159
|
+
audioEncoderMs: null,
|
|
160
|
+
predictorMs: null,
|
|
161
|
+
gaussianMs: null,
|
|
162
|
+
inferenceFps: 0,
|
|
163
|
+
inferenceCompletedAt: 0,
|
|
164
|
+
sourceDecodeMs: null,
|
|
165
|
+
inferenceQueueDepth: 0,
|
|
166
|
+
droppedFrames: 0,
|
|
167
|
+
presentedFps: 0,
|
|
168
|
+
sourceDecoder: "webcodecs",
|
|
169
|
+
calibrationFps: null,
|
|
170
|
+
avDriftMs: 0,
|
|
171
|
+
audioUnderruns: 0,
|
|
172
|
+
lateFrames: 0,
|
|
173
|
+
maxFrameLateMs: 0,
|
|
174
|
+
displayLeadMs: Pe
|
|
175
|
+
}, this.outputWidth = t, this.outputHeight = s, this.avatarCanvas.width = t, this.avatarCanvas.height = s;
|
|
176
|
+
const u = this.avatarCanvas.getContext("2d", { alpha: !0 });
|
|
177
|
+
if (!u)
|
|
178
|
+
throw new Error("Avatar composition canvas is unavailable.");
|
|
179
|
+
this.avatarContext = u, e.width = this.outputWidth, e.height = this.outputHeight, this.worker.onmessage = (p) => {
|
|
180
|
+
if (p.data.stage) {
|
|
181
|
+
this.onStage(p.data.stage);
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
const f = this.pending.get(p.data.id);
|
|
185
|
+
f && (this.pending.delete(p.data.id), p.data.error ? f.reject(new Error(p.data.error)) : f.resolve(p.data.result));
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
static async create(e, t, s, i = {}) {
|
|
189
|
+
const r = e.manifests["portable-manifest.json"], n = r.portableRuntime ?? r, a = Number(r.width), c = Number(r.height), o = Number(r.fps || 25);
|
|
190
|
+
if (!a || !c || o !== 25)
|
|
191
|
+
throw new Error("Portable avatar dimensions or FPS are invalid.");
|
|
192
|
+
const h = new j(
|
|
193
|
+
t,
|
|
194
|
+
a,
|
|
195
|
+
c,
|
|
196
|
+
o,
|
|
197
|
+
s,
|
|
198
|
+
i.mode ?? "stream",
|
|
199
|
+
"reference",
|
|
200
|
+
!!i.diagnostics
|
|
201
|
+
);
|
|
202
|
+
await h.setScene(i.background, i.resolution, i.body), h.inferenceStride = Number(n.inferenceStride), s("initializing_worker"), s("initializing_wasm"), s("initializing_ort");
|
|
203
|
+
const d = await h.call("bootstrap", {
|
|
204
|
+
wasmBytes: e.files.get("runtime/avatar-wasm.wasm"),
|
|
205
|
+
threads: h.metrics.requestedThreads,
|
|
206
|
+
ortWasmPaths: i.ortWasmPaths ?? new URL("/ort/", location.origin).href
|
|
207
|
+
});
|
|
208
|
+
h.metrics.effectiveThreads = d.effectiveThreads, h.decoder = new oe(
|
|
209
|
+
d.sourceWebm,
|
|
210
|
+
a,
|
|
211
|
+
c,
|
|
212
|
+
o
|
|
213
|
+
), await h.decoder.ready();
|
|
214
|
+
const u = await h.decoder.frames(Uint32Array.of(0));
|
|
215
|
+
return h.idleFrame = h.keyIdleFrame(new Uint8Array(u)), await h.call("avatarInit", {
|
|
216
|
+
width: a,
|
|
217
|
+
height: c,
|
|
218
|
+
frameZero: u,
|
|
219
|
+
clauseBoundaryFrames: Math.max(25, Number(n.clauseBoundaryFrames || 25))
|
|
220
|
+
}), h.mode === "stream" && h.startIdleLoop(), h;
|
|
221
|
+
}
|
|
222
|
+
static async createRustV2(e, t, s, i, r = {}) {
|
|
223
|
+
const n = e.manifests["portable-manifest.json"], a = n.portableRuntime, c = Number(n.width), o = Number(n.height), h = Number(n.fps || 25), d = Number(a?.embeddingShape?.[2]);
|
|
224
|
+
if (a?.format !== "portable-ort-v2" || !c || !o || h !== 25 || !d)
|
|
225
|
+
throw new Error("Rust avatar bundle contract is invalid.");
|
|
226
|
+
const u = new j(
|
|
227
|
+
s,
|
|
228
|
+
c,
|
|
229
|
+
o,
|
|
230
|
+
h,
|
|
231
|
+
i,
|
|
232
|
+
r.mode ?? "stream",
|
|
233
|
+
"rust-v2",
|
|
234
|
+
!!r.diagnostics
|
|
235
|
+
);
|
|
236
|
+
await u.setScene(r.background, r.resolution, r.body), u.inferenceStride = Number(a.inferenceStride || 1), u.sourceFrameCount = Number(n.frameCount);
|
|
237
|
+
const p = e.files.get("p12.bin");
|
|
238
|
+
if (!p || p.byteLength < 20)
|
|
239
|
+
throw new Error("Rust avatar bundle has no silence range");
|
|
240
|
+
if (u.silenceStart = new DataView(p).getUint32(16, !0), !Number.isInteger(u.sourceFrameCount) || u.silenceStart >= u.sourceFrameCount)
|
|
241
|
+
throw new Error("Rust avatar bundle has an invalid silence range");
|
|
242
|
+
u.visualPlannerDescriptor = e.files.get("p24.bin"), i("initializing_worker"), i("initializing_wasm"), i("initializing_ort");
|
|
243
|
+
const f = e.files.get("p0.bin"), b = e.files.get("p23.bin") ?? e.files.get("p1.bin"), S = e.files.get("p9.bin"), g = e.files.get("p10.bin"), y = e.files.get("p5.bin"), v = t.coreWasm, w = t.closureRouter, m = e.files.get("p22.bin"), A = r.planner === "segments" ? "segments" : r.planner === "legacy" ? "legacy" : m ? "trajectory" : "legacy", E = await u.call("bootstrap", {
|
|
244
|
+
audioFusion: f,
|
|
245
|
+
corrector: b,
|
|
246
|
+
teacherCorrector: e.files.has("p23.bin"),
|
|
247
|
+
closureRouter: w,
|
|
248
|
+
audioEncoder: S,
|
|
249
|
+
motionPlanner: g,
|
|
250
|
+
source: y,
|
|
251
|
+
coreWasm: v,
|
|
252
|
+
featureDim: d,
|
|
253
|
+
useLearnedPlanner: r.planner !== "segments",
|
|
254
|
+
plannerMode: A,
|
|
255
|
+
preferWebGpu: r.executionBackend !== "wasm" && "gpu" in navigator && window.isSecureContext,
|
|
256
|
+
allowWasmFallback: window.crossOriginIsolated && navigator.hardwareConcurrency >= 4,
|
|
257
|
+
threads: u.metrics.requestedThreads,
|
|
258
|
+
ortWasmPaths: r.ortWasmPaths ?? new URL("/ort/", location.origin).href
|
|
259
|
+
}, [
|
|
260
|
+
f,
|
|
261
|
+
b,
|
|
262
|
+
w,
|
|
263
|
+
S,
|
|
264
|
+
g,
|
|
265
|
+
y,
|
|
266
|
+
v
|
|
267
|
+
]);
|
|
268
|
+
u.metrics.effectiveThreads = E.effectiveThreads, u.metrics.ortBackend = E.backend, E.fallbackReason && console.warn("[BeVyra SDK] WebGPU unavailable; using threaded WASM:", E.fallbackReason), console.info(`[BeVyra SDK] inference backend=${E.backend}`), u.mode === "stream" && console.info(`[BeVyra SDK] PLANNER_ROUTE mode=${A} scope=clause asset=${m ? "p22.bin" : "p11.bin"}`);
|
|
269
|
+
const k = e.files.has("p23.bin") ? "p23.bin" : "p1.bin", _ = k === "p23.bin" ? "teacher" : "student";
|
|
270
|
+
if (E.correctorRole !== _)
|
|
271
|
+
throw new Error("Corrector role disagrees with the verified bundle asset");
|
|
272
|
+
console.info(
|
|
273
|
+
`[BeVyra SDK] CORRECTOR role=${_} asset=${k} sha256=${e.contract.files[k]?.sha256?.slice(0, 16) ?? "unavailable"} backend=${E.backend} workers=${E.correctorWorkers}`
|
|
274
|
+
), u.decoder = new oe(E.source, c, o, h), await u.decoder.ready();
|
|
275
|
+
const P = await u.decoder.frames(Uint32Array.of(0)), x = await new Response(new Blob([JSON.stringify({
|
|
276
|
+
frameCount: n.frameCount,
|
|
277
|
+
fps: h,
|
|
278
|
+
width: c,
|
|
279
|
+
height: o,
|
|
280
|
+
geometry: n.geometry,
|
|
281
|
+
maskSpec: n.maskSpec ?? e.manifests["manifest.json"]?.maskSpec
|
|
282
|
+
})]).stream().pipeThrough(new CompressionStream("deflate"))).arrayBuffer(), R = e.files.get("p2.bin"), D = e.files.get("p4.bin"), L = e.files.get("p21.bin"), O = e.files.get("p11.bin"), M = e.files.get("p12.bin");
|
|
283
|
+
u.idleFrame = u.keyIdleFrame(new Uint8Array(P)), await u.call("avatarInit", {
|
|
284
|
+
descriptor: x,
|
|
285
|
+
gaussians: R,
|
|
286
|
+
landmarks: D,
|
|
287
|
+
faceMattes: L,
|
|
288
|
+
plannerMetadata: O,
|
|
289
|
+
trajectoryMetadata: m,
|
|
290
|
+
segments: M,
|
|
291
|
+
frameZero: P,
|
|
292
|
+
width: u.width,
|
|
293
|
+
height: u.height
|
|
294
|
+
}, [
|
|
295
|
+
x,
|
|
296
|
+
R,
|
|
297
|
+
D,
|
|
298
|
+
L,
|
|
299
|
+
O,
|
|
300
|
+
...m ? [m] : [],
|
|
301
|
+
M,
|
|
302
|
+
P
|
|
303
|
+
]), u.idleCursor = u.silenceStart, console.info(`[BeVyra SDK] IDLE_SOURCE start=${u.silenceStart} frames=${u.sourceFrameCount - u.silenceStart} duration_s=${((u.sourceFrameCount - u.silenceStart) / h).toFixed(2)}`);
|
|
304
|
+
const B = r.minimumCalibrationFps === !1 ? 0 : Number(r.minimumCalibrationFps || 25);
|
|
305
|
+
if (B > 0) {
|
|
306
|
+
i("calibrating_device");
|
|
307
|
+
try {
|
|
308
|
+
await u.calibrate(B);
|
|
309
|
+
} catch (F) {
|
|
310
|
+
throw u.close(), F;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
return u.mode === "stream" && u.startIdleLoop(), u;
|
|
314
|
+
}
|
|
315
|
+
async calibrationBatch() {
|
|
316
|
+
const e = this.liveBatch, t = await this.call("plan", { clauseId: 0, start: 0, count: e }), s = new Uint32Array(t.sources), i = Uint32Array.from(Array.from(new Set(Array.from(s))).sort((n, a) => n - a)), r = await this.decoder.frames(i);
|
|
317
|
+
return await this.call("render", {
|
|
318
|
+
generation: this.generation,
|
|
319
|
+
clauseId: 0,
|
|
320
|
+
start: 0,
|
|
321
|
+
count: e,
|
|
322
|
+
sources: s,
|
|
323
|
+
indices: i,
|
|
324
|
+
decodedBgr: r,
|
|
325
|
+
inferenceStride: this.inferenceStride
|
|
326
|
+
}, [r]), e;
|
|
327
|
+
}
|
|
328
|
+
async calibrate(e) {
|
|
329
|
+
for (let n = 0; n < 3; n++) await this.calibrationBatch();
|
|
330
|
+
const t = [];
|
|
331
|
+
for (let n = 0; n < 10; n++) {
|
|
332
|
+
const a = performance.now(), c = await this.calibrationBatch();
|
|
333
|
+
t.push(c * 1e3 / Math.max(1, performance.now() - a));
|
|
334
|
+
}
|
|
335
|
+
const s = t.slice(2).sort((n, a) => n - a), i = Math.floor(s.length / 2), r = s.length % 2 ? s[i] : (s[i - 1] + s[i]) / 2;
|
|
336
|
+
if (this.metrics.calibrationFps = r, console.info(
|
|
337
|
+
`[BeVyra SDK ${this.runtimeId}] CALIBRATION throughput=${r.toFixed(1)}fps required=${e.toFixed(1)}fps backend=${this.metrics.ortBackend} threads=${this.metrics.effectiveThreads}`
|
|
338
|
+
), r < e)
|
|
339
|
+
throw new Fe(r, e);
|
|
340
|
+
}
|
|
341
|
+
call(e, t, s = []) {
|
|
342
|
+
const i = ++this.id;
|
|
343
|
+
return new Promise((r, n) => {
|
|
344
|
+
this.pending.set(i, { resolve: r, reject: n }), this.worker.postMessage({ id: i, type: e, ...t }, s);
|
|
345
|
+
});
|
|
346
|
+
}
|
|
347
|
+
/** Diagnostics-only: enable or collect Rust worker stage fingerprints. */
|
|
348
|
+
setStageTrace(e) {
|
|
349
|
+
return this.call("trace", { enabled: e });
|
|
350
|
+
}
|
|
351
|
+
takeStageTrace() {
|
|
352
|
+
return this.call("takeTrace", {});
|
|
353
|
+
}
|
|
354
|
+
async enableAudio() {
|
|
355
|
+
if (this.audio ??= new AudioContext({ sampleRate: 24e3 }), this.recordingDestination ??= this.audio.createMediaStreamDestination(), await this.audio.resume(), this.audio.state !== "running")
|
|
356
|
+
throw new Error("Audio playback permission required.");
|
|
357
|
+
}
|
|
358
|
+
/**
|
|
359
|
+
* Reports when each clause of the current turn starts playing:
|
|
360
|
+
* `index` counts clauses within the turn (0, 1, ...), `atMs` is a
|
|
361
|
+
* performance.now() time.
|
|
362
|
+
*/
|
|
363
|
+
onClauseStart(e) {
|
|
364
|
+
return this.clauseStartListener = e, () => {
|
|
365
|
+
this.clauseStartListener === e && (this.clauseStartListener = void 0);
|
|
366
|
+
};
|
|
367
|
+
}
|
|
368
|
+
announceClauses() {
|
|
369
|
+
const e = this.unannouncedClauses;
|
|
370
|
+
this.unannouncedClauses = [];
|
|
371
|
+
for (const t of e) {
|
|
372
|
+
const s = this.playbackOrigins.get(t.key);
|
|
373
|
+
s === void 0 ? this.unannouncedClauses.push(t) : this.clauseStartListener?.(t.index, s + t.frame * (1e3 / this.fps));
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
onSpeechStart(e) {
|
|
377
|
+
return this.speechStartListener = e, () => {
|
|
378
|
+
this.speechStartListener === e && (this.speechStartListener = void 0);
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
/** Fires when the avatar stops being audible: turn played out or interrupted. */
|
|
382
|
+
onSpeechEnd(e) {
|
|
383
|
+
return this.speechEndListener = e, () => {
|
|
384
|
+
this.speechEndListener === e && (this.speechEndListener = void 0);
|
|
385
|
+
};
|
|
386
|
+
}
|
|
387
|
+
endSpeech() {
|
|
388
|
+
this.speechAudible && (this.speechAudible = !1, this.speechEndListener?.());
|
|
389
|
+
}
|
|
390
|
+
/**
|
|
391
|
+
* A frame painted into the canvas reaches the screen at the next refresh,
|
|
392
|
+
* plus about one more for the compositor: on average 1.5 refresh intervals
|
|
393
|
+
* after the paint. Audio is timed to when it is heard, so without this the
|
|
394
|
+
* voice leads the lips by that much (about 25 ms at 60 Hz). Measure the
|
|
395
|
+
* refresh interval and paint that much earlier.
|
|
396
|
+
*/
|
|
397
|
+
measureDisplayLead() {
|
|
398
|
+
if (typeof requestAnimationFrame != "function") return;
|
|
399
|
+
const e = [];
|
|
400
|
+
let t = 0;
|
|
401
|
+
const s = (i) => {
|
|
402
|
+
if (this.stopped) return;
|
|
403
|
+
if (t && e.push(i - t), t = i, e.length < 30) {
|
|
404
|
+
this.displayProbe = requestAnimationFrame(s);
|
|
405
|
+
return;
|
|
406
|
+
}
|
|
407
|
+
const r = [...e].sort((a, c) => a - c), n = r[Math.floor(r.length / 2)];
|
|
408
|
+
n >= 4 && n <= 50 && (this.metrics.displayLeadMs = Math.round(n * 1.5)), console.info(`[BeVyra SDK ${this.runtimeId}] DISPLAY_LEAD refresh=${n.toFixed(1)}ms lead=${this.metrics.displayLeadMs}ms`);
|
|
409
|
+
};
|
|
410
|
+
this.displayProbe = requestAnimationFrame(s);
|
|
411
|
+
}
|
|
412
|
+
scheduleAudio(e, t = !1) {
|
|
413
|
+
const s = this.audio;
|
|
414
|
+
if (!s) throw new Error("Avatar audio output is not initialized.");
|
|
415
|
+
const i = s.createBuffer(1, e.pcm24.length, e.sampleRate), r = i.getChannelData(0);
|
|
416
|
+
for (let p = 0; p < e.pcm24.length; p++)
|
|
417
|
+
r[p] = e.pcm24[p] / 32768;
|
|
418
|
+
const n = s.createBufferSource(), a = Math.max(
|
|
419
|
+
s.currentTime + (t ? 0.12 : 0.03),
|
|
420
|
+
this.nextAudioTime
|
|
421
|
+
);
|
|
422
|
+
(!t && this.nextAudioTime > 0 ? a - this.nextAudioTime : 0) > 1e-3 && this.metrics.audioUnderruns++, this.beforeFirstAudio?.(), this.beforeFirstAudio = void 0, n.buffer = i, this.mode === "stream" && n.connect(s.destination), this.recordingDestination && n.connect(this.recordingDestination), this.nextAudioTime = a + i.duration;
|
|
423
|
+
const o = (s.outputLatency || 0) + (s.baseLatency || 0);
|
|
424
|
+
if (e.playbackStartMs = performance.now() + (a - s.currentTime + o) * 1e3, t) {
|
|
425
|
+
this.speechStartTimer && window.clearTimeout(this.speechStartTimer);
|
|
426
|
+
const p = this.generation;
|
|
427
|
+
this.speechStartTimer = window.setTimeout(() => {
|
|
428
|
+
this.speechStartTimer = 0, !(this.stopped || p !== this.generation) && (this.speechAudible || (this.speechAudible = !0, this.speechStartListener?.()));
|
|
429
|
+
}, Math.max(0, e.playbackStartMs - performance.now()));
|
|
430
|
+
}
|
|
431
|
+
const h = `${e.responseGenerationId}:${e.turnId}`, d = e.playbackStartMs - e.startSample / e.sampleRate * 1e3, u = this.playbackOrigins.get(h);
|
|
432
|
+
(u === void 0 || d > u + 1) && (this.playbackOrigins.set(h, d), u !== void 0 && console.info(
|
|
433
|
+
`[BeVyra SDK ${this.runtimeId}] MEDIA_CLOCK_REBASE generation=${e.responseGenerationId} turn=${e.turnId} delta_ms=${Math.round(d - u)}`
|
|
434
|
+
)), u === void 0 && this.announceClauses(), this.audioNodes.add(n), n.onended = () => {
|
|
435
|
+
this.audioNodes.delete(n), n.disconnect();
|
|
436
|
+
}, n.start(a);
|
|
437
|
+
}
|
|
438
|
+
releasePreparedAudio(e = !1) {
|
|
439
|
+
const t = xe / this.fps;
|
|
440
|
+
if (!(!this.audioStarted && !e && this.preparedAudioSeconds + 1 / this.fps < t)) {
|
|
441
|
+
for (; this.pendingPlayback.length; ) {
|
|
442
|
+
const s = this.pendingPlayback[0], i = s.pcm24.length / s.sampleRate;
|
|
443
|
+
if (this.scheduledAudioSeconds + i > this.preparedAudioSeconds + 1 / this.fps)
|
|
444
|
+
break;
|
|
445
|
+
this.pendingPlayback.shift(), this.scheduleAudio(s, !this.audioStarted), this.audioStarted = !0, this.scheduledAudioSeconds += i;
|
|
446
|
+
}
|
|
447
|
+
this.audioStarted && this.pendingPresentation.length && this.pendingPresentation.splice(0).forEach((i) => this.schedulePresentation(i));
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
pushAssistantAudio(e) {
|
|
451
|
+
if (this.lastWasIdle && this.lastPresentedFrame && (this.speechTransitionSource = this.lastPresentedFrame.slice(), this.speechTransitionRemaining = 4), this.idleHandoffPending = !1, this.idleTransitionSource = void 0, this.idleTransitionRemaining = 0, this.resetVisualOnNextAssistantAudio) {
|
|
452
|
+
if (this.resetVisualOnNextAssistantAudio = !1, this.idleEpoch++, this.idleFrame) {
|
|
453
|
+
this.avatarContext.putImageData(this.idleFrame, 0, 0);
|
|
454
|
+
const s = this.presentationContext();
|
|
455
|
+
s && this.compose(s), this.diagnostics && this.canvasWrites.push({
|
|
456
|
+
writer: "post_barge_neutral",
|
|
457
|
+
generation: this.generation,
|
|
458
|
+
turnId: e.turnId,
|
|
459
|
+
frame: ae(this.idleFrame.data),
|
|
460
|
+
at: performance.now()
|
|
461
|
+
});
|
|
462
|
+
}
|
|
463
|
+
console.info(
|
|
464
|
+
`[BeVyra SDK ${this.runtimeId}] POST_BARGE_NEUTRAL generation=${this.generation} turn=${e.turnId} idle_epoch=${this.idleEpoch}`
|
|
465
|
+
);
|
|
466
|
+
}
|
|
467
|
+
this.assistantAudioActive = !0, this.idleEpoch++, this.currentTurnId = e.turnId;
|
|
468
|
+
const t = new Promise(
|
|
469
|
+
(s, i) => this.audioQueue.push({ chunk: e, resolve: s, reject: i })
|
|
470
|
+
);
|
|
471
|
+
return this.queuedChunks = this.audioQueue.length, this.metrics.inferenceQueueDepth = this.queuedChunks, this.ensureAudioPump(), t;
|
|
472
|
+
}
|
|
473
|
+
ensureAudioPump() {
|
|
474
|
+
this.pumpPromise || this.stopped || !this.audioQueue.length || (this.pumpPromise = this.pumpAudioQueue().finally(() => {
|
|
475
|
+
this.pumpPromise = void 0, this.audioQueue.length && !this.stopped && this.ensureAudioPump();
|
|
476
|
+
}));
|
|
477
|
+
}
|
|
478
|
+
startIdleLoop() {
|
|
479
|
+
this.idleTask || (this.idleTask = this.runIdleLoop().catch((e) => {
|
|
480
|
+
this.stopped || console.error("[BeVyra SDK] continuous silence loop failed", e);
|
|
481
|
+
}).finally(() => {
|
|
482
|
+
this.idleTask = void 0;
|
|
483
|
+
}));
|
|
484
|
+
}
|
|
485
|
+
async runIdleLoop() {
|
|
486
|
+
if (this.workerKind === "rust-v2") {
|
|
487
|
+
await this.runRustIdleLoop();
|
|
488
|
+
return;
|
|
489
|
+
}
|
|
490
|
+
const e = 1e3 / this.fps;
|
|
491
|
+
let t = 0, s = this.idleEpoch;
|
|
492
|
+
const i = [];
|
|
493
|
+
for (; !this.stopped; ) {
|
|
494
|
+
if (s !== this.idleEpoch && (s = this.idleEpoch, t = 0), this.assistantAudioActive || this.idleHolds > 0 || this.audioQueue.length || this.pendingPresentation.length) {
|
|
495
|
+
t = 0, await new Promise((u) => setTimeout(u, 20));
|
|
496
|
+
continue;
|
|
497
|
+
}
|
|
498
|
+
const r = this.idleEpoch, n = this.liveBatch;
|
|
499
|
+
let a, c;
|
|
500
|
+
this.idleIterationDone = new Promise((u) => c = u), this.idleRendering = !0;
|
|
501
|
+
try {
|
|
502
|
+
const u = await this.call("plan", {
|
|
503
|
+
clauseId: 0,
|
|
504
|
+
start: 0,
|
|
505
|
+
count: n
|
|
506
|
+
});
|
|
507
|
+
if (this.stopped || r !== this.idleEpoch) continue;
|
|
508
|
+
const p = new Uint32Array(u.sources), f = Uint32Array.from(
|
|
509
|
+
Array.from(new Set(Array.from(p))).sort((g, y) => g - y)
|
|
510
|
+
), b = await this.decoder.frames(f);
|
|
511
|
+
if (this.diagnostics && this.sourceFrameTrace.push(...Array.from(p)), this.stopped || r !== this.idleEpoch) continue;
|
|
512
|
+
const S = await this.call("render", {
|
|
513
|
+
generation: this.generation,
|
|
514
|
+
clauseId: 0,
|
|
515
|
+
start: 0,
|
|
516
|
+
count: n,
|
|
517
|
+
sources: p,
|
|
518
|
+
indices: f,
|
|
519
|
+
decodedBgr: b,
|
|
520
|
+
inferenceStride: this.inferenceStride
|
|
521
|
+
}, [b]);
|
|
522
|
+
if (this.stopped || r !== this.idleEpoch || S.discarded) continue;
|
|
523
|
+
a = S.rgba instanceof Uint8Array ? S.rgba : new Uint8Array(S.rgba);
|
|
524
|
+
} finally {
|
|
525
|
+
this.idleRendering = !1, c();
|
|
526
|
+
}
|
|
527
|
+
if (!a) {
|
|
528
|
+
t = 0;
|
|
529
|
+
continue;
|
|
530
|
+
}
|
|
531
|
+
const o = Math.max(t, performance.now());
|
|
532
|
+
t = o + n * e;
|
|
533
|
+
const h = {
|
|
534
|
+
sessionId: "idle",
|
|
535
|
+
turnId: -1,
|
|
536
|
+
responseGenerationId: this.generation,
|
|
537
|
+
startSample: 0,
|
|
538
|
+
endSample: 0,
|
|
539
|
+
sampleRate: 24e3,
|
|
540
|
+
pcm24: new Int16Array(0),
|
|
541
|
+
playbackStartMs: o
|
|
542
|
+
}, d = a;
|
|
543
|
+
for (i.push(
|
|
544
|
+
(i.at(-1) ?? Promise.resolve()).then(() => this.present(d, n, h, r)).catch((u) => {
|
|
545
|
+
this.stopped || console.error("[BeVyra SDK] silence presentation failed", u);
|
|
546
|
+
})
|
|
547
|
+
); i.length > this.idleLookahead; ) await i.shift();
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
async nextRustIdleFrame() {
|
|
551
|
+
const e = this.idleCursor, t = new Uint8Array(await this.decoder.frames(Uint32Array.of(e), !1)), s = this.keyIdleFrame(t), i = s.data;
|
|
552
|
+
if (this.idleFrame = s, this.idleCursor = e + 1 < this.sourceFrameCount ? e + 1 : this.silenceStart, this.idleCursor === this.silenceStart && console.info(`[BeVyra SDK ${this.runtimeId}] IDLE_LOOP_WRAP frame=${e}`), !this.idleTransitionRemaining || !this.idleTransitionSource) return i;
|
|
553
|
+
const r = 5 - this.idleTransitionRemaining, n = new Uint8ClampedArray(i.length);
|
|
554
|
+
for (let a = 0; a < i.length; a++)
|
|
555
|
+
n[a] = Math.round(
|
|
556
|
+
(this.idleTransitionSource[a] * (4 - r) + i[a] * r) / 4
|
|
557
|
+
);
|
|
558
|
+
return this.idleTransitionRemaining--, this.idleTransitionRemaining || (this.idleTransitionSource = void 0), n;
|
|
559
|
+
}
|
|
560
|
+
async runRustIdleLoop() {
|
|
561
|
+
const e = 1e3 / this.fps;
|
|
562
|
+
let t = performance.now();
|
|
563
|
+
for (; !this.stopped; ) {
|
|
564
|
+
if (this.assistantAudioActive || this.idleHolds > 0 || this.audioQueue.length || this.pendingPresentation.length) {
|
|
565
|
+
t = performance.now(), await new Promise((a) => setTimeout(a, 20));
|
|
566
|
+
continue;
|
|
567
|
+
}
|
|
568
|
+
this.idleHandoffPending && this.lastPresentedFrame && (this.idleTransitionSource = this.lastPresentedFrame.slice(), this.idleTransitionRemaining = 4, this.idleHandoffPending = !1, console.info(
|
|
569
|
+
`[BeVyra SDK ${this.runtimeId}] IDLE_HANDOFF entry=${this.idleCursor} bridge_frames=4`
|
|
570
|
+
));
|
|
571
|
+
const s = this.idleEpoch, i = await this.nextRustIdleFrame();
|
|
572
|
+
if (this.stopped || s !== this.idleEpoch) continue;
|
|
573
|
+
const r = performance.now(), n = {
|
|
574
|
+
sessionId: "idle",
|
|
575
|
+
turnId: -1,
|
|
576
|
+
responseGenerationId: this.generation,
|
|
577
|
+
startSample: 0,
|
|
578
|
+
endSample: 0,
|
|
579
|
+
sampleRate: 24e3,
|
|
580
|
+
pcm24: new Int16Array(0),
|
|
581
|
+
playbackStartMs: Math.max(t, r)
|
|
582
|
+
};
|
|
583
|
+
await this.present(new Uint8Array(i.buffer), 1, n, s), t = n.playbackStartMs + e;
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
/**
|
|
587
|
+
* Waits for an in-flight idle render without polling. Polling idleRendering
|
|
588
|
+
* cannot observe the instant between back-to-back idle iterations, so a
|
|
589
|
+
* waiter could starve until speech paused the idle loop.
|
|
590
|
+
*/
|
|
591
|
+
async withIdleHeld(e) {
|
|
592
|
+
this.idleHolds++;
|
|
593
|
+
try {
|
|
594
|
+
for (; this.idleRendering && !this.stopped; ) await this.idleIterationDone;
|
|
595
|
+
return await e();
|
|
596
|
+
} finally {
|
|
597
|
+
this.idleHolds--;
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
async pumpAudioQueue() {
|
|
601
|
+
for (await new Promise((e) => setTimeout(e, 20)); this.audioQueue.length && !this.stopped; ) {
|
|
602
|
+
const e = this.audioQueue.shift(), t = e.chunk;
|
|
603
|
+
this.queuedChunks = this.audioQueue.length, this.metrics.inferenceQueueDepth = this.queuedChunks;
|
|
604
|
+
try {
|
|
605
|
+
await this.withIdleHeld(async () => {
|
|
606
|
+
if (this.resetCoreOnNextAssistantAudio) {
|
|
607
|
+
this.resetCoreOnNextAssistantAudio = !1;
|
|
608
|
+
const s = performance.now();
|
|
609
|
+
if (console.info(
|
|
610
|
+
`[BeVyra SDK ${this.runtimeId}] POST_BARGE_CORE_RESET_BEGIN generation=${this.generation} turn=${t.turnId}`
|
|
611
|
+
), await this.call("interrupt", { generation: this.generation }), console.info(
|
|
612
|
+
`[BeVyra SDK ${this.runtimeId}] POST_BARGE_CORE_RESET_END generation=${this.generation} turn=${t.turnId} elapsed_ms=${(performance.now() - s).toFixed(1)}`
|
|
613
|
+
), t.responseGenerationId !== this.generation || this.stopped)
|
|
614
|
+
return;
|
|
615
|
+
}
|
|
616
|
+
await this.processAudio(t, !!t.completeClause);
|
|
617
|
+
}), e.resolve();
|
|
618
|
+
} catch (s) {
|
|
619
|
+
const i = s instanceof Error ? s : new Error(String(s));
|
|
620
|
+
e.reject(i);
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
async processAudio(e, t) {
|
|
625
|
+
if (e.responseGenerationId !== this.generation || this.stopped) return;
|
|
626
|
+
const s = performance.now(), i = `${e.responseGenerationId}:${e.turnId}`;
|
|
627
|
+
e.playbackStartMs !== void 0 && !this.playbackOrigins.has(i) && this.playbackOrigins.set(
|
|
628
|
+
i,
|
|
629
|
+
e.playbackStartMs - e.startSample / e.sampleRate * 1e3
|
|
630
|
+
);
|
|
631
|
+
const r = await this.call("audio", {
|
|
632
|
+
generation: this.generation,
|
|
633
|
+
pcm24: e.pcm24.buffer,
|
|
634
|
+
sampleRate: e.sampleRate,
|
|
635
|
+
preserveSilence: this.mode === "generate",
|
|
636
|
+
flush: t
|
|
637
|
+
});
|
|
638
|
+
if (r.discarded || !r.frames) return;
|
|
639
|
+
this.metrics.audioEncoderMs = r.audioEncoderMs;
|
|
640
|
+
const n = new Float32Array(r.pcm16);
|
|
641
|
+
let a = 0, c = 0, o = 0, h = 0, d = 0;
|
|
642
|
+
const u = {};
|
|
643
|
+
let p = 0;
|
|
644
|
+
const f = [], b = /* @__PURE__ */ new Set(), S = this.width * this.height * 4, g = (m) => {
|
|
645
|
+
this.diagnostics && (this.lastRenderedRgba = m.rgba), this.pendingPlayback.push(m.chunk), this.pendingPresentation.push(m), this.preparedAudioSeconds += m.chunk.pcm24.length / m.chunk.sampleRate, this.releasePreparedAudio();
|
|
646
|
+
}, y = (m) => {
|
|
647
|
+
for (const A of f) {
|
|
648
|
+
const E = m - A.startFrame;
|
|
649
|
+
if (E >= 0 && E < A.frames)
|
|
650
|
+
return { source: A.sources[E], pixels: A.rgba.subarray(E * S, (E + 1) * S) };
|
|
651
|
+
}
|
|
652
|
+
}, v = async (m) => {
|
|
653
|
+
if (!this.fullPlanActive) {
|
|
654
|
+
for (; f.length; ) g(f.shift());
|
|
655
|
+
return;
|
|
656
|
+
}
|
|
657
|
+
if (f.length < 3 && !m) return;
|
|
658
|
+
const A = f[0]?.startFrame ?? 0, E = f.at(-1)?.startFrame + (f.at(-1)?.frames ?? 0);
|
|
659
|
+
for (let k = A + 1; k < E; k++) {
|
|
660
|
+
if (b.has(k)) continue;
|
|
661
|
+
const _ = y(k - 1), P = y(k);
|
|
662
|
+
if (!_ || !P) continue;
|
|
663
|
+
const x = _.source < this.silenceStart && P.source >= this.silenceStart, R = _.source >= this.silenceStart && P.source < this.silenceStart;
|
|
664
|
+
if (!x && !R) continue;
|
|
665
|
+
const D = x ? k - 1 : k - 4, L = x ? k + 3 : k, O = y(D), M = y(L);
|
|
666
|
+
if (!O || !M) continue;
|
|
667
|
+
const B = performance.now();
|
|
668
|
+
for (let F = 0; F < 3; F++) {
|
|
669
|
+
const C = y(x ? k + F : k - 3 + F);
|
|
670
|
+
if (!C) break;
|
|
671
|
+
const N = (F + 1) / 4, U = await this.call("tween", {
|
|
672
|
+
left: O.pixels.slice(),
|
|
673
|
+
right: M.pixels.slice(),
|
|
674
|
+
fraction: N
|
|
675
|
+
});
|
|
676
|
+
C.pixels.set(U.rgba instanceof Uint8Array ? U.rgba : new Uint8Array(U.rgba));
|
|
677
|
+
}
|
|
678
|
+
u.tween_ms = (u.tween_ms || 0) + performance.now() - B, b.add(k);
|
|
679
|
+
}
|
|
680
|
+
for (; f.length > (m ? 0 : 2); ) g(f.shift());
|
|
681
|
+
};
|
|
682
|
+
try {
|
|
683
|
+
for (let m = 0; m < r.frames; m += this.liveBatch) {
|
|
684
|
+
if (e.responseGenerationId !== this.generation || this.stopped) return;
|
|
685
|
+
for (; ; ) {
|
|
686
|
+
const C = this.playbackOrigins.get(i);
|
|
687
|
+
if (C === void 0) break;
|
|
688
|
+
const N = C + this.renderedTimeline.end(i) * (1e3 / this.fps) - performance.now();
|
|
689
|
+
if (N <= ne) break;
|
|
690
|
+
const U = performance.now();
|
|
691
|
+
if (await new Promise((Re) => setTimeout(Re, N - ne)), p += performance.now() - U, e.responseGenerationId !== this.generation || this.stopped) return;
|
|
692
|
+
}
|
|
693
|
+
const A = Math.min(this.liveBatch, r.frames - m), E = await this.call("plan", {
|
|
694
|
+
clauseId: r.clauseId,
|
|
695
|
+
start: m,
|
|
696
|
+
count: A
|
|
697
|
+
}), k = new Uint32Array(E.sources), _ = Uint32Array.from(
|
|
698
|
+
Array.from(new Set(Array.from(k))).sort((C, N) => C - N)
|
|
699
|
+
), P = performance.now(), x = await this.decoder.frames(_);
|
|
700
|
+
this.diagnostics && this.sourceFrameTrace.push(...Array.from(k)), o += performance.now() - P;
|
|
701
|
+
const R = await this.call("render", {
|
|
702
|
+
generation: this.generation,
|
|
703
|
+
clauseId: r.clauseId,
|
|
704
|
+
start: m,
|
|
705
|
+
count: A,
|
|
706
|
+
sources: k,
|
|
707
|
+
indices: _,
|
|
708
|
+
decodedBgr: x,
|
|
709
|
+
inferenceStride: this.inferenceStride
|
|
710
|
+
}, [x]);
|
|
711
|
+
if (R.discarded || e.responseGenerationId !== this.generation) return;
|
|
712
|
+
a += Number(R.predictorMs), c += Number(R.gaussianMs), h += Number(R.profile?.core?.gaussian_ms || 0), d += Number(R.profile?.rgba_pack_ms || 0);
|
|
713
|
+
for (const C of ["preprocess_ms", "corrector_prepare_ms", "corrector_infer_ms", "corrector_paste_ms", "static_cache_hits", "static_cache_misses"])
|
|
714
|
+
u[C] = (u[C] || 0) + Number(R.profile?.core?.[C] || 0);
|
|
715
|
+
for (const C of ["output_copy_ms", "matte_refine_ms"])
|
|
716
|
+
u[C] = (u[C] || 0) + Number(R.profile?.[C] || 0);
|
|
717
|
+
u.wasm_memory_bytes = Math.max(
|
|
718
|
+
u.wasm_memory_bytes || 0,
|
|
719
|
+
Number(R.profile?.wasm_memory_bytes || 0)
|
|
720
|
+
);
|
|
721
|
+
const D = m * 640, L = n.subarray(D, D + A * 640), O = new Int16Array(L.length), M = this.renderedTimeline.append(i, A);
|
|
722
|
+
m === 0 && e.sessionId !== "flush" && (this.unannouncedClauses.push({ index: this.turnClauses++, key: i, frame: M }), this.announceClauses());
|
|
723
|
+
for (let C = 0; C < L.length; C++)
|
|
724
|
+
O[C] = Math.max(-32768, Math.min(32767, Math.round(L[C] * 32768)));
|
|
725
|
+
const B = {
|
|
726
|
+
...e,
|
|
727
|
+
sampleRate: 16e3,
|
|
728
|
+
startSample: M * 640,
|
|
729
|
+
endSample: (M + A) * 640,
|
|
730
|
+
pcm24: O
|
|
731
|
+
}, F = {
|
|
732
|
+
rgba: R.rgba instanceof Uint8Array ? R.rgba : new Uint8Array(R.rgba),
|
|
733
|
+
frames: R.frames,
|
|
734
|
+
chunk: B,
|
|
735
|
+
startFrame: M,
|
|
736
|
+
sources: k
|
|
737
|
+
};
|
|
738
|
+
f.push(F), await v(!1), await Promise.resolve();
|
|
739
|
+
}
|
|
740
|
+
await v(!0);
|
|
741
|
+
} finally {
|
|
742
|
+
await this.call("releaseClause", { clauseId: r.clauseId }).catch(() => {
|
|
743
|
+
});
|
|
744
|
+
}
|
|
745
|
+
this.metrics.sourceDecodeMs = o, this.metrics.predictorMs = a, this.metrics.gaussianMs = c;
|
|
746
|
+
const w = performance.now() - s;
|
|
747
|
+
this.diagnostics && console.info("[BeVyra stage timings]", JSON.stringify({ frames: r.frames, elapsedMs: w, throttleMs: p, activeMs: w - p, encoderMs: r.audioEncoderMs, sourceMs: o, predictorMs: a, gaussianMs: c, gaussianCoreMs: h, rgbaPackMs: d, ...u })), this.metrics.inferenceFps = r.frames * 1e3 / Math.max(1, w), this.metrics.inferenceCompletedAt = performance.now(), console.info(
|
|
748
|
+
`[BeVyra SDK ${this.runtimeId}] READY frames=${r.frames} throughput=${this.metrics.inferenceFps.toFixed(1)}fps encoder=${Number(r.audioEncoderMs).toFixed(1)}ms source=${o.toFixed(1)}ms predictor=${a.toFixed(1)}ms gaussian=${c.toFixed(1)}ms splat=${h.toFixed(1)}ms rgba=${d.toFixed(1)}ms depth=${this.queuedChunks} batch=${this.liveBatch}`
|
|
749
|
+
);
|
|
750
|
+
}
|
|
751
|
+
schedulePresentation(e) {
|
|
752
|
+
const t = `${e.chunk.responseGenerationId}:${e.chunk.turnId}`, s = this.playbackOrigins.get(t);
|
|
753
|
+
if (s === void 0) return;
|
|
754
|
+
const i = {
|
|
755
|
+
...e.chunk,
|
|
756
|
+
playbackStartMs: s + e.startFrame * (1e3 / this.fps)
|
|
757
|
+
};
|
|
758
|
+
e.chunk.playbackStartMs !== void 0 && (this.metrics.avDriftMs = Math.max(this.metrics.avDriftMs, Math.abs(i.playbackStartMs - e.chunk.playbackStartMs))), this.presentationChain = this.presentationChain.then(
|
|
759
|
+
() => this.present(e.rgba, e.frames, i)
|
|
760
|
+
);
|
|
761
|
+
}
|
|
762
|
+
async present(e, t, s, i) {
|
|
763
|
+
const r = this.presentationContext();
|
|
764
|
+
if (!r) throw new Error("Canvas frame presentation failed.");
|
|
765
|
+
const n = this.width * this.height * 4, a = 1e3 / this.fps, c = s.playbackStartMs ?? performance.now();
|
|
766
|
+
let o = 0;
|
|
767
|
+
if (i === void 0) {
|
|
768
|
+
const h = c - this.metrics.displayLeadMs, d = performance.now() - h;
|
|
769
|
+
d > 80 && (o = Math.min(t, Math.ceil((d - 80) / a)), this.metrics.lateFrames += o, this.metrics.maxFrameLateMs = Math.max(this.metrics.maxFrameLateMs, d), console.info(
|
|
770
|
+
`[BeVyra SDK ${this.runtimeId}] BATCH_RECOVERY generation=${s.responseGenerationId} turn=${s.turnId} dropped=${o}/${t} late_ms=${Math.round(d)}`
|
|
771
|
+
));
|
|
772
|
+
}
|
|
773
|
+
for (let h = o; h < t; h++) {
|
|
774
|
+
if (this.stopped || s.responseGenerationId !== this.generation || i !== void 0 && i !== this.idleEpoch)
|
|
775
|
+
return;
|
|
776
|
+
const d = c + h * a - (i === void 0 ? this.metrics.displayLeadMs : 0);
|
|
777
|
+
let u = d - performance.now();
|
|
778
|
+
if (u > 1) {
|
|
779
|
+
if (await new Promise((b) => setTimeout(b, u)), this.stopped || s.responseGenerationId !== this.generation || i !== void 0 && i !== this.idleEpoch)
|
|
780
|
+
return;
|
|
781
|
+
u = d - performance.now();
|
|
782
|
+
}
|
|
783
|
+
let p = new Uint8ClampedArray(
|
|
784
|
+
e.buffer,
|
|
785
|
+
e.byteOffset + h * n,
|
|
786
|
+
n
|
|
787
|
+
);
|
|
788
|
+
if (i === void 0 && this.speechTransitionRemaining && this.speechTransitionSource) {
|
|
789
|
+
const b = 5 - this.speechTransitionRemaining, S = p.slice();
|
|
790
|
+
for (let g = 0; g < n; g++)
|
|
791
|
+
S[g] = Math.round((this.speechTransitionSource[g] * (4 - b) + p[g] * b) / 4);
|
|
792
|
+
p = S, this.speechTransitionRemaining--, this.speechTransitionRemaining || (this.speechTransitionSource = void 0);
|
|
793
|
+
}
|
|
794
|
+
this.lastPresentedFrame = p.slice(), this.lastWasIdle = i !== void 0, this.avatarContext.putImageData(
|
|
795
|
+
new ImageData(p, this.width, this.height),
|
|
796
|
+
0,
|
|
797
|
+
0
|
|
798
|
+
), this.compose(r), this.diagnostics && this.canvasWrites.push({
|
|
799
|
+
writer: i === void 0 ? "speech" : "idle",
|
|
800
|
+
generation: s.responseGenerationId,
|
|
801
|
+
turnId: s.turnId,
|
|
802
|
+
frame: ae(p),
|
|
803
|
+
at: performance.now()
|
|
804
|
+
});
|
|
805
|
+
const f = performance.now();
|
|
806
|
+
for (this.presented.push(f); this.presented.length && this.presented[0] < f - 3e3; )
|
|
807
|
+
this.presented.shift();
|
|
808
|
+
this.presented.length > 1 && (this.metrics.presentedFps = Math.min(
|
|
809
|
+
this.fps,
|
|
810
|
+
(this.presented.length - 1) * 1e3 / (this.presented[this.presented.length - 1] - this.presented[0])
|
|
811
|
+
));
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
compose(e) {
|
|
815
|
+
const t = this.outputWidth, s = this.outputHeight;
|
|
816
|
+
if (e.save(), e.clearRect(0, 0, t, s), (this.background || this.mode === "generate") && (e.fillStyle = "#202126", e.fillRect(0, 0, t, s)), this.background) {
|
|
817
|
+
const i = this.background, r = Number(i.videoWidth || i.naturalWidth || i.width), n = Number(i.videoHeight || i.naturalHeight || i.height), a = Math.max(t / r, s / n), c = r * a, o = n * a;
|
|
818
|
+
e.drawImage(i, (t - c) / 2, (s - o) / 2, c, o);
|
|
819
|
+
}
|
|
820
|
+
if (this.body === "half") {
|
|
821
|
+
const i = Math.floor(this.height / 2), r = s > t ? Math.round(s * 1e3 / 1280) : s, n = this.halfBodySpan ?? this.measureHalfBodySpan(i), a = Math.min(r / i, t / (n * 2)), c = this.width * a, o = i * a;
|
|
822
|
+
e.drawImage(
|
|
823
|
+
this.avatarCanvas,
|
|
824
|
+
0,
|
|
825
|
+
0,
|
|
826
|
+
this.width,
|
|
827
|
+
i,
|
|
828
|
+
(t - c) / 2,
|
|
829
|
+
s - o,
|
|
830
|
+
c,
|
|
831
|
+
o
|
|
832
|
+
);
|
|
833
|
+
} else {
|
|
834
|
+
const i = Math.min(t / this.width, s / this.height), r = this.width * i, n = this.height * i;
|
|
835
|
+
e.drawImage(
|
|
836
|
+
this.avatarCanvas,
|
|
837
|
+
(t - r) / 2,
|
|
838
|
+
(s - n) / 2,
|
|
839
|
+
r,
|
|
840
|
+
n
|
|
841
|
+
);
|
|
842
|
+
}
|
|
843
|
+
e.restore();
|
|
844
|
+
}
|
|
845
|
+
/**
|
|
846
|
+
* Half-width, from the frame centre, of the keyed subject inside the
|
|
847
|
+
* half-body crop, plus margin for gestures. Measured once from the first
|
|
848
|
+
* frame that has a subject; the full frame half-width until then.
|
|
849
|
+
*/
|
|
850
|
+
measureHalfBodySpan(e) {
|
|
851
|
+
const t = this.avatarContext.getImageData(0, 0, this.width, e).data, s = this.width / 2;
|
|
852
|
+
let i = 0;
|
|
853
|
+
for (let r = 0; r < e; r++)
|
|
854
|
+
for (let n = 0; n < this.width; n++)
|
|
855
|
+
t[(r * this.width + n) * 4 + 3] > 128 && (i = Math.max(i, Math.abs(n + 0.5 - s)));
|
|
856
|
+
return i ? (this.halfBodySpan = Math.min(s, i + this.width * 0.03), this.halfBodySpan) : s;
|
|
857
|
+
}
|
|
858
|
+
presentationContext() {
|
|
859
|
+
return this.canvas.getContext("2d", { alpha: !0 });
|
|
860
|
+
}
|
|
861
|
+
async setScene(e, t = "source", s = this.body) {
|
|
862
|
+
if (s !== "full" && s !== "half")
|
|
863
|
+
throw new Error("Body must be full or half.");
|
|
864
|
+
if (this.body = s, t === "source")
|
|
865
|
+
this.outputWidth = this.width, this.outputHeight = this.height;
|
|
866
|
+
else if (t === "portrait")
|
|
867
|
+
this.outputWidth = 720, this.outputHeight = 1280;
|
|
868
|
+
else if (t === "landscape")
|
|
869
|
+
this.outputWidth = 1280, this.outputHeight = 720;
|
|
870
|
+
else if (t === "square")
|
|
871
|
+
this.outputWidth = 720, this.outputHeight = 720;
|
|
872
|
+
else {
|
|
873
|
+
const r = Math.round(t.width), n = Math.round(t.height);
|
|
874
|
+
if (!Number.isSafeInteger(r) || !Number.isSafeInteger(n) || r < 16 || n < 16 || r > 4096 || n > 4096)
|
|
875
|
+
throw new Error("Output resolution must be integer dimensions between 16 and 4096 pixels.");
|
|
876
|
+
this.outputWidth = r, this.outputHeight = n;
|
|
877
|
+
}
|
|
878
|
+
if (this.canvas.width = this.outputWidth, this.canvas.height = this.outputHeight, this.ownedBackground?.close(), this.ownedBackground = void 0, this.background = void 0, e)
|
|
879
|
+
if (typeof e == "string") {
|
|
880
|
+
const r = await fetch(e, { credentials: "same-origin" });
|
|
881
|
+
if (!r.ok)
|
|
882
|
+
throw new Error(`Background download failed (${r.status}).`);
|
|
883
|
+
this.ownedBackground = await createImageBitmap(await r.blob()), this.background = this.ownedBackground;
|
|
884
|
+
} else e instanceof Blob ? (this.ownedBackground = await createImageBitmap(e), this.background = this.ownedBackground) : this.background = e;
|
|
885
|
+
const i = this.presentationContext();
|
|
886
|
+
i && this.compose(i);
|
|
887
|
+
}
|
|
888
|
+
async generate(e, t) {
|
|
889
|
+
if (this.mode !== "generate")
|
|
890
|
+
throw new Error("This runtime was not initialized for finite generation.");
|
|
891
|
+
if (typeof MediaRecorder > "u" || !this.canvas.captureStream)
|
|
892
|
+
throw new Error("This browser cannot encode generated avatar video.");
|
|
893
|
+
await this.enableAudio();
|
|
894
|
+
const s = e instanceof AudioBuffer ? e : await this.audio.decodeAudioData(
|
|
895
|
+
(e instanceof Blob ? await e.arrayBuffer() : e).slice(0)
|
|
896
|
+
), i = new Float32Array(s.length);
|
|
897
|
+
for (let g = 0; g < s.numberOfChannels; g++) {
|
|
898
|
+
const y = s.getChannelData(g);
|
|
899
|
+
for (let v = 0; v < i.length; v++)
|
|
900
|
+
i[v] += y[v] / s.numberOfChannels;
|
|
901
|
+
}
|
|
902
|
+
if (this.workerKind === "rust-v2" && this.visualPlannerDescriptor) {
|
|
903
|
+
const g = e instanceof Blob ? await e.arrayBuffer() : e instanceof ArrayBuffer ? e : void 0;
|
|
904
|
+
let y = g ? Me(g) : void 0;
|
|
905
|
+
if (!y) {
|
|
906
|
+
const w = Math.ceil(i.length * 16e3 / s.sampleRate), m = new OfflineAudioContext(1, w, 16e3), A = m.createBuffer(1, i.length, s.sampleRate);
|
|
907
|
+
A.copyToChannel(i, 0);
|
|
908
|
+
const E = m.createBufferSource();
|
|
909
|
+
E.buffer = A, E.connect(m.destination), E.start(), y = (await m.startRendering()).getChannelData(0).slice();
|
|
910
|
+
}
|
|
911
|
+
const v = Math.max(0, Math.floor(Math.floor((y.length - 80) / 320) / 2));
|
|
912
|
+
if (v > 0) {
|
|
913
|
+
const w = await this.call("fullPlan", {
|
|
914
|
+
wave: y.buffer,
|
|
915
|
+
visualDescriptors: this.visualPlannerDescriptor,
|
|
916
|
+
limit: v
|
|
917
|
+
});
|
|
918
|
+
this.diagnostics && console.info("[BeVyra SDK] FULL_AUDIO_PLAN", {
|
|
919
|
+
frames: new Uint32Array(w.sources).length,
|
|
920
|
+
decisions: w.decisions.length
|
|
921
|
+
}), this.fullPlanActive = !0;
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
this.workerKind === "rust-v2" && console.info(`[BeVyra SDK] PLANNER_ROUTE mode=${this.fullPlanActive ? "python-full-audio-v1" : "clause-fallback"} scope=${this.fullPlanActive ? "full-audio" : "clause"} asset=${this.fullPlanActive ? "p24.bin" : "none"}`);
|
|
925
|
+
const r = new Int16Array(i.length);
|
|
926
|
+
for (let g = 0; g < i.length; g++)
|
|
927
|
+
r[g] = Math.max(-32768, Math.min(32767, Math.round(i[g] * 32768)));
|
|
928
|
+
const n = t || "video/webm;codecs=vp9,opus", a = MediaRecorder.isTypeSupported(n) ? n : MediaRecorder.isTypeSupported("video/webm;codecs=vp8,opus") ? "video/webm;codecs=vp8,opus" : "video/webm", c = this.canvas.captureStream(this.fps);
|
|
929
|
+
this.recordingDestination.stream.getAudioTracks().forEach((g) => c.addTrack(g));
|
|
930
|
+
const o = [], h = new MediaRecorder(c, { mimeType: a });
|
|
931
|
+
h.ondataavailable = (g) => {
|
|
932
|
+
g.data.size && o.push(g.data);
|
|
933
|
+
};
|
|
934
|
+
const d = new Promise((g, y) => {
|
|
935
|
+
h.onstop = () => g(), h.onerror = () => y(new Error("Browser video encoder failed."));
|
|
936
|
+
});
|
|
937
|
+
let u = 0;
|
|
938
|
+
this.beforeFirstAudio = () => {
|
|
939
|
+
u = performance.now(), h.start(250);
|
|
940
|
+
};
|
|
941
|
+
const p = this.generation, f = ++this.currentTurnId, b = Math.round(s.sampleRate * 3);
|
|
942
|
+
for (let g = 0; g < r.length; g += b) {
|
|
943
|
+
const y = Math.min(r.length, g + b);
|
|
944
|
+
await this.pushAssistantAudio({
|
|
945
|
+
sessionId: "sdk-generate",
|
|
946
|
+
turnId: f,
|
|
947
|
+
responseGenerationId: p,
|
|
948
|
+
startSample: g,
|
|
949
|
+
endSample: y,
|
|
950
|
+
sampleRate: s.sampleRate,
|
|
951
|
+
pcm24: r.slice(g, y),
|
|
952
|
+
completeClause: !0
|
|
953
|
+
});
|
|
954
|
+
}
|
|
955
|
+
if (await this.completeTurn(), h.state === "inactive")
|
|
956
|
+
throw this.beforeFirstAudio = void 0, c.getTracks().forEach((g) => g.stop()), new Error("Input audio produced no renderable speech frames.");
|
|
957
|
+
const S = s.duration * 1e3 - (performance.now() - u);
|
|
958
|
+
return S > 0 && await new Promise((g) => setTimeout(g, S)), h.stop(), await d, c.getTracks().forEach((g) => g.stop()), {
|
|
959
|
+
video: new Blob(o, { type: a }),
|
|
960
|
+
mimeType: a,
|
|
961
|
+
width: this.outputWidth,
|
|
962
|
+
height: this.outputHeight,
|
|
963
|
+
durationSeconds: s.duration
|
|
964
|
+
};
|
|
965
|
+
}
|
|
966
|
+
keyIdleFrame(e) {
|
|
967
|
+
const t = new Uint8ClampedArray(this.width * this.height * 4);
|
|
968
|
+
for (let s = 0, i = 0; s < e.length; s += 3, i += 4) {
|
|
969
|
+
const r = e[s] / 255, n = e[s + 1] / 255, a = e[s + 2] / 255, c = Math.max(a, r), o = Math.max(0, n - c), h = o * (0.75 + Math.max(a, n, r)), d = Math.max(0, Math.min(1, (h - 0.025) / 0.32)), u = d * d * (3 - 2 * d), p = d * (1 - d) * 0.35, f = Math.max(0, Math.min(1, (o - 2e-3) / 0.14));
|
|
970
|
+
t[i] = a * 255, t[i + 1] = 255 * (n * (1 - f) + c * f), t[i + 2] = r * 255, t[i + 3] = 255 * (1 - Math.min(1, u + p));
|
|
971
|
+
}
|
|
972
|
+
return new ImageData(ee(t, this.width, this.height), this.width, this.height);
|
|
973
|
+
}
|
|
974
|
+
completeTurn(e = !0, t = this.generation) {
|
|
975
|
+
if (t !== this.generation)
|
|
976
|
+
return this.diagnostics && console.info(
|
|
977
|
+
`[BeVyra SDK ${this.runtimeId}] TURN_COMPLETE_IGNORED requested_generation=${t} active_generation=${this.generation}`
|
|
978
|
+
), Promise.resolve();
|
|
979
|
+
const s = this.generation, i = new Int16Array(0), r = this.currentTurnId;
|
|
980
|
+
this.diagnostics && console.info(
|
|
981
|
+
`[BeVyra SDK ${this.runtimeId}] TURN_COMPLETE_REQUESTED generation=${s} turn=${r} idle_rendering=${this.idleRendering} pump=${!!this.pumpPromise}`
|
|
982
|
+
);
|
|
983
|
+
const n = {
|
|
984
|
+
sessionId: "flush",
|
|
985
|
+
turnId: r,
|
|
986
|
+
responseGenerationId: s,
|
|
987
|
+
startSample: 0,
|
|
988
|
+
endSample: 0,
|
|
989
|
+
sampleRate: 24e3,
|
|
990
|
+
pcm24: i
|
|
991
|
+
}, a = this.pumpPromise ?? Promise.resolve();
|
|
992
|
+
return this.visualChain = a.then(
|
|
993
|
+
() => this.withIdleHeld(
|
|
994
|
+
() => e ? this.processAudio(n, !0) : Promise.resolve()
|
|
995
|
+
)
|
|
996
|
+
).then(() => this.releasePreparedAudio(!0)).then(() => this.presentationChain).then(async () => {
|
|
997
|
+
if (this.stopped || s !== this.generation || !this.idleFrame)
|
|
998
|
+
return;
|
|
999
|
+
this.preparedAudioSeconds = 0, this.scheduledAudioSeconds = 0, this.audioStarted = !1;
|
|
1000
|
+
const c = this.audio ? performance.now() + Math.max(0, this.nextAudioTime - this.audio.currentTime) * 1e3 : performance.now();
|
|
1001
|
+
await new Promise((h) => {
|
|
1002
|
+
this.settleResolve = h, this.settleTimer = window.setTimeout(() => {
|
|
1003
|
+
this.settleResolve = void 0, h();
|
|
1004
|
+
}, Math.max(0, c - performance.now() + 80));
|
|
1005
|
+
}), !this.stopped && s === this.generation && this.idleFrame && (this.diagnostics && console.info(
|
|
1006
|
+
`[BeVyra SDK ${this.runtimeId}] TURN_SETTLED captured_generation=${s} captured_turn=${r} active_turn=${this.currentTurnId} assistant_audio_active=${this.assistantAudioActive}->false`
|
|
1007
|
+
), this.assistantAudioActive = !1, this.idleHandoffPending = !0, this.endSpeech());
|
|
1008
|
+
const o = `${s}:${r}`;
|
|
1009
|
+
this.playbackOrigins.delete(o), this.renderedTimeline.reset(o), this.turnClauses = 0, this.unannouncedClauses = [];
|
|
1010
|
+
}).catch(() => {
|
|
1011
|
+
}), this.visualChain;
|
|
1012
|
+
}
|
|
1013
|
+
interrupt() {
|
|
1014
|
+
const e = this.generation, t = this.audioQueue.length, s = this.pendingPresentation.length, i = this.audioNodes.size;
|
|
1015
|
+
this.speechStartTimer && window.clearTimeout(this.speechStartTimer), this.speechStartTimer = 0, this.settleTimer && window.clearTimeout(this.settleTimer), this.settleResolve?.(), this.settleResolve = void 0, this.generation++, this.idleEpoch++, this.resetVisualOnNextAssistantAudio = !0, this.resetCoreOnNextAssistantAudio = !0, this.fullPlanActive = !1, this.assistantAudioActive = !1, this.idleHandoffPending = !1, this.idleTransitionSource = void 0, this.idleTransitionRemaining = 0, this.playbackOrigins.clear(), this.renderedTimeline.clear(), this.turnClauses = 0, this.unannouncedClauses = [], this.audioNodes.forEach((n) => {
|
|
1016
|
+
try {
|
|
1017
|
+
n.stop();
|
|
1018
|
+
} catch {
|
|
1019
|
+
}
|
|
1020
|
+
}), this.audioNodes.clear(), this.beforeFirstAudio = void 0, this.pendingPlayback = [], this.pendingPresentation = [], this.nextAudioTime = 0, this.audioStarted = !1, this.preparedAudioSeconds = 0, this.scheduledAudioSeconds = 0, this.currentTurnId = 0, this.speechTransitionSource = void 0, this.speechTransitionRemaining = 0, this.audioQueue.splice(0).forEach((n) => n.resolve()), this.queuedChunks = 0, this.metrics.inferenceQueueDepth = 0, console.info(
|
|
1021
|
+
`[BeVyra SDK ${this.runtimeId}] INTERRUPT generation=${e}->${this.generation} audio_queue=${t} presentation_queue=${s} active_audio_nodes=${i}`
|
|
1022
|
+
), this.endSpeech(), this.call("interrupt", { generation: this.generation });
|
|
1023
|
+
}
|
|
1024
|
+
close() {
|
|
1025
|
+
this.stopped = !0, this.displayProbe && cancelAnimationFrame(this.displayProbe), this.speechStartTimer && window.clearTimeout(this.speechStartTimer), this.speechStartTimer = 0, this.speechStartListener = void 0, this.speechEndListener = void 0, this.settleTimer && window.clearTimeout(this.settleTimer), this.settleResolve?.(), this.settleResolve = void 0, this.audioNodes.forEach((e) => {
|
|
1026
|
+
try {
|
|
1027
|
+
e.stop();
|
|
1028
|
+
} catch {
|
|
1029
|
+
}
|
|
1030
|
+
}), this.audioNodes.clear(), this.ownedBackground?.close(), this.audio?.state !== "closed" && this.audio?.close(), this.decoder?.close(), this.worker.postMessage({ id: ++this.id, type: "close" }), this.worker.terminate(), this.pending.forEach(
|
|
1031
|
+
(e) => e.reject(new Error("Local avatar runtime closed."))
|
|
1032
|
+
), this.pending.clear();
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
class Le {
|
|
1036
|
+
constructor(e = 450) {
|
|
1037
|
+
this.latencyMs = e, this.serverFreeAt = 0, this.calibrated = !1;
|
|
1038
|
+
}
|
|
1039
|
+
/** Places a clip reported as presenting at `receivedAt`. */
|
|
1040
|
+
present(e, t) {
|
|
1041
|
+
const s = this.serverFreeAt <= e, i = s ? e : this.serverFreeAt;
|
|
1042
|
+
return this.serverFreeAt = i + t, {
|
|
1043
|
+
/** Nothing was queued: the clip's onset can calibrate the latency. */
|
|
1044
|
+
idle: s,
|
|
1045
|
+
serverStart: i,
|
|
1046
|
+
audibleStart: i + this.latencyMs,
|
|
1047
|
+
audibleEnd: this.serverFreeAt + this.latencyMs
|
|
1048
|
+
};
|
|
1049
|
+
}
|
|
1050
|
+
/** Milliseconds of already-queued media the gateway will still play. */
|
|
1051
|
+
queuedMs(e) {
|
|
1052
|
+
return Math.max(0, this.serverFreeAt - e);
|
|
1053
|
+
}
|
|
1054
|
+
/** When everything queued so far will have been heard. */
|
|
1055
|
+
audibleDrainAt(e) {
|
|
1056
|
+
return Math.max(e, this.serverFreeAt + this.latencyMs);
|
|
1057
|
+
}
|
|
1058
|
+
/** The gateway discarded its queue (an interrupt it confirmed as flushed). */
|
|
1059
|
+
flush(e) {
|
|
1060
|
+
this.serverFreeAt = Math.min(this.serverFreeAt, e);
|
|
1061
|
+
}
|
|
1062
|
+
/**
|
|
1063
|
+
* Folds in one measured onset: audio expected to leave the gateway at
|
|
1064
|
+
* `serverStart` was first heard at `heardAt`. Implausible values are
|
|
1065
|
+
* ignored; the first sample replaces the default outright.
|
|
1066
|
+
*/
|
|
1067
|
+
calibrate(e, t) {
|
|
1068
|
+
const s = t - e;
|
|
1069
|
+
return s >= 30 && s <= 4e3 ? (this.latencyMs = this.calibrated ? Math.round(this.latencyMs * 0.7 + s * 0.3) : Math.round(s), this.calibrated = !0, s) : null;
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
function Oe(l, e = 0) {
|
|
1073
|
+
return Math.max(e, Math.ceil(l / 640)) * 40;
|
|
1074
|
+
}
|
|
1075
|
+
class De {
|
|
1076
|
+
constructor(e, t, s, i = !1) {
|
|
1077
|
+
if (this.video = e, this.target = t, this.onFirstFrame = s, this.composited = i, this.source = document.createElement("canvas"), this.gl = null, this.texture = null, this.texel = null, this.context = null, this.frameCallback = 0, this.animationFrame = 0, this.closed = !1, this.reportedReady = !1, this.presentedFrames = 0, this.sampledFrames = 0, this.sampledAt = performance.now(), this.fps = null, this.draw = () => {
|
|
1078
|
+
if (this.closed) return;
|
|
1079
|
+
const r = this.video.videoWidth, n = this.video.videoHeight;
|
|
1080
|
+
if (r > 0 && n > 0 && this.video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {
|
|
1081
|
+
if (this.gl && this.texture) {
|
|
1082
|
+
const o = this.output.measure(r, n);
|
|
1083
|
+
this.resize(o.width, o.height), this.texel && this.gl.uniform2f(this.texel, 1 / r, 1 / n), this.gl.bindTexture(this.gl.TEXTURE_2D, this.texture), this.gl.texImage2D(
|
|
1084
|
+
this.gl.TEXTURE_2D,
|
|
1085
|
+
0,
|
|
1086
|
+
this.gl.RGBA,
|
|
1087
|
+
this.gl.RGBA,
|
|
1088
|
+
this.gl.UNSIGNED_BYTE,
|
|
1089
|
+
this.video
|
|
1090
|
+
), this.gl.clear(this.gl.COLOR_BUFFER_BIT), this.gl.drawArrays(this.gl.TRIANGLES, 0, 6), this.gl.flush();
|
|
1091
|
+
const h = this.target.getContext("2d");
|
|
1092
|
+
h?.clearRect(0, 0, o.width, o.height), h?.drawImage(this.source, 0, 0);
|
|
1093
|
+
} else
|
|
1094
|
+
this.resize(r, n), this.drawCpu(r, n);
|
|
1095
|
+
this.reportedReady || (this.reportedReady = !0, console.info(
|
|
1096
|
+
`[BeVyra WHEP] first video frame · ${r}x${n}`
|
|
1097
|
+
), this.onFirstFrame()), this.presentedFrames += 1;
|
|
1098
|
+
const a = performance.now(), c = a - this.sampledAt;
|
|
1099
|
+
c >= 750 && (this.fps = (this.presentedFrames - this.sampledFrames) * 1e3 / c, this.sampledFrames = this.presentedFrames, this.sampledAt = a);
|
|
1100
|
+
}
|
|
1101
|
+
this.schedule();
|
|
1102
|
+
}, this.output = new be(t), this.gl = this.createWebGlRenderer(), this.gl)
|
|
1103
|
+
console.info("[BeVyra WHEP] GPU chroma compositor ready");
|
|
1104
|
+
else {
|
|
1105
|
+
if (this.context = t.getContext("2d", { alpha: !0 }), !this.context)
|
|
1106
|
+
throw new Error("This browser cannot composite the server avatar");
|
|
1107
|
+
console.info("[BeVyra WHEP] using CPU chroma compositor");
|
|
1108
|
+
}
|
|
1109
|
+
}
|
|
1110
|
+
createWebGlRenderer() {
|
|
1111
|
+
const e = this.source.getContext("webgl", {
|
|
1112
|
+
alpha: !0,
|
|
1113
|
+
antialias: !1,
|
|
1114
|
+
depth: !1,
|
|
1115
|
+
premultipliedAlpha: !1,
|
|
1116
|
+
preserveDrawingBuffer: !0,
|
|
1117
|
+
stencil: !1
|
|
1118
|
+
});
|
|
1119
|
+
if (!e) return null;
|
|
1120
|
+
const t = (s, i) => {
|
|
1121
|
+
const r = e.createShader(s);
|
|
1122
|
+
if (!r) throw new Error("Unable to create WHEP chroma shader");
|
|
1123
|
+
if (e.shaderSource(r, i), e.compileShader(r), !e.getShaderParameter(r, e.COMPILE_STATUS)) {
|
|
1124
|
+
const n = e.getShaderInfoLog(r) || "WHEP chroma shader failed";
|
|
1125
|
+
throw e.deleteShader(r), new Error(n);
|
|
1126
|
+
}
|
|
1127
|
+
return r;
|
|
1128
|
+
};
|
|
1129
|
+
try {
|
|
1130
|
+
const s = t(
|
|
1131
|
+
e.VERTEX_SHADER,
|
|
1132
|
+
"attribute vec2 p;attribute vec2 uv;varying vec2 v;void main(){gl_Position=vec4(p,0.,1.);v=uv;}"
|
|
1133
|
+
), i = t(
|
|
1134
|
+
e.FRAGMENT_SHADER,
|
|
1135
|
+
this.composited ? "precision mediump float;uniform sampler2D frame;varying vec2 v;void main(){gl_FragColor=texture2D(frame,v);}" : ye
|
|
1136
|
+
), r = e.createProgram();
|
|
1137
|
+
if (!r) throw new Error("Unable to create WHEP chroma program");
|
|
1138
|
+
if (e.attachShader(r, s), e.attachShader(r, i), e.linkProgram(r), e.deleteShader(s), e.deleteShader(i), !e.getProgramParameter(r, e.LINK_STATUS))
|
|
1139
|
+
throw new Error(
|
|
1140
|
+
e.getProgramInfoLog(r) || "WHEP chroma program failed"
|
|
1141
|
+
);
|
|
1142
|
+
e.useProgram(r), this.texel = e.getUniformLocation(r, "texel");
|
|
1143
|
+
const n = new Float32Array([
|
|
1144
|
+
-1,
|
|
1145
|
+
-1,
|
|
1146
|
+
0,
|
|
1147
|
+
1,
|
|
1148
|
+
1,
|
|
1149
|
+
-1,
|
|
1150
|
+
1,
|
|
1151
|
+
1,
|
|
1152
|
+
-1,
|
|
1153
|
+
1,
|
|
1154
|
+
0,
|
|
1155
|
+
0,
|
|
1156
|
+
-1,
|
|
1157
|
+
1,
|
|
1158
|
+
0,
|
|
1159
|
+
0,
|
|
1160
|
+
1,
|
|
1161
|
+
-1,
|
|
1162
|
+
1,
|
|
1163
|
+
1,
|
|
1164
|
+
1,
|
|
1165
|
+
1,
|
|
1166
|
+
1,
|
|
1167
|
+
0
|
|
1168
|
+
]), a = e.createBuffer();
|
|
1169
|
+
if (!a) throw new Error("Unable to create WHEP chroma buffer");
|
|
1170
|
+
e.bindBuffer(e.ARRAY_BUFFER, a), e.bufferData(e.ARRAY_BUFFER, n, e.STATIC_DRAW);
|
|
1171
|
+
const c = 4 * Float32Array.BYTES_PER_ELEMENT, o = e.getAttribLocation(r, "p"), h = e.getAttribLocation(r, "uv");
|
|
1172
|
+
e.enableVertexAttribArray(o), e.vertexAttribPointer(o, 2, e.FLOAT, !1, c, 0), e.enableVertexAttribArray(h), e.vertexAttribPointer(
|
|
1173
|
+
h,
|
|
1174
|
+
2,
|
|
1175
|
+
e.FLOAT,
|
|
1176
|
+
!1,
|
|
1177
|
+
c,
|
|
1178
|
+
2 * Float32Array.BYTES_PER_ELEMENT
|
|
1179
|
+
);
|
|
1180
|
+
const d = e.createTexture();
|
|
1181
|
+
if (!d) throw new Error("Unable to create WHEP chroma texture");
|
|
1182
|
+
return e.bindTexture(e.TEXTURE_2D, d), e.texParameteri(e.TEXTURE_2D, e.TEXTURE_MIN_FILTER, e.LINEAR), e.texParameteri(e.TEXTURE_2D, e.TEXTURE_MAG_FILTER, e.LINEAR), e.texParameteri(e.TEXTURE_2D, e.TEXTURE_WRAP_S, e.CLAMP_TO_EDGE), e.texParameteri(e.TEXTURE_2D, e.TEXTURE_WRAP_T, e.CLAMP_TO_EDGE), e.clearColor(0, 0, 0, 0), this.texture = d, e;
|
|
1183
|
+
} catch (s) {
|
|
1184
|
+
return console.warn("[BeVyra WHEP] GPU chroma compositor unavailable", s), e.getExtension("WEBGL_lose_context")?.loseContext(), null;
|
|
1185
|
+
}
|
|
1186
|
+
}
|
|
1187
|
+
resize(e, t) {
|
|
1188
|
+
(this.target.width !== e || this.target.height !== t) && (this.target.width = e, this.target.height = t), (this.source.width !== e || this.source.height !== t) && (this.source.width = e, this.source.height = t, this.gl?.viewport(0, 0, e, t));
|
|
1189
|
+
}
|
|
1190
|
+
drawCpu(e, t) {
|
|
1191
|
+
const s = this.context;
|
|
1192
|
+
if (!s || (s.drawImage(this.video, 0, 0, e, t), this.composited)) return;
|
|
1193
|
+
const i = s.getImageData(0, 0, e, t);
|
|
1194
|
+
ee(we(i.data), e, t), s.putImageData(i, 0, 0);
|
|
1195
|
+
}
|
|
1196
|
+
schedule() {
|
|
1197
|
+
this.closed || (this.reportedReady && this.video.requestVideoFrameCallback ? this.frameCallback = this.video.requestVideoFrameCallback(this.draw) : this.animationFrame = requestAnimationFrame(this.draw));
|
|
1198
|
+
}
|
|
1199
|
+
start() {
|
|
1200
|
+
this.schedule();
|
|
1201
|
+
}
|
|
1202
|
+
presentationFps() {
|
|
1203
|
+
return this.fps;
|
|
1204
|
+
}
|
|
1205
|
+
close() {
|
|
1206
|
+
this.closed = !0, this.frameCallback && this.video.cancelVideoFrameCallback && this.video.cancelVideoFrameCallback(this.frameCallback), this.animationFrame && cancelAnimationFrame(this.animationFrame), this.texture && this.gl?.deleteTexture(this.texture), this.gl?.getExtension("WEBGL_lose_context")?.loseContext(), this.target.getContext("2d")?.clearRect(0, 0, this.target.width, this.target.height);
|
|
1207
|
+
}
|
|
1208
|
+
}
|
|
1209
|
+
async function Be(l) {
|
|
1210
|
+
l.iceGatheringState !== "complete" && await new Promise((e) => {
|
|
1211
|
+
const t = () => {
|
|
1212
|
+
l.iceGatheringState === "complete" && (l.removeEventListener("icegatheringstatechange", t), e());
|
|
1213
|
+
};
|
|
1214
|
+
l.addEventListener("icegatheringstatechange", t), window.setTimeout(() => {
|
|
1215
|
+
l.removeEventListener("icegatheringstatechange", t), e();
|
|
1216
|
+
}, 2500);
|
|
1217
|
+
});
|
|
1218
|
+
}
|
|
1219
|
+
async function Ne(l, e, t, s, i = !1, r = {}) {
|
|
1220
|
+
const n = new URL(l, window.location.href).href, a = new RTCPeerConnection(), c = new De(e, t, s, i);
|
|
1221
|
+
let o = !1, h = !1, d = {};
|
|
1222
|
+
a.addTransceiver("video", { direction: "recvonly" }), r.receiveAudio !== !1 && a.addTransceiver("audio", { direction: "recvonly" });
|
|
1223
|
+
const u = new MediaStream(), p = window.setInterval(() => {
|
|
1224
|
+
a.getStats().then((g) => {
|
|
1225
|
+
const y = {};
|
|
1226
|
+
g.forEach((v) => {
|
|
1227
|
+
v.type === "inbound-rtp" && (v.kind === "audio" || v.mediaType === "audio" ? (y.audioPacketsLost = v.packetsLost ?? 0, y.audioJitterMs = Math.round((v.jitter ?? 0) * 1e3), y.audioConcealedSamples = v.concealedSamples ?? 0, y.audioConcealmentEvents = v.concealmentEvents ?? 0) : (v.kind === "video" || v.mediaType === "video") && (y.videoPacketsLost = v.packetsLost ?? 0, y.videoPacketsReceived = v.packetsReceived ?? 0, y.videoJitterMs = Math.round((v.jitter ?? 0) * 1e3), y.videoFramesDecoded = v.framesDecoded ?? 0, y.videoFramesDropped = v.framesDropped ?? 0, d = {
|
|
1228
|
+
packetsReceived: v.packetsReceived ?? 0,
|
|
1229
|
+
framesDecoded: v.framesDecoded ?? 0
|
|
1230
|
+
}));
|
|
1231
|
+
}), Object.keys(y).length && console.info("[BeVyra WHEP] transport stats", y);
|
|
1232
|
+
}).catch(() => {
|
|
1233
|
+
});
|
|
1234
|
+
}, 5e3);
|
|
1235
|
+
a.ontrack = ({ track: g }) => {
|
|
1236
|
+
console.info(`[BeVyra WHEP] ${g.kind} track received`), u.getTracks().some((y) => y.id === g.id) || u.addTrack(g), e.srcObject !== u && (e.srcObject = u), e.muted = !0, r.onTrack?.(g), g.kind === "video" && !o && (o = !0, c.start(), e.play().then(() => {
|
|
1237
|
+
h || console.info("[BeVyra WHEP] transport video playing");
|
|
1238
|
+
}).catch((y) => {
|
|
1239
|
+
h || console.error("[BeVyra WHEP] transport playback failed", y);
|
|
1240
|
+
}));
|
|
1241
|
+
}, await a.setLocalDescription(await a.createOffer()), await Be(a);
|
|
1242
|
+
const f = await fetch(n, {
|
|
1243
|
+
method: "POST",
|
|
1244
|
+
headers: { "Content-Type": "application/sdp" },
|
|
1245
|
+
body: a.localDescription?.sdp,
|
|
1246
|
+
signal: AbortSignal.timeout(2e4)
|
|
1247
|
+
});
|
|
1248
|
+
if (!f.ok)
|
|
1249
|
+
throw h = !0, window.clearInterval(p), c.close(), a.close(), new Error(`WHEP ${f.status}`);
|
|
1250
|
+
const b = f.headers.get("Location"), S = b ? new URL(b, n).href : null;
|
|
1251
|
+
try {
|
|
1252
|
+
await a.setRemoteDescription({
|
|
1253
|
+
type: "answer",
|
|
1254
|
+
sdp: await f.text()
|
|
1255
|
+
});
|
|
1256
|
+
} catch (g) {
|
|
1257
|
+
throw h = !0, window.clearInterval(p), c.close(), a.close(), e.pause(), e.srcObject = null, g;
|
|
1258
|
+
}
|
|
1259
|
+
return {
|
|
1260
|
+
presentationFps: () => c.presentationFps(),
|
|
1261
|
+
diagnostics: () => ({
|
|
1262
|
+
connectionState: a.connectionState,
|
|
1263
|
+
iceConnectionState: a.iceConnectionState,
|
|
1264
|
+
...d,
|
|
1265
|
+
videoTracks: u.getVideoTracks().map((g) => ({ state: g.readyState, muted: g.muted }))
|
|
1266
|
+
}),
|
|
1267
|
+
async close() {
|
|
1268
|
+
h = !0, window.clearInterval(p), c.close(), a.close(), e.pause(), e.srcObject = null, S && await fetch(S, {
|
|
1269
|
+
method: "DELETE",
|
|
1270
|
+
signal: AbortSignal.timeout(2e3)
|
|
1271
|
+
}).catch(() => {
|
|
1272
|
+
});
|
|
1273
|
+
}
|
|
1274
|
+
};
|
|
1275
|
+
}
|
|
1276
|
+
const V = 16e3, ce = 8e3, Ue = 3e4, $e = 2e4, We = 80;
|
|
1277
|
+
class T extends Error {
|
|
1278
|
+
constructor(e, t) {
|
|
1279
|
+
super(e), this.code = t, this.name = "RemoteAvatarRuntimeError";
|
|
1280
|
+
}
|
|
1281
|
+
}
|
|
1282
|
+
class Z extends T {
|
|
1283
|
+
constructor(e) {
|
|
1284
|
+
super(e, "avatar_unavailable"), this.name = "RemoteAvatarUnavailableError";
|
|
1285
|
+
}
|
|
1286
|
+
}
|
|
1287
|
+
function I(l, e = {}) {
|
|
1288
|
+
const t = Object.entries(e).map(([s, i]) => `${s}=${typeof i == "number" ? Math.round(i) : i}`).join(" ");
|
|
1289
|
+
console.info(`[BeVyra remote] ${l}${t ? ` ${t}` : ""}`);
|
|
1290
|
+
}
|
|
1291
|
+
function he(l) {
|
|
1292
|
+
return l === "landscape" || l === "square" ? l : l && typeof l == "object" && l.width > l.height ? "landscape" : "portrait";
|
|
1293
|
+
}
|
|
1294
|
+
function le(l) {
|
|
1295
|
+
return `This avatar (${l}) is not available on the hosted renderer yet. Choose another avatar or use a device that can run avatars on-device.`;
|
|
1296
|
+
}
|
|
1297
|
+
async function ve(l, e) {
|
|
1298
|
+
if (e === V || !l.length) return l;
|
|
1299
|
+
const t = Math.max(1, Math.round(l.length * V / e)), s = new OfflineAudioContext(1, t, V), i = s.createBuffer(1, l.length, e), r = i.getChannelData(0);
|
|
1300
|
+
for (let o = 0; o < l.length; o++) r[o] = l[o] / 32768;
|
|
1301
|
+
const n = s.createBufferSource();
|
|
1302
|
+
n.buffer = i, n.connect(s.destination), n.start();
|
|
1303
|
+
const a = (await s.startRendering()).getChannelData(0), c = new Int16Array(t);
|
|
1304
|
+
for (let o = 0; o < t; o++)
|
|
1305
|
+
c[o] = Math.max(-32768, Math.min(32767, Math.round(a[o] * 32768)));
|
|
1306
|
+
return c;
|
|
1307
|
+
}
|
|
1308
|
+
class q {
|
|
1309
|
+
constructor(e, t, s, i, r) {
|
|
1310
|
+
this.ws = e, this.whep = t, this.video = s, this.sessionId = i, this.aspect = r, this.generation = 0, this.clauseSeq = 0, this.turnKey = "", this.turnClauses = 0, this.sendChain = Promise.resolve(), this.inflight = /* @__PURE__ */ new Map(), this.timeline = new Le(), this.speechAudible = !1, this.timers = /* @__PURE__ */ new Set(), this.wakers = /* @__PURE__ */ new Set(), this.closed = !1, this.failure = null, this.nextAudioTime = 0, this.audioNodes = /* @__PURE__ */ new Set(), this.openedAt = performance.now(), this.state = {
|
|
1311
|
+
requestedThreads: 0,
|
|
1312
|
+
effectiveThreads: 0,
|
|
1313
|
+
crossOriginIsolated: window.crossOriginIsolated,
|
|
1314
|
+
hardwareConcurrency: navigator.hardwareConcurrency || 1,
|
|
1315
|
+
ortBackend: "wasm",
|
|
1316
|
+
audioEncoderMs: null,
|
|
1317
|
+
predictorMs: null,
|
|
1318
|
+
gaussianMs: null,
|
|
1319
|
+
inferenceFps: 0,
|
|
1320
|
+
sourceDecodeMs: null,
|
|
1321
|
+
inferenceQueueDepth: 0,
|
|
1322
|
+
droppedFrames: 0,
|
|
1323
|
+
presentedFps: 0,
|
|
1324
|
+
sourceDecoder: "webcodecs",
|
|
1325
|
+
calibrationFps: null,
|
|
1326
|
+
avDriftMs: 0,
|
|
1327
|
+
audioUnderruns: 0,
|
|
1328
|
+
lateFrames: 0,
|
|
1329
|
+
maxFrameLateMs: 0,
|
|
1330
|
+
// The gateway's WHEP timeline handles its own audio/video alignment.
|
|
1331
|
+
displayLeadMs: 0
|
|
1332
|
+
}, e.onmessage = (n) => this.receive(n), e.onclose = (n) => this.lost(n);
|
|
1333
|
+
}
|
|
1334
|
+
static async create(e) {
|
|
1335
|
+
const t = e.onStage ?? (() => {
|
|
1336
|
+
}), s = performance.now(), i = he(e.resolution);
|
|
1337
|
+
t("authorizing_remote_renderer");
|
|
1338
|
+
const r = await fetch("/api/avatar-render/session", {
|
|
1339
|
+
method: "POST",
|
|
1340
|
+
credentials: "same-origin",
|
|
1341
|
+
headers: { "Content-Type": "application/json" },
|
|
1342
|
+
body: JSON.stringify({
|
|
1343
|
+
avatarId: e.avatarId,
|
|
1344
|
+
surface: e.surface ?? "studio",
|
|
1345
|
+
tutorShare: e.tutorShare
|
|
1346
|
+
}),
|
|
1347
|
+
signal: e.signal
|
|
1348
|
+
}), n = await r.json().catch(() => ({}));
|
|
1349
|
+
if (!r.ok)
|
|
1350
|
+
throw I("AUTHORIZATION_FAILED", { avatar: e.avatarId, status: r.status, code: n.code }), n.code === "avatar_unavailable" || r.status === 404 ? new Z(n.error || le(e.avatarId)) : new T(
|
|
1351
|
+
n.error || `Hosted avatar rendering was refused (${r.status}).`,
|
|
1352
|
+
n.code || "render_unauthorized"
|
|
1353
|
+
);
|
|
1354
|
+
t("connecting_remote_renderer");
|
|
1355
|
+
const a = new URL(n.streamPath, window.location.origin);
|
|
1356
|
+
a.protocol = a.protocol === "https:" ? "wss:" : "ws:", a.searchParams.set("ticket", n.ticket);
|
|
1357
|
+
const c = new WebSocket(a);
|
|
1358
|
+
c.binaryType = "arraybuffer";
|
|
1359
|
+
const o = await new Promise((S, g) => {
|
|
1360
|
+
let y = !1;
|
|
1361
|
+
const v = (A, E) => {
|
|
1362
|
+
y || (y = !0, window.clearTimeout(m), e.signal?.removeEventListener("abort", w), A ? (c.onclose = null, c.readyState <= WebSocket.OPEN && c.close(1e3, "setup abandoned"), g(A)) : S(E));
|
|
1363
|
+
}, w = () => v(new DOMException("Avatar preparation aborted", "AbortError")), m = window.setTimeout(
|
|
1364
|
+
() => v(new T("The hosted avatar renderer is busy. Try again in a moment.", "render_timeout")),
|
|
1365
|
+
Ue
|
|
1366
|
+
);
|
|
1367
|
+
e.signal?.addEventListener("abort", w, { once: !0 }), c.onopen = () => {
|
|
1368
|
+
c.send(JSON.stringify({
|
|
1369
|
+
type: "session.setup",
|
|
1370
|
+
protocol: 1,
|
|
1371
|
+
avatar: e.avatarId,
|
|
1372
|
+
aspectRatio: i,
|
|
1373
|
+
audio: { encoding: "pcm_s16le", sampleRate: V, channels: 1 },
|
|
1374
|
+
videoComposite: "client"
|
|
1375
|
+
})), I("SETUP", { avatar: e.avatarId, aspect: i, audio: "pcm_s16le/16000/mono", ms: performance.now() - s });
|
|
1376
|
+
}, c.onmessage = (A) => {
|
|
1377
|
+
if (typeof A.data != "string") return;
|
|
1378
|
+
let E;
|
|
1379
|
+
try {
|
|
1380
|
+
E = JSON.parse(A.data);
|
|
1381
|
+
} catch {
|
|
1382
|
+
return;
|
|
1383
|
+
}
|
|
1384
|
+
if (E.type === "session.ready") v(void 0, E);
|
|
1385
|
+
else if (E.type === "error") {
|
|
1386
|
+
const k = String(E.message || "");
|
|
1387
|
+
v(/unknown avatar/i.test(k) ? new Z(le(e.avatarId)) : new T(k || "The hosted avatar renderer rejected the session.", String(E.code || "render_error")));
|
|
1388
|
+
}
|
|
1389
|
+
}, c.onclose = (A) => v(new T(A.reason || "The hosted avatar renderer closed the connection.", "render_closed"));
|
|
1390
|
+
});
|
|
1391
|
+
I("READY", {
|
|
1392
|
+
session: o.sessionId,
|
|
1393
|
+
ms: performance.now() - s,
|
|
1394
|
+
composited: o.videoComposited !== !1,
|
|
1395
|
+
aspect: o.aspectRatio ?? i
|
|
1396
|
+
}), o.videoComposited !== !1 && console.warn("[BeVyra remote] gateway composited its own background; the page background will show a second layer until it supports videoComposite=client"), e.body === "half" && console.warn("[BeVyra remote] half-body framing is not supported by the gateway; rendering full body");
|
|
1397
|
+
const h = (S) => {
|
|
1398
|
+
throw c.onclose = null, c.close(1e3, "media setup failed"), S;
|
|
1399
|
+
};
|
|
1400
|
+
o.whepUrl || h(new T("The hosted avatar renderer has no media endpoint.", "render_media_unavailable")), t("connecting_media");
|
|
1401
|
+
const d = document.createElement("video");
|
|
1402
|
+
d.playsInline = !0, d.autoplay = !0, d.muted = !0, d.setAttribute("aria-hidden", "true"), Object.assign(d.style, { position: "absolute", width: "1px", height: "1px", opacity: "0", pointerEvents: "none" }), (e.canvas.parentElement ?? document.body).appendChild(d);
|
|
1403
|
+
let u = () => {
|
|
1404
|
+
};
|
|
1405
|
+
const p = new Promise((S) => u = S);
|
|
1406
|
+
let f;
|
|
1407
|
+
try {
|
|
1408
|
+
f = await Ne(
|
|
1409
|
+
o.whepUrl,
|
|
1410
|
+
d,
|
|
1411
|
+
e.canvas,
|
|
1412
|
+
() => u(),
|
|
1413
|
+
o.videoComposited !== !1,
|
|
1414
|
+
{ receiveAudio: !1 }
|
|
1415
|
+
);
|
|
1416
|
+
let S = 0;
|
|
1417
|
+
try {
|
|
1418
|
+
await Promise.race([
|
|
1419
|
+
p,
|
|
1420
|
+
new Promise((g, y) => {
|
|
1421
|
+
S = window.setTimeout(() => {
|
|
1422
|
+
console.error("[BeVyra WHEP] first frame timeout", {
|
|
1423
|
+
...f?.diagnostics(),
|
|
1424
|
+
videoReadyState: d.readyState,
|
|
1425
|
+
videoWidth: d.videoWidth,
|
|
1426
|
+
videoHeight: d.videoHeight,
|
|
1427
|
+
paused: d.paused,
|
|
1428
|
+
streamActive: d.srcObject?.active
|
|
1429
|
+
}), y(new T("Hosted avatar media did not start. Check your network and try again.", "render_media_timeout"));
|
|
1430
|
+
}, $e);
|
|
1431
|
+
})
|
|
1432
|
+
]);
|
|
1433
|
+
} finally {
|
|
1434
|
+
window.clearTimeout(S);
|
|
1435
|
+
}
|
|
1436
|
+
if (e.signal?.aborted) throw new DOMException("Avatar preparation aborted", "AbortError");
|
|
1437
|
+
} catch (S) {
|
|
1438
|
+
await f?.close().catch(() => {
|
|
1439
|
+
}), d.remove(), h(S instanceof Error ? S : new Error(String(S)));
|
|
1440
|
+
}
|
|
1441
|
+
I("FIRST_MEDIA_FRAME", { session: o.sessionId, ms: performance.now() - s, audio_owner: "browser" });
|
|
1442
|
+
const b = new q(c, f, d, o.sessionId, i);
|
|
1443
|
+
return b.audio = e.audioContext, b;
|
|
1444
|
+
}
|
|
1445
|
+
get metrics() {
|
|
1446
|
+
return this.state.presentedFps = this.whep.presentationFps() ?? 0, this.state.inferenceQueueDepth = this.inflight.size, this.state;
|
|
1447
|
+
}
|
|
1448
|
+
async setScene(e, t, s) {
|
|
1449
|
+
e && I("SCENE_BACKGROUND_IGNORED"), t !== void 0 && he(t) !== this.aspect && console.warn(`[BeVyra remote] aspect ratio is fixed at ${this.aspect} for this session`), s === "half" && console.warn("[BeVyra remote] half-body framing is not supported by the gateway");
|
|
1450
|
+
}
|
|
1451
|
+
async enableAudio() {
|
|
1452
|
+
if (this.audio ??= new AudioContext({ sampleRate: 24e3 }), await this.audio.resume(), this.audio.state !== "running")
|
|
1453
|
+
throw new Error("Audio playback permission required.");
|
|
1454
|
+
this.video.muted = !0;
|
|
1455
|
+
try {
|
|
1456
|
+
await this.video.play();
|
|
1457
|
+
} catch {
|
|
1458
|
+
throw new Error("Audio playback permission required.");
|
|
1459
|
+
}
|
|
1460
|
+
I("AUDIO_ENABLED", { session: this.sessionId, owner: "browser", video_lead_ms: this.timeline.latencyMs });
|
|
1461
|
+
}
|
|
1462
|
+
onSpeechStart(e) {
|
|
1463
|
+
return this.speechStartListener = e, () => {
|
|
1464
|
+
this.speechStartListener === e && (this.speechStartListener = void 0);
|
|
1465
|
+
};
|
|
1466
|
+
}
|
|
1467
|
+
onSpeechEnd(e) {
|
|
1468
|
+
return this.speechEndListener = e, () => {
|
|
1469
|
+
this.speechEndListener === e && (this.speechEndListener = void 0);
|
|
1470
|
+
};
|
|
1471
|
+
}
|
|
1472
|
+
onClauseStart(e) {
|
|
1473
|
+
return this.clauseStartListener = e, () => {
|
|
1474
|
+
this.clauseStartListener === e && (this.clauseStartListener = void 0);
|
|
1475
|
+
};
|
|
1476
|
+
}
|
|
1477
|
+
/** The gateway connection failed after the session started. */
|
|
1478
|
+
onFailure(e) {
|
|
1479
|
+
return this.failureListener = e, () => {
|
|
1480
|
+
this.failureListener === e && (this.failureListener = void 0);
|
|
1481
|
+
};
|
|
1482
|
+
}
|
|
1483
|
+
pushAssistantAudio(e) {
|
|
1484
|
+
if (this.failure) return Promise.reject(this.failure);
|
|
1485
|
+
const t = e.responseGenerationId;
|
|
1486
|
+
if (this.closed || t !== this.generation)
|
|
1487
|
+
return I("STALE_DROP", { generation: t, current: this.generation, turn: e.turnId, stage: "push" }), Promise.resolve();
|
|
1488
|
+
const s = `${t}:${e.turnId}`;
|
|
1489
|
+
s !== this.turnKey && (this.turnKey = s, this.turnClauses = 0);
|
|
1490
|
+
const i = this.turnClauses++, r = this.sendChain.then(async () => {
|
|
1491
|
+
const n = await ve(e.pcm24, e.sampleRate);
|
|
1492
|
+
if (this.closed || t !== this.generation) {
|
|
1493
|
+
I("STALE_DROP", { generation: t, current: this.generation, turn: e.turnId, stage: "send" });
|
|
1494
|
+
return;
|
|
1495
|
+
}
|
|
1496
|
+
if (this.failure) throw this.failure;
|
|
1497
|
+
if (!n.length) return;
|
|
1498
|
+
const a = ++this.clauseSeq;
|
|
1499
|
+
let c = () => {
|
|
1500
|
+
};
|
|
1501
|
+
const o = new Promise((d) => c = d), h = {
|
|
1502
|
+
clauseId: a,
|
|
1503
|
+
turnId: e.turnId,
|
|
1504
|
+
generation: t,
|
|
1505
|
+
index: i,
|
|
1506
|
+
samples: n.length,
|
|
1507
|
+
sentAt: performance.now(),
|
|
1508
|
+
pcm: e.pcm24.slice(),
|
|
1509
|
+
sampleRate: e.sampleRate,
|
|
1510
|
+
presented: o,
|
|
1511
|
+
resolvePresented: c
|
|
1512
|
+
};
|
|
1513
|
+
this.inflight.set(a, h), this.lastClause = h, this.send({
|
|
1514
|
+
type: "clause.start",
|
|
1515
|
+
clauseId: a,
|
|
1516
|
+
turnId: e.turnId,
|
|
1517
|
+
generation: t,
|
|
1518
|
+
sampleCount: n.length,
|
|
1519
|
+
boundary: e.clauseBoundary ?? "terminal"
|
|
1520
|
+
});
|
|
1521
|
+
for (let d = 0; d < n.length; d += ce)
|
|
1522
|
+
this.send(n.subarray(d, d + ce));
|
|
1523
|
+
this.send({ type: "clause.end", clauseId: a }), I("CLAUSE_SENT", {
|
|
1524
|
+
clause: a,
|
|
1525
|
+
turn: e.turnId,
|
|
1526
|
+
generation: t,
|
|
1527
|
+
index: i,
|
|
1528
|
+
samples: n.length,
|
|
1529
|
+
source_rate: e.sampleRate,
|
|
1530
|
+
boundary: e.clauseBoundary ?? "-",
|
|
1531
|
+
queue: this.inflight.size,
|
|
1532
|
+
ws_buffered: this.ws.bufferedAmount
|
|
1533
|
+
});
|
|
1534
|
+
});
|
|
1535
|
+
return this.sendChain = r.catch(() => {
|
|
1536
|
+
}), r;
|
|
1537
|
+
}
|
|
1538
|
+
async completeTurn(e = !0, t = this.generation) {
|
|
1539
|
+
if (await this.sendChain, this.closed || t !== this.generation) return;
|
|
1540
|
+
const s = this.lastClause?.generation === t ? this.lastClause : void 0;
|
|
1541
|
+
if (s && await Promise.race([s.presented, this.wake()]), this.closed || t !== this.generation) return;
|
|
1542
|
+
const i = (s?.audibleEnd ?? performance.now()) + We;
|
|
1543
|
+
await Promise.race([new Promise((r) => this.at(i, r)), this.wake()]), !(this.closed || t !== this.generation) && (I("TURN_SETTLED", { generation: t, turn: s?.turnId ?? "-", clauses: this.turnClauses, latency_ms: this.timeline.latencyMs }), this.turnKey = "", this.turnClauses = 0, this.lastClause = void 0, this.endSpeech());
|
|
1544
|
+
}
|
|
1545
|
+
interrupt() {
|
|
1546
|
+
const e = this.generation, t = performance.now(), s = this.timeline.queuedMs(t);
|
|
1547
|
+
this.generation++, this.clearTimers(), this.wakeAll(), this.audioNodes.forEach((i) => {
|
|
1548
|
+
try {
|
|
1549
|
+
i.stop();
|
|
1550
|
+
} catch {
|
|
1551
|
+
}
|
|
1552
|
+
}), this.audioNodes.clear(), this.nextAudioTime = 0, this.video.muted = !0, this.timeline.flush(t), this.turnKey = "", this.turnClauses = 0, this.lastClause = void 0, !this.failure && this.ws.readyState === WebSocket.OPEN && this.send({ type: "session.interrupt", generation: this.generation }), I("INTERRUPT", {
|
|
1553
|
+
generation: `${e}->${this.generation}`,
|
|
1554
|
+
inflight: this.inflight.size,
|
|
1555
|
+
server_queued_ms: s,
|
|
1556
|
+
local_audio_flushed: !0
|
|
1557
|
+
}), this.endSpeech();
|
|
1558
|
+
}
|
|
1559
|
+
synchronizeGeneration(e) {
|
|
1560
|
+
if (!Number.isSafeInteger(e) || e < 0)
|
|
1561
|
+
throw new Error(`Invalid renderer generation: ${e}`);
|
|
1562
|
+
if (e === this.generation) return;
|
|
1563
|
+
const t = this.generation;
|
|
1564
|
+
this.clearTimers(), this.wakeAll(), this.timeline.flush(performance.now()), this.turnKey = "", this.turnClauses = 0, this.lastClause = void 0, this.generation = e, !this.failure && this.ws.readyState === WebSocket.OPEN && this.send({ type: "session.interrupt", generation: e }), I("GENERATION_SYNC", { generation: `${t}->${e}` });
|
|
1565
|
+
}
|
|
1566
|
+
async generate() {
|
|
1567
|
+
throw new T(
|
|
1568
|
+
"Video generation runs on-device; the hosted renderer streams live sessions only.",
|
|
1569
|
+
"generate_unsupported"
|
|
1570
|
+
);
|
|
1571
|
+
}
|
|
1572
|
+
close() {
|
|
1573
|
+
this.closed || (this.closed = !0, this.clearTimers(), this.wakeAll(), this.speechStartListener = this.speechEndListener = void 0, this.clauseStartListener = void 0, this.failureListener = void 0, this.ws.onclose = null, this.ws.readyState === WebSocket.OPEN && (this.send({ type: "session.end" }), this.ws.close(1e3, "session ended")), this.whep.close().catch(() => {
|
|
1574
|
+
}), this.video.remove(), this.audioNodes.forEach((e) => {
|
|
1575
|
+
try {
|
|
1576
|
+
e.stop();
|
|
1577
|
+
} catch {
|
|
1578
|
+
}
|
|
1579
|
+
}), this.audioNodes.clear(), this.audio?.close().catch(() => {
|
|
1580
|
+
}), I("CLOSED", { session: this.sessionId, seconds: (performance.now() - this.openedAt) / 1e3 }));
|
|
1581
|
+
}
|
|
1582
|
+
send(e) {
|
|
1583
|
+
if (this.ws.readyState !== WebSocket.OPEN)
|
|
1584
|
+
throw this.failure ?? new T("The hosted avatar connection was lost.", "render_closed");
|
|
1585
|
+
this.ws.send(e instanceof Int16Array ? e : JSON.stringify(e));
|
|
1586
|
+
}
|
|
1587
|
+
receive(e) {
|
|
1588
|
+
if (typeof e.data != "string") return;
|
|
1589
|
+
let t;
|
|
1590
|
+
try {
|
|
1591
|
+
t = JSON.parse(e.data);
|
|
1592
|
+
} catch {
|
|
1593
|
+
return;
|
|
1594
|
+
}
|
|
1595
|
+
const s = performance.now();
|
|
1596
|
+
switch (t.type) {
|
|
1597
|
+
case "clause.accepted": {
|
|
1598
|
+
const i = this.inflight.get(Number(t.clauseId));
|
|
1599
|
+
i && typeof t.frames == "number" && (i.frames = t.frames), I("CLAUSE_ACCEPTED", {
|
|
1600
|
+
clause: Number(t.clauseId),
|
|
1601
|
+
generation: Number(t.generation),
|
|
1602
|
+
current: this.generation,
|
|
1603
|
+
samples: Number(t.samples),
|
|
1604
|
+
frames: Number(t.frames),
|
|
1605
|
+
render_ms: i ? s - i.sentAt : "-",
|
|
1606
|
+
queue: this.inflight.size
|
|
1607
|
+
});
|
|
1608
|
+
break;
|
|
1609
|
+
}
|
|
1610
|
+
case "clause.presenting":
|
|
1611
|
+
this.presenting(Number(t.clauseId), s);
|
|
1612
|
+
break;
|
|
1613
|
+
case "session.interrupted":
|
|
1614
|
+
t.flushed === !0 && this.timeline.flush(s), I("INTERRUPTED", {
|
|
1615
|
+
generation: Number(t.generation),
|
|
1616
|
+
flushed: t.flushed === !0,
|
|
1617
|
+
server_queued_ms: this.timeline.queuedMs(s)
|
|
1618
|
+
});
|
|
1619
|
+
break;
|
|
1620
|
+
case "session.ended":
|
|
1621
|
+
I("ENDED", { session: this.sessionId });
|
|
1622
|
+
break;
|
|
1623
|
+
case "error":
|
|
1624
|
+
this.fail(new T(String(t.message || "The hosted avatar renderer failed."), String(t.code || "render_error")));
|
|
1625
|
+
break;
|
|
1626
|
+
}
|
|
1627
|
+
}
|
|
1628
|
+
presenting(e, t) {
|
|
1629
|
+
const s = this.inflight.get(e);
|
|
1630
|
+
if (!s) return void I("UNKNOWN_CLAUSE", { clause: e });
|
|
1631
|
+
this.inflight.delete(e);
|
|
1632
|
+
const i = this.timeline.present(t, Oe(s.samples, s.frames)), r = s.generation !== this.generation;
|
|
1633
|
+
if (s.resolvePresented(), I("CLAUSE_PRESENTING", {
|
|
1634
|
+
clause: e,
|
|
1635
|
+
generation: s.generation,
|
|
1636
|
+
current: this.generation,
|
|
1637
|
+
stale: r,
|
|
1638
|
+
index: s.index,
|
|
1639
|
+
queue: this.inflight.size,
|
|
1640
|
+
server_queued_ms: this.timeline.queuedMs(t),
|
|
1641
|
+
audible_in_ms: i.audibleStart - t,
|
|
1642
|
+
latency_ms: this.timeline.latencyMs
|
|
1643
|
+
}), r) return;
|
|
1644
|
+
const n = this.scheduleAudio(s, i.audibleStart);
|
|
1645
|
+
s.audibleEnd = n.end, this.at(n.start, () => {
|
|
1646
|
+
this.closed || s.generation !== this.generation || this.speechAudible || (this.speechAudible = !0, I("SPEECH_START", { generation: s.generation, turn: s.turnId }), this.speechStartListener?.());
|
|
1647
|
+
}), this.clauseStartListener?.(s.index, n.start);
|
|
1648
|
+
}
|
|
1649
|
+
scheduleAudio(e, t) {
|
|
1650
|
+
const s = this.audio;
|
|
1651
|
+
if (!s) throw new Error("Avatar audio output is not initialized.");
|
|
1652
|
+
const i = s.createBuffer(1, e.pcm.length, e.sampleRate), r = i.getChannelData(0);
|
|
1653
|
+
for (let u = 0; u < e.pcm.length; u++)
|
|
1654
|
+
r[u] = e.pcm[u] / 32768;
|
|
1655
|
+
const n = s.createBufferSource(), a = s.currentTime + Math.max(0, t - performance.now()) / 1e3, c = Math.max(s.currentTime + 0.02, a, this.nextAudioTime), o = (s.outputLatency || 0) + (s.baseLatency || 0), h = performance.now() + (c - s.currentTime + o) * 1e3, d = h + i.duration * 1e3;
|
|
1656
|
+
return n.buffer = i, n.connect(s.destination), this.audioNodes.add(n), n.onended = () => {
|
|
1657
|
+
this.audioNodes.delete(n), n.disconnect();
|
|
1658
|
+
}, n.start(c), this.nextAudioTime = c + i.duration, I("LOCAL_AUDIO_SCHEDULED", {
|
|
1659
|
+
clause: e.clauseId,
|
|
1660
|
+
generation: e.generation,
|
|
1661
|
+
starts_in_ms: h - performance.now(),
|
|
1662
|
+
duration_ms: i.duration * 1e3
|
|
1663
|
+
}), { start: h, end: d };
|
|
1664
|
+
}
|
|
1665
|
+
lost(e) {
|
|
1666
|
+
this.closed || this.fail(new T(e.reason || "The hosted avatar connection was lost.", "render_closed"));
|
|
1667
|
+
}
|
|
1668
|
+
fail(e) {
|
|
1669
|
+
this.failure || this.closed || (this.failure = e, I("FAILED", { session: this.sessionId, code: e.code, message: JSON.stringify(e.message) }), this.clearTimers(), this.wakeAll(), this.inflight.forEach((t) => t.resolvePresented()), this.endSpeech(), this.failureListener?.(e));
|
|
1670
|
+
}
|
|
1671
|
+
endSpeech() {
|
|
1672
|
+
this.speechAudible && (this.speechAudible = !1, I("SPEECH_END", { generation: this.generation }), this.speechEndListener?.());
|
|
1673
|
+
}
|
|
1674
|
+
at(e, t) {
|
|
1675
|
+
const s = window.setTimeout(() => {
|
|
1676
|
+
this.timers.delete(s), t();
|
|
1677
|
+
}, Math.max(0, e - performance.now()));
|
|
1678
|
+
this.timers.add(s);
|
|
1679
|
+
}
|
|
1680
|
+
clearTimers() {
|
|
1681
|
+
this.timers.forEach((e) => window.clearTimeout(e)), this.timers.clear();
|
|
1682
|
+
}
|
|
1683
|
+
/** Resolves on the next interrupt, failure or close. */
|
|
1684
|
+
wake() {
|
|
1685
|
+
return new Promise((e) => this.wakers.add(e));
|
|
1686
|
+
}
|
|
1687
|
+
wakeAll() {
|
|
1688
|
+
const e = Array.from(this.wakers);
|
|
1689
|
+
this.wakers.clear(), e.forEach((t) => t());
|
|
1690
|
+
}
|
|
1691
|
+
}
|
|
1692
|
+
class Ve {
|
|
1693
|
+
constructor(e) {
|
|
1694
|
+
if (this.target = e, this.source = document.createElement("canvas"), this.texture = null, this.texel = null, this.output = new be(e), this.gl = this.source.getContext("webgl", { alpha: !0, premultipliedAlpha: !1, preserveDrawingBuffer: !0 }), !this.gl) return;
|
|
1695
|
+
const t = this.gl, s = (i, r) => {
|
|
1696
|
+
const n = t.createShader(i);
|
|
1697
|
+
if (t.shaderSource(n, r), t.compileShader(n), !t.getShaderParameter(n, t.COMPILE_STATUS)) throw new Error(t.getShaderInfoLog(n) || "chroma shader failed");
|
|
1698
|
+
return n;
|
|
1699
|
+
};
|
|
1700
|
+
try {
|
|
1701
|
+
const i = s(t.VERTEX_SHADER, "attribute vec2 p;attribute vec2 uv;varying vec2 v;void main(){gl_Position=vec4(p,0.,1.);v=uv;}"), r = s(t.FRAGMENT_SHADER, ye), n = t.createProgram();
|
|
1702
|
+
if (t.attachShader(n, i), t.attachShader(n, r), t.linkProgram(n), t.deleteShader(i), t.deleteShader(r), !t.getProgramParameter(n, t.LINK_STATUS)) throw new Error(t.getProgramInfoLog(n) || "chroma program failed");
|
|
1703
|
+
t.useProgram(n), this.texel = t.getUniformLocation(n, "texel");
|
|
1704
|
+
const a = t.createBuffer();
|
|
1705
|
+
t.bindBuffer(t.ARRAY_BUFFER, a), t.bufferData(t.ARRAY_BUFFER, new Float32Array([
|
|
1706
|
+
-1,
|
|
1707
|
+
-1,
|
|
1708
|
+
0,
|
|
1709
|
+
1,
|
|
1710
|
+
1,
|
|
1711
|
+
-1,
|
|
1712
|
+
1,
|
|
1713
|
+
1,
|
|
1714
|
+
-1,
|
|
1715
|
+
1,
|
|
1716
|
+
0,
|
|
1717
|
+
0,
|
|
1718
|
+
-1,
|
|
1719
|
+
1,
|
|
1720
|
+
0,
|
|
1721
|
+
0,
|
|
1722
|
+
1,
|
|
1723
|
+
-1,
|
|
1724
|
+
1,
|
|
1725
|
+
1,
|
|
1726
|
+
1,
|
|
1727
|
+
1,
|
|
1728
|
+
1,
|
|
1729
|
+
0
|
|
1730
|
+
]), t.STATIC_DRAW);
|
|
1731
|
+
const c = t.getAttribLocation(n, "p"), o = t.getAttribLocation(n, "uv");
|
|
1732
|
+
t.enableVertexAttribArray(c), t.vertexAttribPointer(c, 2, t.FLOAT, !1, 16, 0), t.enableVertexAttribArray(o), t.vertexAttribPointer(o, 2, t.FLOAT, !1, 16, 8), this.texture = t.createTexture(), t.bindTexture(t.TEXTURE_2D, this.texture), t.texParameteri(t.TEXTURE_2D, t.TEXTURE_MIN_FILTER, t.LINEAR), t.texParameteri(t.TEXTURE_2D, t.TEXTURE_MAG_FILTER, t.LINEAR), t.texParameteri(t.TEXTURE_2D, t.TEXTURE_WRAP_S, t.CLAMP_TO_EDGE), t.texParameteri(t.TEXTURE_2D, t.TEXTURE_WRAP_T, t.CLAMP_TO_EDGE);
|
|
1733
|
+
} catch {
|
|
1734
|
+
t.getExtension("WEBGL_lose_context")?.loseContext(), this.gl = null, this.texture = null;
|
|
1735
|
+
}
|
|
1736
|
+
}
|
|
1737
|
+
draw(e) {
|
|
1738
|
+
if (this.gl && this.texture) {
|
|
1739
|
+
const n = this.gl, { width: a, height: c } = this.output.measure(e.width, e.height);
|
|
1740
|
+
(this.target.width !== a || this.target.height !== c) && (this.target.width = a, this.target.height = c), (this.source.width !== a || this.source.height !== c) && (this.source.width = a, this.source.height = c, n.viewport(0, 0, a, c)), this.texel && n.uniform2f(this.texel, 1 / e.width, 1 / e.height), n.bindTexture(n.TEXTURE_2D, this.texture), n.texImage2D(n.TEXTURE_2D, 0, n.RGBA, n.RGBA, n.UNSIGNED_BYTE, e), n.clearColor(0, 0, 0, 0), n.clear(n.COLOR_BUFFER_BIT), n.drawArrays(n.TRIANGLES, 0, 6);
|
|
1741
|
+
const o = this.target.getContext("2d", { alpha: !0 });
|
|
1742
|
+
o?.clearRect(0, 0, a, c), o?.drawImage(this.source, 0, 0);
|
|
1743
|
+
return;
|
|
1744
|
+
}
|
|
1745
|
+
const { width: t, height: s } = e;
|
|
1746
|
+
(this.target.width !== t || this.target.height !== s) && (this.target.width = t, this.target.height = s);
|
|
1747
|
+
const i = this.target.getContext("2d", { alpha: !0 });
|
|
1748
|
+
if (!i) throw new Error("Canvas chroma compositor unavailable");
|
|
1749
|
+
i.drawImage(e, 0, 0);
|
|
1750
|
+
const r = i.getImageData(0, 0, t, s);
|
|
1751
|
+
ee(we(r.data), t, s), i.putImageData(r, 0, 0);
|
|
1752
|
+
}
|
|
1753
|
+
close() {
|
|
1754
|
+
this.gl?.getExtension("WEBGL_lose_context")?.loseContext();
|
|
1755
|
+
}
|
|
1756
|
+
}
|
|
1757
|
+
const He = 16e3, de = 240, ue = 2;
|
|
1758
|
+
class te {
|
|
1759
|
+
constructor(e) {
|
|
1760
|
+
this.options = e, this.generation = 0, this.closed = !1, this.chunks = [], this.hasVideo = !1, this.timers = /* @__PURE__ */ new Set(), this.wakers = /* @__PURE__ */ new Set(), this.state = {
|
|
1761
|
+
requestedThreads: 0,
|
|
1762
|
+
effectiveThreads: 0,
|
|
1763
|
+
crossOriginIsolated: window.crossOriginIsolated,
|
|
1764
|
+
hardwareConcurrency: navigator.hardwareConcurrency || 1,
|
|
1765
|
+
ortBackend: "wasm",
|
|
1766
|
+
audioEncoderMs: null,
|
|
1767
|
+
predictorMs: null,
|
|
1768
|
+
gaussianMs: null,
|
|
1769
|
+
inferenceFps: 0,
|
|
1770
|
+
sourceDecodeMs: null,
|
|
1771
|
+
inferenceQueueDepth: 0,
|
|
1772
|
+
droppedFrames: 0,
|
|
1773
|
+
presentedFps: 0,
|
|
1774
|
+
sourceDecoder: "webcodecs",
|
|
1775
|
+
calibrationFps: null,
|
|
1776
|
+
avDriftMs: 0,
|
|
1777
|
+
audioUnderruns: 0,
|
|
1778
|
+
lateFrames: 0,
|
|
1779
|
+
maxFrameLateMs: 0,
|
|
1780
|
+
displayLeadMs: 0
|
|
1781
|
+
}, this.audio = e.audioContext, this.compositor = new Ve(e.canvas);
|
|
1782
|
+
}
|
|
1783
|
+
static async create(e) {
|
|
1784
|
+
if (console.info(`[BeVyra runpod] configured avatar=${e.avatarId}`), e.onStage?.("connecting_remote_renderer"), e.signal?.aborted) throw new DOMException("Avatar preparation aborted", "AbortError");
|
|
1785
|
+
const t = new te(e);
|
|
1786
|
+
return fetch(`/avatar-library/${encodeURIComponent(e.avatarId)}/camera.jpg`, {
|
|
1787
|
+
credentials: "same-origin",
|
|
1788
|
+
signal: e.signal
|
|
1789
|
+
}).then(async (s) => {
|
|
1790
|
+
if (!s.ok || t.closed || t.hasVideo) return;
|
|
1791
|
+
const i = await createImageBitmap(await s.blob());
|
|
1792
|
+
!t.hasVideo && !t.closed && t.compositor.draw(i), i.close();
|
|
1793
|
+
}).catch(() => {
|
|
1794
|
+
}), t;
|
|
1795
|
+
}
|
|
1796
|
+
get metrics() {
|
|
1797
|
+
return this.state;
|
|
1798
|
+
}
|
|
1799
|
+
async setScene(e, t, s) {
|
|
1800
|
+
}
|
|
1801
|
+
async enableAudio() {
|
|
1802
|
+
if (this.audio ??= new AudioContext({ sampleRate: 24e3 }), await this.audio.resume(), this.audio.state !== "running") throw new Error("Audio playback permission required.");
|
|
1803
|
+
(!this.warmSocket || this.warmSocket.readyState !== WebSocket.OPEN) && await this.warm();
|
|
1804
|
+
}
|
|
1805
|
+
onSpeechStart(e) {
|
|
1806
|
+
return this.speechStartListener = e, () => {
|
|
1807
|
+
this.speechStartListener === e && (this.speechStartListener = void 0);
|
|
1808
|
+
};
|
|
1809
|
+
}
|
|
1810
|
+
onSpeechEnd(e) {
|
|
1811
|
+
return this.speechEndListener = e, () => {
|
|
1812
|
+
this.speechEndListener === e && (this.speechEndListener = void 0);
|
|
1813
|
+
};
|
|
1814
|
+
}
|
|
1815
|
+
onClauseStart(e) {
|
|
1816
|
+
return this.clauseStartListener = e, () => {
|
|
1817
|
+
this.clauseStartListener === e && (this.clauseStartListener = void 0);
|
|
1818
|
+
};
|
|
1819
|
+
}
|
|
1820
|
+
onFailure(e) {
|
|
1821
|
+
return this.failureListener = e, () => {
|
|
1822
|
+
this.failureListener === e && (this.failureListener = void 0);
|
|
1823
|
+
};
|
|
1824
|
+
}
|
|
1825
|
+
async pushAssistantAudio(e) {
|
|
1826
|
+
this.closed || e.responseGenerationId !== this.generation || this.chunks.push({ ...e, pcm24: e.pcm24.slice() });
|
|
1827
|
+
}
|
|
1828
|
+
async completeTurn(e = !0, t = this.generation) {
|
|
1829
|
+
if (this.closed || t !== this.generation) return;
|
|
1830
|
+
const s = this.chunks.splice(0);
|
|
1831
|
+
if (s.length)
|
|
1832
|
+
try {
|
|
1833
|
+
await this.renderTurn(s, t);
|
|
1834
|
+
} catch (i) {
|
|
1835
|
+
if (this.closed || t !== this.generation) return;
|
|
1836
|
+
const r = i instanceof Error ? i : new Error(String(i));
|
|
1837
|
+
throw this.interrupt(), this.failureListener?.(r), r;
|
|
1838
|
+
}
|
|
1839
|
+
}
|
|
1840
|
+
async renderTurn(e, t) {
|
|
1841
|
+
const s = this.audio;
|
|
1842
|
+
if (!s) throw new Error("Avatar audio output is not initialized.");
|
|
1843
|
+
const i = e[0].sampleRate;
|
|
1844
|
+
if (e.some((d) => d.sampleRate !== i)) throw new Error("Mixed audio rates in one turn");
|
|
1845
|
+
const r = e[0].startSample, n = Math.max(...e.map((d) => d.endSample - r));
|
|
1846
|
+
if (n < 400 || n > i * 120) throw new Error("RunPod audio unit must be 25 ms to 120 s");
|
|
1847
|
+
const a = new Int16Array(n);
|
|
1848
|
+
e.forEach((d) => a.set(d.pcm24, d.startSample - r));
|
|
1849
|
+
const c = await ve(a, i);
|
|
1850
|
+
if (t !== this.generation || this.closed) return;
|
|
1851
|
+
const o = s.createBuffer(1, a.length, i), h = o.getChannelData(0);
|
|
1852
|
+
for (let d = 0; d < a.length; d++) h[d] = a[d] / 32768;
|
|
1853
|
+
for (let d = 0; d <= ue; d++) {
|
|
1854
|
+
if (this.closed || t !== this.generation) return;
|
|
1855
|
+
try {
|
|
1856
|
+
await this.streamOnce(c, o, e, t);
|
|
1857
|
+
return;
|
|
1858
|
+
} catch (u) {
|
|
1859
|
+
if (!(u instanceof T) || u.code !== "busy" || d === ue) throw u;
|
|
1860
|
+
let p = () => {
|
|
1861
|
+
};
|
|
1862
|
+
const f = new Promise((b) => p = b);
|
|
1863
|
+
this.wakers.add(p), await Promise.race([new Promise((b) => this.after(500 * (d + 1), b)), f]), this.wakers.delete(p);
|
|
1864
|
+
}
|
|
1865
|
+
}
|
|
1866
|
+
}
|
|
1867
|
+
async warm() {
|
|
1868
|
+
this.options.onStage?.("warming_remote_renderer");
|
|
1869
|
+
const e = await this.connectSocket(this.generation);
|
|
1870
|
+
try {
|
|
1871
|
+
await new Promise((t, s) => {
|
|
1872
|
+
let i = !1;
|
|
1873
|
+
const r = (c) => {
|
|
1874
|
+
i || (i = !0, window.clearTimeout(a), this.options.signal?.removeEventListener("abort", n), e.onmessage = null, e.onclose = null, e.onerror = null, c ? s(c) : t());
|
|
1875
|
+
}, n = () => r(new DOMException("Avatar preparation aborted", "AbortError")), a = window.setTimeout(() => r(new T("RunPod model warmup timed out", "render_timeout")), 15e4);
|
|
1876
|
+
this.options.signal?.addEventListener("abort", n, { once: !0 }), e.onmessage = (c) => {
|
|
1877
|
+
if (typeof c.data == "string")
|
|
1878
|
+
try {
|
|
1879
|
+
const o = JSON.parse(c.data);
|
|
1880
|
+
o.type === "ready" ? r() : o.type === "busy" ? r(new T("RunPod worker is busy", "busy")) : o.type === "error" && r(new T(String(o.message || "RunPod warmup failed"), "render_error"));
|
|
1881
|
+
} catch (o) {
|
|
1882
|
+
r(o instanceof Error ? o : new Error(String(o)));
|
|
1883
|
+
}
|
|
1884
|
+
}, e.onclose = (c) => r(new T(c.reason || "RunPod closed during warmup", "render_closed")), e.onerror = () => r(new T("RunPod warmup connection failed", "render_unavailable")), e.send(JSON.stringify({ type: "warm", avatar: this.options.avatarId }));
|
|
1885
|
+
}), this.warmSocket = e;
|
|
1886
|
+
} catch (t) {
|
|
1887
|
+
throw e.close(), this.socket === e && (this.socket = void 0), t;
|
|
1888
|
+
}
|
|
1889
|
+
}
|
|
1890
|
+
async connectSocket(e) {
|
|
1891
|
+
const t = await fetch("/api/avatar-render/session", {
|
|
1892
|
+
method: "POST",
|
|
1893
|
+
credentials: "same-origin",
|
|
1894
|
+
headers: { "Content-Type": "application/json" },
|
|
1895
|
+
body: JSON.stringify({ avatarId: this.options.avatarId, surface: this.options.surface ?? "studio", tutorShare: this.options.tutorShare }),
|
|
1896
|
+
signal: this.options.signal
|
|
1897
|
+
}), s = await t.json().catch(() => ({}));
|
|
1898
|
+
if (!t.ok)
|
|
1899
|
+
throw t.status === 404 || s.code === "avatar_unavailable" ? new Z(s.error || "Avatar unavailable") : new T(s.error || "RunPod authorization failed", s.code || "render_unauthorized");
|
|
1900
|
+
if (this.closed || e !== this.generation) throw new DOMException("Avatar preparation aborted", "AbortError");
|
|
1901
|
+
if (s.provider !== "runpod")
|
|
1902
|
+
throw console.warn(`[BeVyra runpod] server provider is ${s.provider}, not runpod`), new T("RunPod provider is not configured", "render_unconfigured");
|
|
1903
|
+
const i = performance.now();
|
|
1904
|
+
console.info(`[BeVyra runpod] connecting avatar=${this.options.avatarId}`);
|
|
1905
|
+
const r = new URL(s.streamPath, window.location.origin);
|
|
1906
|
+
r.protocol = r.protocol === "https:" ? "wss:" : "ws:", r.searchParams.set("ticket", s.ticket);
|
|
1907
|
+
const n = new WebSocket(r);
|
|
1908
|
+
if (n.binaryType = "arraybuffer", this.socket = n, await new Promise((a, c) => {
|
|
1909
|
+
let o = !1;
|
|
1910
|
+
const h = (u) => {
|
|
1911
|
+
o || (o = !0, window.clearTimeout(d), u ? c(u) : a());
|
|
1912
|
+
}, d = window.setTimeout(() => h(new T("RunPod connection timed out", "render_timeout")), 7e4);
|
|
1913
|
+
n.onopen = () => h(), n.onerror = () => h(new T("RunPod connection failed", "render_unavailable")), n.onclose = (u) => h(new T(`RunPod connection closed (${u.code})`, "render_closed"));
|
|
1914
|
+
}).catch((a) => {
|
|
1915
|
+
throw console.warn(`[BeVyra runpod] connection failed after ${Math.round(performance.now() - i)}ms: ${a instanceof Error ? a.message : a}`), n.close(), a;
|
|
1916
|
+
}), console.info(`[BeVyra runpod] connected in ${Math.round(performance.now() - i)}ms`), this.closed || e !== this.generation)
|
|
1917
|
+
throw n.close(), new DOMException("Avatar preparation aborted", "AbortError");
|
|
1918
|
+
return n;
|
|
1919
|
+
}
|
|
1920
|
+
async streamOnce(e, t, s, i) {
|
|
1921
|
+
const r = this.audio;
|
|
1922
|
+
if (!r) throw new Error("Avatar audio output is not initialized.");
|
|
1923
|
+
const n = this.warmSocket?.readyState === WebSocket.OPEN ? this.warmSocket : await this.connectSocket(i);
|
|
1924
|
+
if (this.warmSocket = void 0, this.closed || i !== this.generation) {
|
|
1925
|
+
n.close();
|
|
1926
|
+
return;
|
|
1927
|
+
}
|
|
1928
|
+
n.send(JSON.stringify({ type: "start", avatar: this.options.avatarId, format: "pcm_s16le", sample_rate: He, channels: 1 }));
|
|
1929
|
+
for (let o = 0; o < e.length; o += 8e3)
|
|
1930
|
+
n.send(e.subarray(o, o + 8e3));
|
|
1931
|
+
n.send(JSON.stringify({ type: "end" }));
|
|
1932
|
+
let a = () => {
|
|
1933
|
+
};
|
|
1934
|
+
const c = new Promise((o) => a = o);
|
|
1935
|
+
this.wakers.add(a), await Promise.race([new Promise((o, h) => {
|
|
1936
|
+
let d = !1, u = !1, p = null, f = 0, b = 0, S = 0, g = Promise.resolve();
|
|
1937
|
+
const y = (w) => {
|
|
1938
|
+
d || (d = !0, w ? h(w) : o());
|
|
1939
|
+
}, v = () => {
|
|
1940
|
+
if (u || i !== this.generation || this.closed) return;
|
|
1941
|
+
u = !0;
|
|
1942
|
+
const w = r.createBufferSource();
|
|
1943
|
+
w.buffer = t, w.connect(r.destination), this.source = w;
|
|
1944
|
+
const m = r.currentTime + 0.05;
|
|
1945
|
+
w.start(m), this.after(50, () => {
|
|
1946
|
+
i === this.generation && this.speechStartListener?.();
|
|
1947
|
+
}), s.forEach((A, E) => this.after(
|
|
1948
|
+
Math.max(0, f + (A.startSample - s[0].startSample) / A.sampleRate * 1e3 - performance.now()),
|
|
1949
|
+
() => {
|
|
1950
|
+
i === this.generation && this.clauseStartListener?.(E, performance.now());
|
|
1951
|
+
}
|
|
1952
|
+
)), w.onended = () => {
|
|
1953
|
+
w.disconnect(), this.source === w && (this.source = void 0), i === this.generation && this.speechEndListener?.(), y();
|
|
1954
|
+
};
|
|
1955
|
+
};
|
|
1956
|
+
n.onmessage = (w) => {
|
|
1957
|
+
if (!(d || i !== this.generation || this.closed))
|
|
1958
|
+
if (typeof w.data == "string")
|
|
1959
|
+
try {
|
|
1960
|
+
const m = JSON.parse(w.data);
|
|
1961
|
+
m.type === "busy" || m.type === "error" ? console.warn(`[BeVyra runpod] ${m.type}${m.message ? `: ${m.message}` : ""}`) : m.type === "done" && console.info(`[BeVyra runpod] done frames=${b}`), m.type === "busy" ? y(new T("RunPod worker is busy", "busy")) : m.type === "error" ? y(new T(String(m.message || "RunPod render failed"), "render_error")) : m.type === "frame" ? p = { pts_ms: Number(m.pts_ms), bytes: Number(m.bytes) } : m.type === "done" && b === 0 && y(new T("RunPod returned no frames", "render_empty"));
|
|
1962
|
+
} catch (m) {
|
|
1963
|
+
y(m instanceof Error ? m : new Error(String(m)));
|
|
1964
|
+
}
|
|
1965
|
+
else {
|
|
1966
|
+
const m = p;
|
|
1967
|
+
if (p = null, !m || !Number.isFinite(m.pts_ms) || !(w.data instanceof ArrayBuffer) || w.data.byteLength !== m.bytes)
|
|
1968
|
+
return y(new Error("Malformed RunPod frame"));
|
|
1969
|
+
b++, f || (f = performance.now() + de + 50, this.after(de, v)), g = g.then(async () => {
|
|
1970
|
+
const A = await createImageBitmap(new Blob([w.data], { type: "image/jpeg" }));
|
|
1971
|
+
if (i !== this.generation || this.closed || d) {
|
|
1972
|
+
A.close();
|
|
1973
|
+
return;
|
|
1974
|
+
}
|
|
1975
|
+
const E = f + m.pts_ms, k = Math.max(0, E - performance.now());
|
|
1976
|
+
this.after(k, () => {
|
|
1977
|
+
i === this.generation && !this.closed && !d && (performance.now() - E < 80 ? (this.compositor.draw(A), this.hasVideo = !0, S && (this.state.presentedFps = 1e3 / (performance.now() - S)), S = performance.now()) : this.state.droppedFrames++), A.close();
|
|
1978
|
+
});
|
|
1979
|
+
}).catch(y);
|
|
1980
|
+
}
|
|
1981
|
+
}, n.onclose = (w) => {
|
|
1982
|
+
b || y(new T(w.reason || "RunPod closed before video", "render_closed"));
|
|
1983
|
+
}, n.onerror = () => y(new T("RunPod transport failed", "render_unavailable"));
|
|
1984
|
+
}), c]).finally(() => {
|
|
1985
|
+
this.wakers.delete(a), n.readyState === WebSocket.OPEN && n.close(1e3, "turn complete"), this.socket === n && (this.socket = void 0);
|
|
1986
|
+
});
|
|
1987
|
+
}
|
|
1988
|
+
interrupt() {
|
|
1989
|
+
this.generation++, this.chunks = [], this.socket?.close(1e3, "interrupted"), this.socket = void 0, this.warmSocket = void 0;
|
|
1990
|
+
try {
|
|
1991
|
+
this.source?.stop();
|
|
1992
|
+
} catch {
|
|
1993
|
+
}
|
|
1994
|
+
this.source = void 0, this.timers.forEach((e) => window.clearTimeout(e)), this.timers.clear(), this.wakeAll();
|
|
1995
|
+
}
|
|
1996
|
+
synchronizeGeneration(e) {
|
|
1997
|
+
if (!Number.isSafeInteger(e) || e < 0) throw new Error("Invalid generation");
|
|
1998
|
+
e !== this.generation && (this.interrupt(), this.generation = e);
|
|
1999
|
+
}
|
|
2000
|
+
async generate() {
|
|
2001
|
+
throw new T("RunPod rendering supports live sessions only", "generate_unsupported");
|
|
2002
|
+
}
|
|
2003
|
+
close() {
|
|
2004
|
+
this.closed || (this.closed = !0, this.interrupt(), this.compositor.close(), this.audio?.close().catch(() => {
|
|
2005
|
+
}));
|
|
2006
|
+
}
|
|
2007
|
+
after(e, t) {
|
|
2008
|
+
const s = window.setTimeout(() => {
|
|
2009
|
+
this.timers.delete(s), t();
|
|
2010
|
+
}, e);
|
|
2011
|
+
this.timers.add(s);
|
|
2012
|
+
}
|
|
2013
|
+
wakeAll() {
|
|
2014
|
+
this.wakers.forEach((e) => e()), this.wakers.clear();
|
|
2015
|
+
}
|
|
2016
|
+
}
|
|
2017
|
+
class se extends Error {
|
|
2018
|
+
constructor(e) {
|
|
2019
|
+
super(e), this.code = "AVATAR_PLANNER_INCOMPATIBLE", this.name = "AvatarPlannerCompatibilityError";
|
|
2020
|
+
}
|
|
2021
|
+
}
|
|
2022
|
+
const $ = "portable-ort-v2", H = "self-supervised-whole-sequence-v1", W = "portable-ort-v1", Ge = "bevyra-client-avatar-v1", je = "bc8dce2ea50a56e45dd1353fd8f8a6a0d30701ed8827d220e7957cb7d9207cc1", qe = ["web-bundle.json", "manifest.json", "portable-manifest.json", "model.json", "runtime/runtime-build.json"];
|
|
2023
|
+
async function z(l, e, t, s, i = {}) {
|
|
2024
|
+
const r = await fetch(`/api/v1/avatar-bundles/${encodeURIComponent(l)}/download`, {
|
|
2025
|
+
method: "POST",
|
|
2026
|
+
credentials: "same-origin",
|
|
2027
|
+
headers: { "Content-Type": "application/json" },
|
|
2028
|
+
body: JSON.stringify({ surface: e, ...s ? { runtime: s } : {}, ...i }),
|
|
2029
|
+
signal: t
|
|
2030
|
+
}), n = await r.json().catch(() => ({}));
|
|
2031
|
+
if (!r.ok)
|
|
2032
|
+
throw n.code === "AVATAR_PLANNER_INCOMPATIBLE" || r.status === 409 ? new se(n.error || "Avatar planner contract is incompatible.") : new Error(n.error || "Bundle authorization failed.");
|
|
2033
|
+
return n;
|
|
2034
|
+
}
|
|
2035
|
+
async function pe(l, e, t, s = {}) {
|
|
2036
|
+
const i = await fetch("/api/v1/avatar-runtime/download", {
|
|
2037
|
+
method: "POST",
|
|
2038
|
+
credentials: "same-origin",
|
|
2039
|
+
headers: { "Content-Type": "application/json" },
|
|
2040
|
+
body: JSON.stringify({ avatarId: l, surface: e, ...s }),
|
|
2041
|
+
signal: t
|
|
2042
|
+
}), r = await i.json().catch(() => ({}));
|
|
2043
|
+
if (!i.ok) throw new Error(r.error || "Runtime authorization failed.");
|
|
2044
|
+
return r;
|
|
2045
|
+
}
|
|
2046
|
+
function ze(l) {
|
|
2047
|
+
return Array.from(new Uint8Array(l), (e) => e.toString(16).padStart(2, "0")).join("");
|
|
2048
|
+
}
|
|
2049
|
+
async function fe(l, e, t) {
|
|
2050
|
+
if (e.byteLength !== t.bytes) throw new Error(`Bundle size mismatch: ${l}.`);
|
|
2051
|
+
if (ze(await crypto.subtle.digest("SHA-256", e)) !== t.sha256.toLowerCase()) throw new Error(`Bundle checksum mismatch: ${l}.`);
|
|
2052
|
+
return e;
|
|
2053
|
+
}
|
|
2054
|
+
function Ke(l, e, t) {
|
|
2055
|
+
return new Request(`${location.origin}/__verified-avatar-cache/${encodeURIComponent(l)}/${e}?sha256=${t}`);
|
|
2056
|
+
}
|
|
2057
|
+
async function K(l, e, t, s = !0) {
|
|
2058
|
+
const i = l.files[e];
|
|
2059
|
+
if (!i) throw new Error(`Bundle contract is missing ${e}.`);
|
|
2060
|
+
const r = s && "caches" in globalThis ? await caches.open(Ge) : null, n = Ke(l.bundleId, e, i.sha256), a = await r?.match(n);
|
|
2061
|
+
if (a)
|
|
2062
|
+
try {
|
|
2063
|
+
return { bytes: await fe(e, await a.arrayBuffer(), i), hit: !0 };
|
|
2064
|
+
} catch {
|
|
2065
|
+
await r?.delete(n);
|
|
2066
|
+
}
|
|
2067
|
+
const c = await fetch(i.url, { cache: "no-store", signal: t });
|
|
2068
|
+
if (!c.ok) {
|
|
2069
|
+
const h = c.status === 401 || c.status === 403;
|
|
2070
|
+
throw Object.assign(new Error(h ? "Presigned URL expired." : `Bundle download failed (${c.status}): ${e}.`), { expired: h });
|
|
2071
|
+
}
|
|
2072
|
+
const o = await fe(e, await c.arrayBuffer(), i);
|
|
2073
|
+
return await r?.put(n, new Response(o, { headers: { "Content-Type": e.endsWith(".json") ? "application/json" : "application/wasm" } })), { bytes: o, hit: !1 };
|
|
2074
|
+
}
|
|
2075
|
+
function Se(l, e) {
|
|
2076
|
+
try {
|
|
2077
|
+
return JSON.parse(new TextDecoder().decode(l));
|
|
2078
|
+
} catch {
|
|
2079
|
+
throw new Error(`Bundle metadata is invalid JSON: ${e}.`);
|
|
2080
|
+
}
|
|
2081
|
+
}
|
|
2082
|
+
function Je(l) {
|
|
2083
|
+
const e = l["web-bundle.json"];
|
|
2084
|
+
if (e?.version !== 1) throw new Error(`Avatar bundle requires version ${e?.version ?? "unknown"}, but this website supports version 1.`);
|
|
2085
|
+
if (e?.format !== W) throw new Error(`Avatar bundle requires ${e?.format ?? "an unknown format"}, but this website supports ${W}.`);
|
|
2086
|
+
const t = l["portable-manifest.json"]?.portableRuntime ?? l["portable-manifest.json"];
|
|
2087
|
+
if (t?.format !== W) throw new Error(`Unsupported portable runtime version: ${t?.format ?? "missing"}.`);
|
|
2088
|
+
if (t?.compositor?.mode !== "gaussian-only-v1") throw new Error(`Unsupported compositor: ${t?.compositor?.mode ?? "missing"}.`);
|
|
2089
|
+
if ((t?.audioEncoderUrl ?? l["portable-manifest.json"]?.audioEncoderUrl) !== "embedded") throw new Error("Embedded audio encoder missing.");
|
|
2090
|
+
const s = t?.embeddingShape;
|
|
2091
|
+
if (!Array.isArray(s) || s.length !== 3 || s[1] !== 2 || s[2] !== 1024) throw new Error("Unsupported embedded audio-encoder output shape.");
|
|
2092
|
+
const i = l["runtime/runtime-build.json"], r = i?.webAssets?.find?.((n) => n?.file === "avatar-wasm.js");
|
|
2093
|
+
if (i?.format !== W || r?.sha256 !== je) throw new Error("Avatar runtime requires a different WASM loader build than this website supports.");
|
|
2094
|
+
}
|
|
2095
|
+
async function pt(l, e, t, s) {
|
|
2096
|
+
t("authorizing_bundle");
|
|
2097
|
+
let i = await z(l, e, s);
|
|
2098
|
+
const r = /* @__PURE__ */ new Map();
|
|
2099
|
+
let n = 0, a = 0, c = !1, o = 0;
|
|
2100
|
+
const h = performance.now(), d = async (p) => {
|
|
2101
|
+
for (; ; )
|
|
2102
|
+
try {
|
|
2103
|
+
t("downloading_bundle");
|
|
2104
|
+
const f = await K(i, p, s);
|
|
2105
|
+
return f.hit ? n++ : a++, o += f.hit ? 0 : f.bytes.byteLength, r.set(p, f.bytes), f.bytes;
|
|
2106
|
+
} catch (f) {
|
|
2107
|
+
if (!f.expired || c) throw f;
|
|
2108
|
+
c = !0, i = await z(l, e, s);
|
|
2109
|
+
}
|
|
2110
|
+
}, u = {};
|
|
2111
|
+
for (const p of qe) u[p] = Se(await d(p), p);
|
|
2112
|
+
return t("verifying_bundle"), Je(u), await d("runtime/avatar-wasm.wasm"), {
|
|
2113
|
+
contract: i,
|
|
2114
|
+
files: r,
|
|
2115
|
+
manifests: u,
|
|
2116
|
+
bytes: o,
|
|
2117
|
+
downloadMs: performance.now() - h,
|
|
2118
|
+
cache: n && a ? "mixed" : n ? "hit" : "miss",
|
|
2119
|
+
bundleSha256: i.files["runtime/avatar-wasm.wasm"].sha256
|
|
2120
|
+
};
|
|
2121
|
+
}
|
|
2122
|
+
const Qe = [
|
|
2123
|
+
"web-bundle.json",
|
|
2124
|
+
"manifest.json",
|
|
2125
|
+
"portable-manifest.json",
|
|
2126
|
+
"model.json",
|
|
2127
|
+
"p0.bin",
|
|
2128
|
+
"p1.bin",
|
|
2129
|
+
"p2.bin",
|
|
2130
|
+
"p4.bin",
|
|
2131
|
+
"p5.bin",
|
|
2132
|
+
"p6.bin",
|
|
2133
|
+
"p9.bin",
|
|
2134
|
+
"p10.bin",
|
|
2135
|
+
"p11.bin",
|
|
2136
|
+
"p12.bin",
|
|
2137
|
+
"p21.bin"
|
|
2138
|
+
];
|
|
2139
|
+
function Xe(l) {
|
|
2140
|
+
const e = l["web-bundle.json"], t = l["portable-manifest.json"], s = t?.portableRuntime, i = (r) => {
|
|
2141
|
+
throw new se(
|
|
2142
|
+
`Avatar bundle is incompatible with planner ${H}: ${r}`
|
|
2143
|
+
);
|
|
2144
|
+
};
|
|
2145
|
+
(e?.version !== 1 || e?.format !== $) && i(`expected ${$} bundle index`), s?.format !== $ && i("portable runtime format is missing or unsupported"), s?.planners?.available?.includes(H) || i(`learned planner ${H} is unavailable`), s?.compositor?.contextPaddingSourcePixels !== 5 && i(`context padding is ${s?.compositor?.contextPaddingSourcePixels ?? "missing"}, expected 5`), (s?.compositor?.edgeFeatherPixels !== 0 || s?.m1?.model !== "p0.bin" || s?.m1?.visualChannels !== 6 || t?.correctorArchitecture !== "student-corrector-v3" || t?.correctorUrl !== "p1.bin" || t?.faceMattesUrl !== "p21.bin") && i("M1 plus student-v3 inference assets are incomplete"), s?.compositor?.audioContext != null && s.compositor.audioContext !== "minus4-through-plus4" && i("audio context must be the normal minus4-through-plus4 window"), t?.segmentPlannerVersion !== 1 && i("segmentPlannerVersion must be 1"), t?.segmentsUrl !== "p12.bin" && i("contiguous segments must be p12.bin"), (t?.motionPlannerVersion !== 1 || t?.motionPlannerModelUrl !== "p10.bin" || t?.motionPlannerUrl !== "p11.bin") && i("learned planner assets must be p10.bin and p11.bin"), t?.motionPlannerV2Version != null && (t.motionPlannerV2Version !== 2 || t.motionPlannerV2Url !== "p22.bin" || !s?.planners?.available?.includes("contiguous-trajectory-v2")) && i("trajectory planner v2 metadata is invalid"), t?.teacherCorrectorUrl != null && t.teacherCorrectorUrl !== "p23.bin" && i("teacher corrector asset must be p23.bin");
|
|
2146
|
+
}
|
|
2147
|
+
async function Ye(l, e, t, s, i = {}) {
|
|
2148
|
+
t("authorizing_bundle");
|
|
2149
|
+
let r = await z(l, e, s, "rust-v2", i);
|
|
2150
|
+
if (r.format !== $)
|
|
2151
|
+
throw new se(`Rust core requires ${$}, received ${r.format}.`);
|
|
2152
|
+
const n = /* @__PURE__ */ new Map();
|
|
2153
|
+
let a = 0, c = 0, o = !1, h = 0;
|
|
2154
|
+
const d = performance.now(), u = async (f) => {
|
|
2155
|
+
for (; ; )
|
|
2156
|
+
try {
|
|
2157
|
+
t("downloading_bundle");
|
|
2158
|
+
const b = await K(r, f, s, !1);
|
|
2159
|
+
return b.hit ? a++ : c++, h += b.hit ? 0 : b.bytes.byteLength, n.set(f, b.bytes), b.bytes;
|
|
2160
|
+
} catch (b) {
|
|
2161
|
+
if (!b.expired || o) throw b;
|
|
2162
|
+
o = !0, r = await z(l, e, s, "rust-v2", i);
|
|
2163
|
+
}
|
|
2164
|
+
}, p = {};
|
|
2165
|
+
for (const f of Qe) {
|
|
2166
|
+
const b = await u(f);
|
|
2167
|
+
f.endsWith(".json") && (p[f] = Se(b, f));
|
|
2168
|
+
}
|
|
2169
|
+
return p["portable-manifest.json"]?.motionPlannerV2Version === 2 && await u("p22.bin"), p["portable-manifest.json"]?.visualPlannerDescriptorUrl === "p24.bin" && await u("p24.bin"), p["portable-manifest.json"]?.teacherCorrectorUrl === "p23.bin" && await u("p23.bin"), t("verifying_bundle"), Xe(p), {
|
|
2170
|
+
contract: r,
|
|
2171
|
+
files: n,
|
|
2172
|
+
manifests: p,
|
|
2173
|
+
bytes: h,
|
|
2174
|
+
downloadMs: performance.now() - d,
|
|
2175
|
+
cache: a && c ? "mixed" : a ? "hit" : "miss",
|
|
2176
|
+
bundleSha256: r.bundleId
|
|
2177
|
+
};
|
|
2178
|
+
}
|
|
2179
|
+
async function Ze(l, e, t, s, i = {}) {
|
|
2180
|
+
t("authorizing_bundle");
|
|
2181
|
+
let r = await pe(l, e, s, i);
|
|
2182
|
+
if (r.schema !== "bevyra-rust-browser-core-v1" || !r.capabilities?.bundleFormats?.includes("portable-ort-v2") || r.capabilities.planner !== H || r.capabilities.audioFrontend !== "clause-encoder-v1" || r.capabilities.simd128 !== !0) throw new Error("Rust runtime contract is incompatible.");
|
|
2183
|
+
const n = performance.now();
|
|
2184
|
+
let a = !1;
|
|
2185
|
+
for (; ; )
|
|
2186
|
+
try {
|
|
2187
|
+
t("downloading_bundle");
|
|
2188
|
+
const c = {
|
|
2189
|
+
bundleId: r.runtimeId,
|
|
2190
|
+
avatarId: "runtime",
|
|
2191
|
+
format: r.schema,
|
|
2192
|
+
expiresAt: r.expiresAt,
|
|
2193
|
+
entrypoint: "bevyra-core.wasm",
|
|
2194
|
+
files: r.files
|
|
2195
|
+
}, [o, h] = await Promise.all([
|
|
2196
|
+
K(c, "bevyra-core.wasm", s, !1),
|
|
2197
|
+
K(c, "closure_router.onnx", s, !1)
|
|
2198
|
+
]);
|
|
2199
|
+
return t("verifying_bundle"), {
|
|
2200
|
+
contract: r,
|
|
2201
|
+
coreWasm: o.bytes,
|
|
2202
|
+
closureRouter: h.bytes,
|
|
2203
|
+
cache: o.hit ? "hit" : "miss",
|
|
2204
|
+
bytes: (o.hit ? 0 : o.bytes.byteLength) + (h.hit ? 0 : h.bytes.byteLength),
|
|
2205
|
+
downloadMs: performance.now() - n
|
|
2206
|
+
};
|
|
2207
|
+
} catch (c) {
|
|
2208
|
+
if (!c.expired || a) throw c;
|
|
2209
|
+
a = !0, r = await pe(l, e, s, i);
|
|
2210
|
+
}
|
|
2211
|
+
}
|
|
2212
|
+
class me {
|
|
2213
|
+
constructor(e, t = 16e3) {
|
|
2214
|
+
if (this.sourceRate = e, this.targetRate = t, this.samples = new Float32Array(0), this.sourceBase = 0, this.sourceEnd = 0, this.outputIndex = 0, e <= 0 || t <= 0)
|
|
2215
|
+
throw new Error("Audio sample rates must be positive.");
|
|
2216
|
+
}
|
|
2217
|
+
push(e) {
|
|
2218
|
+
if (e.length) {
|
|
2219
|
+
const c = new Float32Array(this.samples.length + e.length);
|
|
2220
|
+
c.set(this.samples);
|
|
2221
|
+
for (let o = 0; o < e.length; o++)
|
|
2222
|
+
c[this.samples.length + o] = e[o] / 32768;
|
|
2223
|
+
this.samples = c, this.sourceEnd += e.length;
|
|
2224
|
+
}
|
|
2225
|
+
if (!this.samples.length) return new Float32Array(0);
|
|
2226
|
+
const t = this.sourceEnd - 1, s = Math.floor(t * this.targetRate / this.sourceRate);
|
|
2227
|
+
if (s < this.outputIndex) return new Float32Array(0);
|
|
2228
|
+
const i = new Float32Array(s - this.outputIndex + 1);
|
|
2229
|
+
for (let c = 0; c < i.length; c++) {
|
|
2230
|
+
const o = (this.outputIndex + c) * this.sourceRate / this.targetRate, h = o - this.sourceBase, d = Math.max(0, Math.min(this.samples.length - 1, Math.floor(h))), u = Math.min(this.samples.length - 1, d + 1), p = h - Math.floor(h);
|
|
2231
|
+
i[c] = this.samples[d] * (1 - p) + this.samples[u] * p;
|
|
2232
|
+
}
|
|
2233
|
+
this.outputIndex = s + 1;
|
|
2234
|
+
const r = this.outputIndex * this.sourceRate / this.targetRate, n = Math.max(this.sourceBase, Math.floor(r)), a = Math.min(n - this.sourceBase, this.samples.length);
|
|
2235
|
+
return a > 0 && (this.samples = this.samples.slice(a), this.sourceBase += a), i;
|
|
2236
|
+
}
|
|
2237
|
+
}
|
|
2238
|
+
class et {
|
|
2239
|
+
constructor(e = 25, t = 4e-3, s = 0.1, i = 20, r = 20) {
|
|
2240
|
+
this.pauseFrames = e, this.absoluteFloor = t, this.relativeFloor = s, this.frameSamples = 640, this.pending = new Float32Array(0), this.analyzed = 0, this.level = 0, this.speechSeen = !1, this.quietFrames = 0, this.generation = 0, this.turnId = 0, this.clauseId = 0, this.releasedSamples = 0;
|
|
2241
|
+
}
|
|
2242
|
+
get vadQuietThresholdFrames() {
|
|
2243
|
+
return this.pauseFrames;
|
|
2244
|
+
}
|
|
2245
|
+
push(e, t, s = this.generation) {
|
|
2246
|
+
if (s !== this.generation) return [];
|
|
2247
|
+
if (this.sourceRate !== void 0 && this.sourceRate !== t)
|
|
2248
|
+
throw new Error("Assistant audio sample rate changed within a turn.");
|
|
2249
|
+
this.resampler || (this.sourceRate = t, this.resampler = new me(t)), this.append(this.resampler.push(e));
|
|
2250
|
+
const i = [];
|
|
2251
|
+
for (; this.analyzed + this.frameSamples <= this.pending.length; ) {
|
|
2252
|
+
const r = this.analyzed + this.frameSamples;
|
|
2253
|
+
let n = 0;
|
|
2254
|
+
for (let o = this.analyzed; o < r; o++)
|
|
2255
|
+
n += this.pending[o] * this.pending[o];
|
|
2256
|
+
const a = Math.sqrt(n / this.frameSamples);
|
|
2257
|
+
this.level = Math.max(this.level * 0.995, a);
|
|
2258
|
+
const c = Math.max(this.absoluteFloor, this.relativeFloor * this.level);
|
|
2259
|
+
a >= c ? (this.speechSeen = !0, this.quietFrames = 0) : this.speechSeen && this.quietFrames++, this.analyzed = r, this.speechSeen && this.quietFrames >= this.pauseFrames && i.push(this.release(r, "vad", this.pauseFrames));
|
|
2260
|
+
}
|
|
2261
|
+
return i;
|
|
2262
|
+
}
|
|
2263
|
+
/**
|
|
2264
|
+
* Releases already-segmented speech (e.g. pre-recorded narration split at
|
|
2265
|
+
* its pauses) as one complete clause, without waiting for a VAD pause.
|
|
2266
|
+
*/
|
|
2267
|
+
pushClause(e, t, s = this.generation) {
|
|
2268
|
+
return s !== this.generation || !e.length ? [] : (this.append(new me(t).push(e)), [this.release(this.pending.length, "vad", 0)]);
|
|
2269
|
+
}
|
|
2270
|
+
finishTurn(e = this.generation) {
|
|
2271
|
+
if (e !== this.generation) return [];
|
|
2272
|
+
const t = [];
|
|
2273
|
+
if (this.pending.length && (this.speechSeen || this.terminalContainsSpeech())) {
|
|
2274
|
+
const s = Math.max(4 * this.frameSamples, this.pending.length), i = new Float32Array(s);
|
|
2275
|
+
i.set(this.pending), this.pending = i, t.push(this.release(s, "terminal", this.quietFrames));
|
|
2276
|
+
}
|
|
2277
|
+
return this.resetAudio(), this.turnId++, this.releasedSamples = 0, t;
|
|
2278
|
+
}
|
|
2279
|
+
interrupt() {
|
|
2280
|
+
return this.generation++, this.turnId++, this.releasedSamples = 0, this.resetAudio(), this.generation;
|
|
2281
|
+
}
|
|
2282
|
+
append(e) {
|
|
2283
|
+
if (!e.length) return;
|
|
2284
|
+
const t = new Float32Array(this.pending.length + e.length);
|
|
2285
|
+
t.set(this.pending), t.set(e, this.pending.length), this.pending = t;
|
|
2286
|
+
}
|
|
2287
|
+
terminalContainsSpeech() {
|
|
2288
|
+
let e = 0;
|
|
2289
|
+
for (let t = 0; t < this.pending.length; t += this.frameSamples) {
|
|
2290
|
+
const s = Math.min(this.pending.length, t + this.frameSamples);
|
|
2291
|
+
let i = 0;
|
|
2292
|
+
for (let r = t; r < s; r++)
|
|
2293
|
+
i += this.pending[r] * this.pending[r];
|
|
2294
|
+
e = Math.max(e, Math.sqrt(i / Math.max(1, s - t)));
|
|
2295
|
+
}
|
|
2296
|
+
return e >= Math.max(this.absoluteFloor, this.relativeFloor * this.level);
|
|
2297
|
+
}
|
|
2298
|
+
release(e, t, s) {
|
|
2299
|
+
const i = this.pending.slice(0, e), r = new Int16Array(i.length);
|
|
2300
|
+
for (let a = 0; a < i.length; a++)
|
|
2301
|
+
r[a] = Math.max(-32768, Math.min(32767, Math.round(i[a] * 32768)));
|
|
2302
|
+
const n = this.releasedSamples;
|
|
2303
|
+
return this.releasedSamples += r.length, this.clauseId++, this.pending = this.pending.slice(e), this.analyzed = 0, t !== "rolling" && (this.speechSeen = !1, this.quietFrames = 0), {
|
|
2304
|
+
generation: this.generation,
|
|
2305
|
+
turnId: this.turnId,
|
|
2306
|
+
clauseId: this.clauseId,
|
|
2307
|
+
startSample: n,
|
|
2308
|
+
endSample: this.releasedSamples,
|
|
2309
|
+
sampleRate: 16e3,
|
|
2310
|
+
pcm: r,
|
|
2311
|
+
boundary: t,
|
|
2312
|
+
quietFrames: s
|
|
2313
|
+
};
|
|
2314
|
+
}
|
|
2315
|
+
resetAudio() {
|
|
2316
|
+
this.pending = new Float32Array(0), this.analyzed = 0, this.level = 0, this.speechSeen = !1, this.quietFrames = 0, this.resampler = void 0, this.sourceRate = void 0;
|
|
2317
|
+
}
|
|
2318
|
+
}
|
|
2319
|
+
class X extends Error {
|
|
2320
|
+
constructor(e) {
|
|
2321
|
+
super(`On-device avatar inference is unsupported: ${e}`), this.reason = e, this.code = "AVATAR_DEVICE_UNSUPPORTED", this.name = "AvatarDeviceUnsupportedError";
|
|
2322
|
+
}
|
|
2323
|
+
}
|
|
2324
|
+
function tt(l) {
|
|
2325
|
+
if (typeof Worker > "u" || typeof WebAssembly > "u")
|
|
2326
|
+
throw new X("Web Workers or WebAssembly are unavailable");
|
|
2327
|
+
if (typeof VideoDecoder > "u")
|
|
2328
|
+
throw new X("WebCodecs VideoDecoder is unavailable");
|
|
2329
|
+
const e = l.executionBackend ?? "auto", t = e !== "wasm" && window.isSecureContext && "gpu" in navigator, s = window.crossOriginIsolated && typeof SharedArrayBuffer < "u" && (navigator.hardwareConcurrency || 1) >= 4;
|
|
2330
|
+
if (!t && !s)
|
|
2331
|
+
throw new X(
|
|
2332
|
+
"neither WebGPU nor isolated threaded WASM is available"
|
|
2333
|
+
);
|
|
2334
|
+
}
|
|
2335
|
+
class G {
|
|
2336
|
+
constructor(e, t) {
|
|
2337
|
+
this.runtime = e, this.mode = t, this.generation = 0, this.turn = 0, this.clauses = new et(), this.releaseChain = Promise.resolve(), this.handoff = null, this.listeners = {
|
|
2338
|
+
speechStart: /* @__PURE__ */ new Set(),
|
|
2339
|
+
speechEnd: /* @__PURE__ */ new Set(),
|
|
2340
|
+
clauseStart: /* @__PURE__ */ new Set(),
|
|
2341
|
+
failure: /* @__PURE__ */ new Set()
|
|
2342
|
+
}, this.detach = [], this.audioEnabled = !1, this.speaking = !1, this.turnOpen = !1, this.clock = { open: !1, received: 0, clauses: [], starts: [] }, this.responses = 0, t === "stream" && console.info(`[BeVyra SDK] VAD_THRESHOLD quiet_frames=${this.clauses.vadQuietThresholdFrames} quiet_ms=${this.clauses.vadQuietThresholdFrames * 40}`), this.attach();
|
|
2343
|
+
}
|
|
2344
|
+
attach() {
|
|
2345
|
+
const e = this.runtime;
|
|
2346
|
+
this.detach = [
|
|
2347
|
+
e.onSpeechStart(() => {
|
|
2348
|
+
this.speaking = !0, this.listeners.speechStart.forEach((t) => t());
|
|
2349
|
+
}),
|
|
2350
|
+
e.onSpeechEnd(() => {
|
|
2351
|
+
this.speaking = !1, this.listeners.speechEnd.forEach((t) => t());
|
|
2352
|
+
}),
|
|
2353
|
+
e.onClauseStart((t, s) => {
|
|
2354
|
+
this.clock.starts[t] = s, this.listeners.clauseStart.forEach((i) => i(t, s));
|
|
2355
|
+
}),
|
|
2356
|
+
e.onFailure?.((t) => this.listeners.failure.forEach((s) => s(t))) ?? (() => {
|
|
2357
|
+
})
|
|
2358
|
+
];
|
|
2359
|
+
}
|
|
2360
|
+
subscribe(e, t) {
|
|
2361
|
+
return e.add(t), () => void e.delete(t);
|
|
2362
|
+
}
|
|
2363
|
+
/** Nothing is being spoken or queued: a safe point to change renderer. */
|
|
2364
|
+
get idle() {
|
|
2365
|
+
return !this.speaking && !this.turnOpen;
|
|
2366
|
+
}
|
|
2367
|
+
/**
|
|
2368
|
+
* Moves this stream onto another renderer (for example the hosted one when
|
|
2369
|
+
* the device falls behind) at the next quiet moment, keeping the live AI
|
|
2370
|
+
* session and every listener. The other SDK's runtime is adopted and the
|
|
2371
|
+
* current one closed.
|
|
2372
|
+
*/
|
|
2373
|
+
async adoptRenderer(e, t) {
|
|
2374
|
+
if (this.mode !== "stream" || e.mode !== "stream")
|
|
2375
|
+
throw new Error("Only stream renderers can be swapped.");
|
|
2376
|
+
for (; !this.idle; ) {
|
|
2377
|
+
if (t?.aborted) return e.close();
|
|
2378
|
+
await new Promise((i) => setTimeout(i, 150));
|
|
2379
|
+
}
|
|
2380
|
+
if (t?.aborted) return e.close();
|
|
2381
|
+
e.detach.forEach((i) => i());
|
|
2382
|
+
const s = this.runtime;
|
|
2383
|
+
this.detach.forEach((i) => i()), e.runtime.synchronizeGeneration?.(this.generation), this.runtime = e.runtime, this.attach(), this.releaseChain = Promise.resolve(), s.close(), this.audioEnabled && await this.runtime.enableAudio();
|
|
2384
|
+
}
|
|
2385
|
+
static async create(e) {
|
|
2386
|
+
const t = e.onStage ?? (() => {
|
|
2387
|
+
}), s = e.surface ?? "studio";
|
|
2388
|
+
if (e.renderer === "remote") {
|
|
2389
|
+
if (e.mode !== "stream")
|
|
2390
|
+
throw new Error("The remote renderer supports stream mode only.");
|
|
2391
|
+
const o = await fetch("/api/avatar-render/provider", { credentials: "same-origin", signal: e.signal }), h = o.ok ? (await o.json()).provider : "ecs";
|
|
2392
|
+
console.info(`[BeVyra render] provider=${h} avatar=${e.avatarId} surface=${s}${o.ok ? "" : ` (provider lookup failed: ${o.status})`}`);
|
|
2393
|
+
const d = await (h === "runpod" ? te : q).create({
|
|
2394
|
+
avatarId: e.avatarId,
|
|
2395
|
+
canvas: e.canvas,
|
|
2396
|
+
surface: s,
|
|
2397
|
+
tutorShare: e.tutorShare,
|
|
2398
|
+
resolution: e.resolution,
|
|
2399
|
+
body: e.body,
|
|
2400
|
+
onStage: t,
|
|
2401
|
+
signal: e.signal,
|
|
2402
|
+
audioContext: e.audioContext
|
|
2403
|
+
});
|
|
2404
|
+
return new G(d, e.mode);
|
|
2405
|
+
}
|
|
2406
|
+
tt(e);
|
|
2407
|
+
const i = e.assetProvider ? await e.assetProvider() : null, [r, n] = i ? [i.bundle, i.runtime] : await Promise.all([
|
|
2408
|
+
Ye(e.avatarId, s, t, e.signal, { tutorShare: e.tutorShare }),
|
|
2409
|
+
Ze(e.avatarId, s, t, e.signal, { tutorShare: e.tutorShare })
|
|
2410
|
+
]), a = {
|
|
2411
|
+
mode: e.mode,
|
|
2412
|
+
background: e.background,
|
|
2413
|
+
resolution: e.resolution,
|
|
2414
|
+
body: e.body,
|
|
2415
|
+
ortWasmPaths: e.ortWasmPaths ?? new URL("/ort/", window.location.origin).href,
|
|
2416
|
+
executionBackend: e.executionBackend,
|
|
2417
|
+
minimumCalibrationFps: e.minimumCalibrationFps
|
|
2418
|
+
}, c = await j.createRustV2(r, n, e.canvas, t, a);
|
|
2419
|
+
return new G(c, e.mode);
|
|
2420
|
+
}
|
|
2421
|
+
get metrics() {
|
|
2422
|
+
return this.runtime.metrics;
|
|
2423
|
+
}
|
|
2424
|
+
async setScene(e, t, s) {
|
|
2425
|
+
await this.runtime.setScene(e, t, s);
|
|
2426
|
+
}
|
|
2427
|
+
/**
|
|
2428
|
+
* Switches a running on-device stream to BeVyra's hosted renderer. The
|
|
2429
|
+
* hosted runtime connects on a hidden twin of the canvas, takes over at the
|
|
2430
|
+
* next quiet moment, and only then becomes visible, so the two renderers
|
|
2431
|
+
* never paint over each other. Resolves false if aborted first.
|
|
2432
|
+
*/
|
|
2433
|
+
async moveToHostedRenderer(e, t) {
|
|
2434
|
+
const s = e.canvas, i = s.cloneNode(!1);
|
|
2435
|
+
if (this.handoff) throw new Error("A renderer handoff is already in progress.");
|
|
2436
|
+
let r = () => {
|
|
2437
|
+
};
|
|
2438
|
+
const n = {
|
|
2439
|
+
clauses: [],
|
|
2440
|
+
settled: new Promise((o) => r = o),
|
|
2441
|
+
resolve: () => r()
|
|
2442
|
+
};
|
|
2443
|
+
this.handoff = n;
|
|
2444
|
+
const a = this.releaseChain, c = this.generation;
|
|
2445
|
+
i.dataset.bevyraHostedTwin = "true", i.style.visibility = "hidden", s.after(i);
|
|
2446
|
+
try {
|
|
2447
|
+
const o = G.create({ ...e, canvas: i, mode: "stream", renderer: "remote", signal: t });
|
|
2448
|
+
await a, await this.runtime.completeTurn(!1, c);
|
|
2449
|
+
const h = await o;
|
|
2450
|
+
if (t?.aborted)
|
|
2451
|
+
return this.handoff = null, n.resolve(), h.close(), i.remove(), !1;
|
|
2452
|
+
h.detach.forEach((u) => u());
|
|
2453
|
+
const d = this.runtime;
|
|
2454
|
+
this.detach.forEach((u) => u()), h.runtime.synchronizeGeneration?.(this.generation), this.runtime = h.runtime, this.attach(), this.audioEnabled && await this.runtime.enableAudio(), this.releaseChain = Promise.resolve();
|
|
2455
|
+
for (let u = 0; u < n.clauses.length; u++) {
|
|
2456
|
+
const p = n.clauses[u];
|
|
2457
|
+
this.releaseChain = this.releaseChain.then(() => this.runtime.pushAssistantAudio(p));
|
|
2458
|
+
}
|
|
2459
|
+
return await this.releaseChain, this.handoff = null, n.resolve(), i.style.visibility = "", s.style.visibility = "hidden", d.close(), !0;
|
|
2460
|
+
} catch (o) {
|
|
2461
|
+
if (this.handoff === n) {
|
|
2462
|
+
this.handoff = null;
|
|
2463
|
+
for (const h of n.clauses)
|
|
2464
|
+
this.releaseChain = this.releaseChain.then(() => this.runtime.pushAssistantAudio(h));
|
|
2465
|
+
await this.releaseChain.catch(() => {
|
|
2466
|
+
}), n.resolve();
|
|
2467
|
+
}
|
|
2468
|
+
throw i.remove(), e.audioContext?.close().catch(() => {
|
|
2469
|
+
}), o;
|
|
2470
|
+
}
|
|
2471
|
+
}
|
|
2472
|
+
async enableAudio() {
|
|
2473
|
+
await this.runtime.enableAudio(), this.audioEnabled = !0;
|
|
2474
|
+
}
|
|
2475
|
+
onSpeechStart(e) {
|
|
2476
|
+
return this.subscribe(this.listeners.speechStart, e);
|
|
2477
|
+
}
|
|
2478
|
+
/** Fires when the avatar stops being audible (played out or interrupted). */
|
|
2479
|
+
/** When each clause of the current turn starts playing (index within the turn). */
|
|
2480
|
+
onClauseStart(e) {
|
|
2481
|
+
return this.subscribe(this.listeners.clauseStart, e);
|
|
2482
|
+
}
|
|
2483
|
+
onSpeechEnd(e) {
|
|
2484
|
+
return this.subscribe(this.listeners.speechEnd, e);
|
|
2485
|
+
}
|
|
2486
|
+
/** Renderer failures after creation (the remote gateway connection dropping). */
|
|
2487
|
+
onFailure(e) {
|
|
2488
|
+
return this.subscribe(this.listeners.failure, e);
|
|
2489
|
+
}
|
|
2490
|
+
/**
|
|
2491
|
+
* Seconds of the current response heard so far, against seconds of it
|
|
2492
|
+
* received; null before any audio. `response` counts responses, so a host
|
|
2493
|
+
* can tell a new one from the last; `complete` once all of it is in. Live captions follow it, so they keep
|
|
2494
|
+
* to the voice through rendering stalls and stop where a barge-in cut it.
|
|
2495
|
+
*/
|
|
2496
|
+
speechClock(e = performance.now()) {
|
|
2497
|
+
const { received: t, clauses: s, starts: i } = this.clock;
|
|
2498
|
+
if (!t) return null;
|
|
2499
|
+
let r = 0;
|
|
2500
|
+
return s.forEach((n, a) => {
|
|
2501
|
+
const c = i[a];
|
|
2502
|
+
c !== void 0 && (r += Math.min(n, Math.max(0, (e - c) / 1e3)));
|
|
2503
|
+
}), { response: this.responses, complete: !this.clock.open, receivedSeconds: t, heardSeconds: r };
|
|
2504
|
+
}
|
|
2505
|
+
/** The first audio of a response starts its clock afresh. */
|
|
2506
|
+
countReceived(e) {
|
|
2507
|
+
this.clock.open || (this.clock = { open: !0, received: 0, clauses: [], starts: [] }, this.responses++), this.clock.received += e.pcm.length / e.sampleRate;
|
|
2508
|
+
}
|
|
2509
|
+
get renderer() {
|
|
2510
|
+
return this.runtime instanceof q ? "remote" : "device";
|
|
2511
|
+
}
|
|
2512
|
+
async pushAudio(e) {
|
|
2513
|
+
if (this.mode !== "stream")
|
|
2514
|
+
throw new Error("pushAudio is available only in stream mode.");
|
|
2515
|
+
this.turnOpen = !0, this.countReceived(e), await this.release(
|
|
2516
|
+
this.clauses.push(e.pcm, e.sampleRate, this.generation)
|
|
2517
|
+
);
|
|
2518
|
+
}
|
|
2519
|
+
/** Speech that is already split into clauses (pre-recorded narration). */
|
|
2520
|
+
async pushClause(e) {
|
|
2521
|
+
if (this.mode !== "stream")
|
|
2522
|
+
throw new Error("pushClause is available only in stream mode.");
|
|
2523
|
+
this.turnOpen = !0, this.countReceived(e), await this.release(this.clauses.pushClause(e.pcm, e.sampleRate, this.generation));
|
|
2524
|
+
}
|
|
2525
|
+
release(e) {
|
|
2526
|
+
for (const t of e) {
|
|
2527
|
+
this.clock.clauses.push(t.pcm.length / t.sampleRate), console.info(
|
|
2528
|
+
`[BeVyra SDK] CLAUSE_BOUNDARY generation=${t.generation} turn=${t.turnId} clause=${t.clauseId} reason=${t.boundary} samples=${t.pcm.length} frames=${Math.floor(t.pcm.length / 640)} quiet_frames=${t.quietFrames}`
|
|
2529
|
+
);
|
|
2530
|
+
const s = {
|
|
2531
|
+
sessionId: "sdk-stream",
|
|
2532
|
+
turnId: t.turnId,
|
|
2533
|
+
responseGenerationId: t.generation,
|
|
2534
|
+
startSample: t.startSample,
|
|
2535
|
+
endSample: t.endSample,
|
|
2536
|
+
sampleRate: t.sampleRate,
|
|
2537
|
+
pcm24: t.pcm,
|
|
2538
|
+
clauseId: t.clauseId,
|
|
2539
|
+
clauseBoundary: t.boundary,
|
|
2540
|
+
completeClause: !0
|
|
2541
|
+
};
|
|
2542
|
+
this.handoff ? this.handoff.clauses.push(s) : this.releaseChain = this.releaseChain.then(() => this.runtime.pushAssistantAudio(s));
|
|
2543
|
+
}
|
|
2544
|
+
return this.handoff?.settled ?? this.releaseChain;
|
|
2545
|
+
}
|
|
2546
|
+
async completeTurn() {
|
|
2547
|
+
if (this.mode !== "stream")
|
|
2548
|
+
throw new Error("completeTurn is available only in stream mode.");
|
|
2549
|
+
const e = this.generation, t = this.release(this.clauses.finishTurn(e));
|
|
2550
|
+
this.clock.open = !1, await t, await this.releaseChain, e === this.generation && (await this.runtime.completeTurn(!1, e), e === this.generation && (this.turnOpen = !1), this.turn++);
|
|
2551
|
+
}
|
|
2552
|
+
interrupt() {
|
|
2553
|
+
this.generation++, this.turn++, this.clauses.interrupt(), this.releaseChain = Promise.resolve(), this.turnOpen = !1, this.clock = { open: !1, received: 0, clauses: [], starts: [] }, this.runtime.interrupt();
|
|
2554
|
+
}
|
|
2555
|
+
async generate(e) {
|
|
2556
|
+
if (this.mode !== "generate")
|
|
2557
|
+
throw new Error("generate is available only in generate mode.");
|
|
2558
|
+
return (e.background !== void 0 || e.resolution !== void 0 || e.body !== void 0) && await this.runtime.setScene(e.background, e.resolution, e.body), this.runtime.generate(e.audio, e.mimeType);
|
|
2559
|
+
}
|
|
2560
|
+
close() {
|
|
2561
|
+
this.detach.forEach((e) => e()), this.runtime.close();
|
|
2562
|
+
}
|
|
2563
|
+
}
|
|
2564
|
+
class Ae {
|
|
2565
|
+
constructor() {
|
|
2566
|
+
this.holding = !1, this.generated = !1, this.heard = !1;
|
|
2567
|
+
}
|
|
2568
|
+
get active() {
|
|
2569
|
+
return this.holding;
|
|
2570
|
+
}
|
|
2571
|
+
start(e = 2e4) {
|
|
2572
|
+
this.holding = !0, this.generated = !1, this.heard = !1, window.clearTimeout(this.timer), this.timer = window.setTimeout(() => this.release(), e);
|
|
2573
|
+
}
|
|
2574
|
+
/** The model finished generating a turn. */
|
|
2575
|
+
turnComplete() {
|
|
2576
|
+
this.holding && (this.generated = !0);
|
|
2577
|
+
}
|
|
2578
|
+
/** The renderer reports whether the avatar is audible. */
|
|
2579
|
+
playback(e) {
|
|
2580
|
+
this.holding && (e ? this.heard = !0 : this.generated && this.heard && this.release());
|
|
2581
|
+
}
|
|
2582
|
+
release() {
|
|
2583
|
+
this.holding = !1, window.clearTimeout(this.timer), this.timer = void 0;
|
|
2584
|
+
}
|
|
2585
|
+
}
|
|
2586
|
+
function st(l) {
|
|
2587
|
+
let e = 0;
|
|
2588
|
+
for (let t = 0; t < l.length; t += 1)
|
|
2589
|
+
e += l[t] * l[t];
|
|
2590
|
+
return Math.sqrt(e / Math.max(1, l.length));
|
|
2591
|
+
}
|
|
2592
|
+
class it {
|
|
2593
|
+
constructor(e = {}) {
|
|
2594
|
+
this.chunks = 0, this.loudChunks = 0, this.echoFloor = 4e-3, this.fired = !1, this.calibrationChunks = e.calibrationChunks ?? 4, this.onsetChunks = e.onsetChunks ?? 2, this.minimumLevel = e.minimumLevel ?? 0.018, this.floorMultiplier = e.floorMultiplier ?? 2.4;
|
|
2595
|
+
}
|
|
2596
|
+
/** Start a new assistant-speech period. The echo floor decays, not resets. */
|
|
2597
|
+
reset() {
|
|
2598
|
+
this.chunks = 0, this.loudChunks = 0, this.fired = !1, this.echoFloor = Math.max(4e-3, this.echoFloor * 0.75);
|
|
2599
|
+
}
|
|
2600
|
+
/** Returns true exactly once per speech period, on the detected onset. */
|
|
2601
|
+
push(e) {
|
|
2602
|
+
if (this.fired) return !1;
|
|
2603
|
+
const t = st(e);
|
|
2604
|
+
if (this.chunks += 1, this.chunks <= this.calibrationChunks)
|
|
2605
|
+
return this.echoFloor = this.echoFloor * 0.65 + t * 0.35, !1;
|
|
2606
|
+
const s = Math.max(this.minimumLevel, this.echoFloor * this.floorMultiplier);
|
|
2607
|
+
return t >= s ? this.loudChunks += 1 : (this.loudChunks = 0, this.echoFloor = this.echoFloor * 0.92 + t * 0.08), this.loudChunks < this.onsetChunks ? !1 : (this.fired = !0, !0);
|
|
2608
|
+
}
|
|
2609
|
+
get floor() {
|
|
2610
|
+
return this.echoFloor;
|
|
2611
|
+
}
|
|
2612
|
+
}
|
|
2613
|
+
const ge = {
|
|
2614
|
+
channelCount: { ideal: 1 },
|
|
2615
|
+
echoCancellation: { ideal: !0 },
|
|
2616
|
+
noiseSuppression: { ideal: !0 },
|
|
2617
|
+
autoGainControl: { ideal: !1 }
|
|
2618
|
+
};
|
|
2619
|
+
function rt(l, e, t) {
|
|
2620
|
+
if (e === t) return l;
|
|
2621
|
+
const s = Math.max(1, Math.round(l.length * t / e)), i = new Float32Array(s), r = e / t;
|
|
2622
|
+
for (let n = 0; n < s; n += 1) {
|
|
2623
|
+
const a = n * r, c = Math.floor(a), o = Math.min(l.length - 1, c + 1), h = a - c;
|
|
2624
|
+
i[n] = l[c] * (1 - h) + l[o] * h;
|
|
2625
|
+
}
|
|
2626
|
+
return i;
|
|
2627
|
+
}
|
|
2628
|
+
function nt(l) {
|
|
2629
|
+
const e = new Uint8Array(l.length * 2), t = new DataView(e.buffer);
|
|
2630
|
+
l.forEach((i, r) => {
|
|
2631
|
+
const n = Math.max(-1, Math.min(1, i));
|
|
2632
|
+
t.setInt16(r * 2, n < 0 ? n * 32768 : n * 32767, !0);
|
|
2633
|
+
});
|
|
2634
|
+
let s = "";
|
|
2635
|
+
for (let i = 0; i < e.length; i += 32768) {
|
|
2636
|
+
const r = e.subarray(i, i + 32768);
|
|
2637
|
+
for (let n = 0; n < r.length; n += 1) s += String.fromCharCode(r[n]);
|
|
2638
|
+
}
|
|
2639
|
+
return btoa(s);
|
|
2640
|
+
}
|
|
2641
|
+
async function Ee(l, e, t = {}) {
|
|
2642
|
+
const s = e ?? await navigator.mediaDevices.getUserMedia({
|
|
2643
|
+
audio: ge
|
|
2644
|
+
}), i = s.getAudioTracks()[0];
|
|
2645
|
+
if (!i) throw new Error("Microphone audio track is unavailable");
|
|
2646
|
+
await i.applyConstraints(ge).catch(
|
|
2647
|
+
(w) => console.warn("[BeVyra audio] unable to apply all microphone processing constraints", w)
|
|
2648
|
+
), i.contentHint = "speech", console.info("[BeVyra audio] microphone processing", {
|
|
2649
|
+
supported: navigator.mediaDevices.getSupportedConstraints(),
|
|
2650
|
+
settings: i.getSettings(),
|
|
2651
|
+
constraints: i.getConstraints()
|
|
2652
|
+
});
|
|
2653
|
+
const r = new AudioContext(), n = `class VyraPcmCapture extends AudioWorkletProcessor {
|
|
2654
|
+
constructor() {
|
|
2655
|
+
super(); this.buffer = new Float32Array(2048); this.offset = 0;
|
|
2656
|
+
this.port.onmessage = ({data}) => {
|
|
2657
|
+
if (data && data.type === "flush") {
|
|
2658
|
+
if (this.offset > 0) {
|
|
2659
|
+
const chunk = this.buffer.slice(0, this.offset);
|
|
2660
|
+
this.port.postMessage({type:"chunk", samples:chunk}, [chunk.buffer]);
|
|
2661
|
+
this.buffer = new Float32Array(2048); this.offset = 0;
|
|
2662
|
+
}
|
|
2663
|
+
this.port.postMessage({type:"flushed", id:data.id});
|
|
2664
|
+
}
|
|
2665
|
+
};
|
|
2666
|
+
}
|
|
2667
|
+
process(inputs) {
|
|
2668
|
+
const input = inputs[0] && inputs[0][0];
|
|
2669
|
+
if (!input) return true;
|
|
2670
|
+
for (let index = 0; index < input.length; index += 1) {
|
|
2671
|
+
this.buffer[this.offset++] = input[index];
|
|
2672
|
+
if (this.offset === this.buffer.length) {
|
|
2673
|
+
const chunk = this.buffer;
|
|
2674
|
+
this.port.postMessage({type:"chunk", samples:chunk}, [chunk.buffer]);
|
|
2675
|
+
this.buffer = new Float32Array(2048); this.offset = 0;
|
|
2676
|
+
}
|
|
2677
|
+
}
|
|
2678
|
+
return true;
|
|
2679
|
+
}
|
|
2680
|
+
}
|
|
2681
|
+
registerProcessor("vyra-pcm-capture", VyraPcmCapture);`, a = URL.createObjectURL(new Blob([n], { type: "text/javascript" }));
|
|
2682
|
+
let c = null, o = null, h = !1;
|
|
2683
|
+
const d = matchMedia("(pointer: coarse)").matches || /Android|iPhone|iPad|iPod/i.test(navigator.userAgent);
|
|
2684
|
+
let u = !1, p = !1, f = 0;
|
|
2685
|
+
const b = new it(
|
|
2686
|
+
d ? { onsetChunks: 2 } : { onsetChunks: 3 }
|
|
2687
|
+
), S = [], g = (w) => l(nt(rt(w, r.sampleRate, t.sampleRate ?? 16e3)));
|
|
2688
|
+
let y = 1;
|
|
2689
|
+
const v = /* @__PURE__ */ new Map();
|
|
2690
|
+
try {
|
|
2691
|
+
await r.audioWorklet.addModule(a), await r.resume(), c = r.createMediaStreamSource(s), o = new AudioWorkletNode(r, "vyra-pcm-capture", { numberOfInputs: 1, numberOfOutputs: 0 }), o.port.onmessage = (w) => {
|
|
2692
|
+
if (w.data.type === "flushed") {
|
|
2693
|
+
v.get(w.data.id)?.(), v.delete(w.data.id);
|
|
2694
|
+
return;
|
|
2695
|
+
}
|
|
2696
|
+
if (h) return;
|
|
2697
|
+
const m = w.data.samples;
|
|
2698
|
+
if (!u) {
|
|
2699
|
+
g(m);
|
|
2700
|
+
return;
|
|
2701
|
+
}
|
|
2702
|
+
const A = b.push(m);
|
|
2703
|
+
if (!d || p) {
|
|
2704
|
+
g(m), A && t.onSpeechOnset?.();
|
|
2705
|
+
return;
|
|
2706
|
+
}
|
|
2707
|
+
for (S.push(m); S.length > 6; ) S.shift();
|
|
2708
|
+
if (A) {
|
|
2709
|
+
p = !0, console.info("[BeVyra audio] barge-in onset", {
|
|
2710
|
+
echoFloor: Number(b.floor.toFixed(4))
|
|
2711
|
+
});
|
|
2712
|
+
for (const E of S) g(E);
|
|
2713
|
+
S.length = 0, t.onSpeechOnset?.();
|
|
2714
|
+
}
|
|
2715
|
+
}, c.connect(o);
|
|
2716
|
+
} catch (w) {
|
|
2717
|
+
throw s.getTracks().forEach((m) => m.stop()), await r.close().catch(() => {
|
|
2718
|
+
}), w;
|
|
2719
|
+
} finally {
|
|
2720
|
+
URL.revokeObjectURL(a);
|
|
2721
|
+
}
|
|
2722
|
+
return {
|
|
2723
|
+
async setMuted(w) {
|
|
2724
|
+
if (w !== h) {
|
|
2725
|
+
if (w && o) {
|
|
2726
|
+
const m = y++;
|
|
2727
|
+
await new Promise((A) => {
|
|
2728
|
+
const E = window.setTimeout(() => {
|
|
2729
|
+
v.delete(m), A();
|
|
2730
|
+
}, 250);
|
|
2731
|
+
v.set(m, () => {
|
|
2732
|
+
window.clearTimeout(E), A();
|
|
2733
|
+
}), o.port.postMessage({ type: "flush", id: m });
|
|
2734
|
+
});
|
|
2735
|
+
}
|
|
2736
|
+
h = w, s.getAudioTracks().forEach((m) => {
|
|
2737
|
+
m.enabled = !w;
|
|
2738
|
+
});
|
|
2739
|
+
}
|
|
2740
|
+
},
|
|
2741
|
+
setAssistantSpeaking(w) {
|
|
2742
|
+
if (f && (window.clearTimeout(f), f = 0), !w && u && d) {
|
|
2743
|
+
f = window.setTimeout(() => {
|
|
2744
|
+
u = !1, p = !1, S.length = 0, f = 0;
|
|
2745
|
+
}, 180);
|
|
2746
|
+
return;
|
|
2747
|
+
}
|
|
2748
|
+
w && !u && b.reset(), u = w, p = !1, S.length = 0;
|
|
2749
|
+
},
|
|
2750
|
+
async stop() {
|
|
2751
|
+
f && window.clearTimeout(f), v.forEach((w) => w()), v.clear(), o && (o.port.onmessage = null, o.disconnect()), c?.disconnect(), s.getTracks().forEach((w) => w.stop()), await r.close().catch(() => {
|
|
2752
|
+
});
|
|
2753
|
+
}
|
|
2754
|
+
};
|
|
2755
|
+
}
|
|
2756
|
+
function Te(l, e = 0.8) {
|
|
2757
|
+
return btoa("\0".repeat(Math.round(l * e) * 2));
|
|
2758
|
+
}
|
|
2759
|
+
class ke {
|
|
2760
|
+
constructor() {
|
|
2761
|
+
this.responseActive = !1, this.discardOldAudio = !1, this.staleTurnComplete = !1;
|
|
2762
|
+
}
|
|
2763
|
+
audio() {
|
|
2764
|
+
return this.discardOldAudio ? "drop" : (this.staleTurnComplete = !1, this.responseActive = !0, "deliver");
|
|
2765
|
+
}
|
|
2766
|
+
/** The user started speaking over the avatar. */
|
|
2767
|
+
localBargeIn() {
|
|
2768
|
+
this.discardOldAudio = this.responseActive, this.staleTurnComplete = this.responseActive, this.responseActive = !1;
|
|
2769
|
+
}
|
|
2770
|
+
/** Returns whether the client must still interrupt locally. */
|
|
2771
|
+
serverInterrupted() {
|
|
2772
|
+
const e = this.discardOldAudio;
|
|
2773
|
+
return this.discardOldAudio = !1, this.responseActive = !1, this.staleTurnComplete = !0, !e;
|
|
2774
|
+
}
|
|
2775
|
+
/** Returns whether this turnComplete belongs to the live response. */
|
|
2776
|
+
turnComplete() {
|
|
2777
|
+
const e = this.discardOldAudio || this.staleTurnComplete;
|
|
2778
|
+
return this.discardOldAudio = !1, this.staleTurnComplete = !1, this.responseActive = !1, !e;
|
|
2779
|
+
}
|
|
2780
|
+
}
|
|
2781
|
+
const at = 15e3, Ie = `(Stage direction, not spoken by the user: the user has been silent for a while. Respond with one short, warm sentence that begins with "You haven't said anything in a while" and asks whether there is anything else you can help with. Do not answer or repeat anything from earlier in the conversation.)`, ot = {
|
|
2782
|
+
set: (l, e) => window.setTimeout(l, e),
|
|
2783
|
+
clear: (l) => window.clearTimeout(l)
|
|
2784
|
+
};
|
|
2785
|
+
class Ce {
|
|
2786
|
+
constructor(e, t = at, s = ot) {
|
|
2787
|
+
this.onCheckIn = e, this.delayMs = t, this.timers = s, this.askedSinceUserSpoke = !1, this.muted = !1;
|
|
2788
|
+
}
|
|
2789
|
+
/** The avatar finished speaking: start counting the user's silence. */
|
|
2790
|
+
assistantFinished() {
|
|
2791
|
+
this.cancel(), !(this.askedSinceUserSpoke || this.muted) && (this.timer = this.timers.set(() => {
|
|
2792
|
+
this.timer = void 0, this.askedSinceUserSpoke = !0, this.onCheckIn();
|
|
2793
|
+
}, this.delayMs));
|
|
2794
|
+
}
|
|
2795
|
+
/** The avatar started talking or a response is arriving. */
|
|
2796
|
+
assistantActive() {
|
|
2797
|
+
this.cancel();
|
|
2798
|
+
}
|
|
2799
|
+
/** The user spoke: they are engaged again, so a later silence may ask. */
|
|
2800
|
+
userSpoke() {
|
|
2801
|
+
this.cancel(), this.askedSinceUserSpoke = !1;
|
|
2802
|
+
}
|
|
2803
|
+
setMuted(e) {
|
|
2804
|
+
this.muted = e, e && this.cancel();
|
|
2805
|
+
}
|
|
2806
|
+
cancel() {
|
|
2807
|
+
this.timer !== void 0 && this.timers.clear(this.timer), this.timer = void 0;
|
|
2808
|
+
}
|
|
2809
|
+
}
|
|
2810
|
+
async function J(l, e = "") {
|
|
2811
|
+
const { tutor: t, chatbot: s, surface: i, avatarId: r } = l, n = await fetch(t ? "/api/v1/ai/tutor/session" : "/api/v1/ai/chat/session", {
|
|
2812
|
+
method: "POST",
|
|
2813
|
+
// The private-preview entry gate is cookie-backed. The landing-page
|
|
2814
|
+
// demo must carry that same-origin cookie just like Studio; omitting
|
|
2815
|
+
// credentials lets bundle calibration succeed and then rejects the
|
|
2816
|
+
// actual live session with "Entry passkey required".
|
|
2817
|
+
credentials: "same-origin",
|
|
2818
|
+
headers: { "Content-Type": "application/json" },
|
|
2819
|
+
body: JSON.stringify(
|
|
2820
|
+
t ? { avatarId: r, ...t, ...e ? { resumeHandle: e } : {} } : { surface: i, avatarId: r, ...s?.id ? { chatbotId: s.id } : {}, ...s?.shareToken ? { chatbotShare: s.shareToken } : {} }
|
|
2821
|
+
)
|
|
2822
|
+
}), a = await n.json().catch(() => ({}));
|
|
2823
|
+
if (!n.ok)
|
|
2824
|
+
throw new Error(a.error || "Live session authorization failed.");
|
|
2825
|
+
if (a.version !== 1 || a.mode !== (t ? "tutor" : "chat") || a.avatarId !== r)
|
|
2826
|
+
throw new Error("Live session contract did not match this conversation.");
|
|
2827
|
+
if (!a.configurationSignature || !a.immutableConfig)
|
|
2828
|
+
throw new Error("Live session contract is incomplete.");
|
|
2829
|
+
if (Date.parse(a.expiresAt) <= Date.now() + 3e4)
|
|
2830
|
+
throw new Error("Live session credential expired before connection.");
|
|
2831
|
+
return a;
|
|
2832
|
+
}
|
|
2833
|
+
function ct(l) {
|
|
2834
|
+
const e = atob(l), t = new Uint8Array(e.length);
|
|
2835
|
+
for (let s = 0; s < e.length; s++) t[s] = e.charCodeAt(s);
|
|
2836
|
+
return new Int16Array(t.buffer);
|
|
2837
|
+
}
|
|
2838
|
+
const ht = 4;
|
|
2839
|
+
class ie {
|
|
2840
|
+
constructor(e) {
|
|
2841
|
+
this.callbacks = e, this.sampleCursor = 0, this.generation = 0, this.turn = 0, this.closed = !1, this.gate = new ke(), this.greetingHold = new Ae(), this.silence = new Ce(() => this.checkIn()), this.checkInPrompt = () => Ie, this.resumeHandle = "", this.goAwayPending = !1, this.spokeSinceUnmute = !1, this.reconnects = 0, this.responding = !1, this.appContentAt = Number.NEGATIVE_INFINITY, this.userInputAt = Number.NEGATIVE_INFINITY, this.playbackActive = !1, this.sessionId = crypto.randomUUID();
|
|
2842
|
+
}
|
|
2843
|
+
/** `contract`: one already issued for this conversation (see connectLiveSession). */
|
|
2844
|
+
static async connect(e, t, s) {
|
|
2845
|
+
const i = new ie(e);
|
|
2846
|
+
i.options = t;
|
|
2847
|
+
try {
|
|
2848
|
+
return await i.open(s), i;
|
|
2849
|
+
} catch (r) {
|
|
2850
|
+
throw i.closed = !0, i.socket?.close(), await i.microphone?.stop(), r;
|
|
2851
|
+
}
|
|
2852
|
+
}
|
|
2853
|
+
contract(e = "") {
|
|
2854
|
+
return J(this.options, e);
|
|
2855
|
+
}
|
|
2856
|
+
async open(e) {
|
|
2857
|
+
this.callbacks.status("connecting");
|
|
2858
|
+
const t = e ?? await this.contract();
|
|
2859
|
+
await this.openSocket(t), this.callbacks.status("ready"), this.microphone = await Ee((s) => {
|
|
2860
|
+
const i = this.socket;
|
|
2861
|
+
i?.readyState === WebSocket.OPEN && !this.reconnecting && !this.greetingHold.active && (this.spokeSinceUnmute = !0, i.send(
|
|
2862
|
+
JSON.stringify({
|
|
2863
|
+
realtimeInput: {
|
|
2864
|
+
audio: { mimeType: "audio/pcm;rate=16000", data: s }
|
|
2865
|
+
}
|
|
2866
|
+
})
|
|
2867
|
+
));
|
|
2868
|
+
}, this.options.suppliedStream, {
|
|
2869
|
+
onSpeechOnset: () => this.localBargeIn()
|
|
2870
|
+
}), this.options.startMuted && await this.setMuted(!0), this.callbacks.status("listening"), t.greeting && (this.greetingHold.start(), this.sendText(t.greeting));
|
|
2871
|
+
}
|
|
2872
|
+
openSocket(e) {
|
|
2873
|
+
const t = `${e.endpoint}?access_token=${encodeURIComponent(e.credential)}`, s = this.socket = new WebSocket(t);
|
|
2874
|
+
return s.binaryType = "arraybuffer", new Promise((i, r) => {
|
|
2875
|
+
let n = !1, a = !1;
|
|
2876
|
+
const c = (h) => {
|
|
2877
|
+
a || (a = !0, window.clearTimeout(o), r(h));
|
|
2878
|
+
}, o = window.setTimeout(() => {
|
|
2879
|
+
c(
|
|
2880
|
+
new Error(
|
|
2881
|
+
`Gemini Live setup timed out (socket state ${s.readyState}).`
|
|
2882
|
+
)
|
|
2883
|
+
), s.close();
|
|
2884
|
+
}, 2e4);
|
|
2885
|
+
s.onopen = () => {
|
|
2886
|
+
s.send(
|
|
2887
|
+
JSON.stringify({ setup: e.immutableConfig })
|
|
2888
|
+
);
|
|
2889
|
+
}, s.onerror = () => c(new Error("Gemini Live WebSocket connection failed.")), s.onmessage = async (h) => {
|
|
2890
|
+
let d;
|
|
2891
|
+
try {
|
|
2892
|
+
const u = typeof h.data == "string" ? h.data : h.data instanceof ArrayBuffer ? new TextDecoder().decode(h.data) : h.data instanceof Blob ? await h.data.text() : String(h.data);
|
|
2893
|
+
d = JSON.parse(u);
|
|
2894
|
+
} catch (u) {
|
|
2895
|
+
console.error(
|
|
2896
|
+
"[BeVyra Gemini Live] Could not decode server frame",
|
|
2897
|
+
u
|
|
2898
|
+
), c(new Error("Gemini Live returned an unreadable setup response."));
|
|
2899
|
+
return;
|
|
2900
|
+
}
|
|
2901
|
+
if (d.error) {
|
|
2902
|
+
const u = d.error.message || d.error.status || d.error.code || "setup rejected";
|
|
2903
|
+
c(new Error(`Gemini Live setup failed: ${u}`));
|
|
2904
|
+
return;
|
|
2905
|
+
}
|
|
2906
|
+
if (d.setupComplete) {
|
|
2907
|
+
n = !0, a || (a = !0, window.clearTimeout(o), i());
|
|
2908
|
+
return;
|
|
2909
|
+
}
|
|
2910
|
+
s === this.socket && this.receive(d);
|
|
2911
|
+
}, s.onclose = (h) => {
|
|
2912
|
+
window.clearTimeout(o);
|
|
2913
|
+
const d = h.reason ? `: ${h.reason}` : "";
|
|
2914
|
+
if (!n) {
|
|
2915
|
+
c(
|
|
2916
|
+
new Error(
|
|
2917
|
+
`Gemini Live closed during setup (${h.code})${d}`
|
|
2918
|
+
)
|
|
2919
|
+
);
|
|
2920
|
+
return;
|
|
2921
|
+
}
|
|
2922
|
+
if (!(this.closed || s !== this.socket)) {
|
|
2923
|
+
if (this.options.tutor && this.reconnects < ht) {
|
|
2924
|
+
this.reconnect();
|
|
2925
|
+
return;
|
|
2926
|
+
}
|
|
2927
|
+
this.callbacks.status("failed"), this.callbacks.error(
|
|
2928
|
+
new Error(`Gemini Live connection closed (${h.code})${d}`)
|
|
2929
|
+
);
|
|
2930
|
+
}
|
|
2931
|
+
};
|
|
2932
|
+
});
|
|
2933
|
+
}
|
|
2934
|
+
/**
|
|
2935
|
+
* Moves the conversation to a new connection, resuming the old session
|
|
2936
|
+
* when Gemini gave a handle. The microphone and avatar are unaffected.
|
|
2937
|
+
*/
|
|
2938
|
+
reconnect() {
|
|
2939
|
+
if (this.reconnecting) return this.reconnecting;
|
|
2940
|
+
this.goAwayPending = !1, window.clearTimeout(this.goAwayTimer), this.reconnects += 1;
|
|
2941
|
+
const e = this.socket;
|
|
2942
|
+
return this.reconnecting = (async () => {
|
|
2943
|
+
this.callbacks.status("reconnecting");
|
|
2944
|
+
let t = !1;
|
|
2945
|
+
try {
|
|
2946
|
+
try {
|
|
2947
|
+
await this.openSocket(await this.contract(this.resumeHandle));
|
|
2948
|
+
} catch (s) {
|
|
2949
|
+
if (!this.resumeHandle) throw s;
|
|
2950
|
+
console.warn("[BeVyra Gemini Live] resume failed; starting a fresh session", s), this.resumeHandle = "", t = !0, await this.openSocket(await this.contract());
|
|
2951
|
+
}
|
|
2952
|
+
this.resumeHandle || (t = !0), e && e !== this.socket && e.readyState === WebSocket.OPEN && e.close(1e3, "session moved"), this.reconnects = 0, this.responding = !1, console.info(`[BeVyra Gemini Live] reconnected (${t ? "fresh session" : "resumed"})`), this.callbacks.status(this.playbackActive ? "responding" : "listening");
|
|
2953
|
+
} finally {
|
|
2954
|
+
this.reconnecting = void 0;
|
|
2955
|
+
}
|
|
2956
|
+
t && this.callbacks.sessionRestarted?.();
|
|
2957
|
+
})().catch((t) => {
|
|
2958
|
+
this.closed || (this.callbacks.status("failed"), this.callbacks.error(t instanceof Error ? t : new Error(String(t))));
|
|
2959
|
+
}), this.reconnecting;
|
|
2960
|
+
}
|
|
2961
|
+
/** Sends a text turn (lesson instructions or a typed message). */
|
|
2962
|
+
sendOpening(e) {
|
|
2963
|
+
return this.greetingHold.start(), this.sendText(e) ? !0 : (this.greetingHold.release(), !1);
|
|
2964
|
+
}
|
|
2965
|
+
sendText(e) {
|
|
2966
|
+
return this.socket?.readyState !== WebSocket.OPEN ? !1 : (this.appContentAt = performance.now(), this.socket.send(
|
|
2967
|
+
JSON.stringify({
|
|
2968
|
+
clientContent: {
|
|
2969
|
+
turns: [{ role: "user", parts: [{ text: e }] }],
|
|
2970
|
+
turnComplete: !0
|
|
2971
|
+
}
|
|
2972
|
+
})
|
|
2973
|
+
), !0);
|
|
2974
|
+
}
|
|
2975
|
+
/** Adds context Gemini should consider in its next reply, without replying now. */
|
|
2976
|
+
addContext(e) {
|
|
2977
|
+
return this.socket?.readyState !== WebSocket.OPEN ? !1 : (this.appContentAt = performance.now(), this.socket.send(
|
|
2978
|
+
JSON.stringify({
|
|
2979
|
+
clientContent: {
|
|
2980
|
+
turns: [{ role: "user", parts: [{ text: e }] }],
|
|
2981
|
+
turnComplete: !1
|
|
2982
|
+
}
|
|
2983
|
+
})
|
|
2984
|
+
), !0);
|
|
2985
|
+
}
|
|
2986
|
+
/** Shows Gemini an image (a JPEG, base64) as a realtime video frame. */
|
|
2987
|
+
sendImage(e) {
|
|
2988
|
+
return this.socket?.readyState !== WebSocket.OPEN ? !1 : (this.appContentAt = performance.now(), this.socket.send(
|
|
2989
|
+
JSON.stringify({ realtimeInput: { video: { mimeType: "image/jpeg", data: e } } })
|
|
2990
|
+
), !0);
|
|
2991
|
+
}
|
|
2992
|
+
respondToTool(e, t) {
|
|
2993
|
+
this.appContentAt = performance.now(), this.socket?.send(
|
|
2994
|
+
JSON.stringify({ toolResponse: { functionResponses: [{ id: e.id, name: e.name, response: { output: t } }] } })
|
|
2995
|
+
);
|
|
2996
|
+
}
|
|
2997
|
+
/** Decides what the silence check-in says; null skips it. */
|
|
2998
|
+
setCheckIn(e) {
|
|
2999
|
+
this.checkInPrompt = e;
|
|
3000
|
+
}
|
|
3001
|
+
/** Stops the avatar for the app (e.g. the learner picked a gallery image). */
|
|
3002
|
+
interruptForApp() {
|
|
3003
|
+
this.closed || (this.greetingHold.release(), this.gate.localBargeIn(), this.interrupt());
|
|
3004
|
+
}
|
|
3005
|
+
receive(e) {
|
|
3006
|
+
const t = e.sessionResumptionUpdate;
|
|
3007
|
+
if (t?.resumable && t.newHandle && (this.resumeHandle = String(t.newHandle)), e.goAway) {
|
|
3008
|
+
this.goAwayPending = !0;
|
|
3009
|
+
const a = Number.parseFloat(String(e.goAway.timeLeft || "10")) * 1e3;
|
|
3010
|
+
console.info(`[BeVyra Gemini Live] goAway, ${Math.round(a / 1e3)}s left`), this.responding ? this.goAwayTimer = window.setTimeout(() => void this.reconnect(), Math.max(0, a - 3e3)) : this.reconnect();
|
|
3011
|
+
return;
|
|
3012
|
+
}
|
|
3013
|
+
if (e.toolCall?.functionCalls?.length) {
|
|
3014
|
+
this.callbacks.toolCall?.(
|
|
3015
|
+
e.toolCall.functionCalls.map((a) => ({ id: String(a.id || ""), name: String(a.name || ""), args: a.args || {} }))
|
|
3016
|
+
);
|
|
3017
|
+
return;
|
|
3018
|
+
}
|
|
3019
|
+
const s = e.serverContent;
|
|
3020
|
+
if (!s) return;
|
|
3021
|
+
if (s.interrupted) {
|
|
3022
|
+
if (this.appCausedInterruption()) {
|
|
3023
|
+
this.gate.serverInterrupted(), console.info("[BeVyra Gemini Live] interruption caused by app content; playback continues");
|
|
3024
|
+
return;
|
|
3025
|
+
}
|
|
3026
|
+
this.gate.serverInterrupted() && this.interrupt();
|
|
3027
|
+
return;
|
|
3028
|
+
}
|
|
3029
|
+
const i = s.inputTranscription?.text || s.interimInputTranscription?.text, r = s.outputTranscription?.text;
|
|
3030
|
+
i && (this.userInputAt = performance.now(), this.silence.userSpoke(), this.callbacks.userSpeech?.()), i && this.callbacks.transcript(
|
|
3031
|
+
"user",
|
|
3032
|
+
i,
|
|
3033
|
+
!!s.inputTranscription
|
|
3034
|
+
), r && this.callbacks.transcript(
|
|
3035
|
+
"assistant",
|
|
3036
|
+
r,
|
|
3037
|
+
!!s.turnComplete
|
|
3038
|
+
);
|
|
3039
|
+
const n = s.modelTurn?.parts || [];
|
|
3040
|
+
for (const a of n) {
|
|
3041
|
+
const c = a.inlineData?.data, o = String(a.inlineData?.mimeType || "");
|
|
3042
|
+
if (!c || !o.includes("audio/pcm")) continue;
|
|
3043
|
+
const h = Number(
|
|
3044
|
+
o.match(/(?:^|;)rate=(\d+)/i)?.[1] || 24e3
|
|
3045
|
+
);
|
|
3046
|
+
if (!Number.isFinite(h) || h < 8e3 || h > 96e3) {
|
|
3047
|
+
this.callbacks.error(
|
|
3048
|
+
new Error(
|
|
3049
|
+
`Gemini returned an unsupported audio rate: ${h}.`
|
|
3050
|
+
)
|
|
3051
|
+
);
|
|
3052
|
+
continue;
|
|
3053
|
+
}
|
|
3054
|
+
if (this.gate.audio() === "drop") continue;
|
|
3055
|
+
this.responding = !0, this.silence.assistantActive(), this.callbacks.status("responding");
|
|
3056
|
+
const d = ct(c), u = {
|
|
3057
|
+
sessionId: this.sessionId,
|
|
3058
|
+
turnId: this.turn,
|
|
3059
|
+
responseGenerationId: this.generation,
|
|
3060
|
+
startSample: this.sampleCursor,
|
|
3061
|
+
endSample: this.sampleCursor + d.length,
|
|
3062
|
+
sampleRate: h,
|
|
3063
|
+
pcm24: d
|
|
3064
|
+
};
|
|
3065
|
+
this.sampleCursor = u.endSample, this.callbacks.assistantAudio(u);
|
|
3066
|
+
}
|
|
3067
|
+
if (s.turnComplete) {
|
|
3068
|
+
if (this.responding = !1, this.greetingHold.turnComplete(), this.goAwayPending && this.reconnect(), this.gate.turnComplete()) {
|
|
3069
|
+
const a = this.generation;
|
|
3070
|
+
Promise.resolve(this.callbacks.turnComplete?.()).finally(() => {
|
|
3071
|
+
this.closed || a !== this.generation || this.callbacks.status("listening");
|
|
3072
|
+
});
|
|
3073
|
+
}
|
|
3074
|
+
this.turn += 1, this.sampleCursor = 0;
|
|
3075
|
+
}
|
|
3076
|
+
}
|
|
3077
|
+
/**
|
|
3078
|
+
* The renderer reports when the avatar is audible. Barge-in detection and
|
|
3079
|
+
* the mobile echo guard follow actual playback, which lags generation.
|
|
3080
|
+
*/
|
|
3081
|
+
setAssistantPlayback(e) {
|
|
3082
|
+
this.playbackActive = e, this.greetingHold.playback(e), this.microphone?.setAssistantSpeaking(e), e ? this.silence.assistantActive() : this.silence.assistantFinished();
|
|
3083
|
+
}
|
|
3084
|
+
/** Tutor mode: `interrupted` right after app content, with no user input since. */
|
|
3085
|
+
appCausedInterruption() {
|
|
3086
|
+
return this.options.tutor ? performance.now() - this.appContentAt < 2e3 && this.userInputAt < this.appContentAt : !1;
|
|
3087
|
+
}
|
|
3088
|
+
/** Ask once, gently, whether the user needs anything else. */
|
|
3089
|
+
checkIn() {
|
|
3090
|
+
const e = this.socket;
|
|
3091
|
+
if (this.closed || this.playbackActive || e?.readyState !== WebSocket.OPEN) return;
|
|
3092
|
+
const t = this.checkInPrompt();
|
|
3093
|
+
t && (console.info("[BeVyra Gemini Live] silence check-in"), this.sendText(t));
|
|
3094
|
+
}
|
|
3095
|
+
/** The user started talking over the audible avatar. */
|
|
3096
|
+
localBargeIn() {
|
|
3097
|
+
this.closed || !this.playbackActive || (console.info("[BeVyra Gemini Live] local barge-in"), this.greetingHold.release(), this.userInputAt = performance.now(), this.silence.userSpoke(), this.gate.localBargeIn(), this.interrupt(), this.callbacks.userSpeech?.());
|
|
3098
|
+
}
|
|
3099
|
+
interrupt() {
|
|
3100
|
+
this.callbacks.status("interrupted"), this.setAssistantPlayback(!1), this.generation += 1, this.sampleCursor = 0, this.callbacks.status("listening");
|
|
3101
|
+
}
|
|
3102
|
+
async setMuted(e) {
|
|
3103
|
+
this.silence.setMuted(e), await this.microphone?.setMuted(e), e && this.spokeSinceUnmute && this.socket?.readyState === WebSocket.OPEN && !this.reconnecting && (this.socket.send(JSON.stringify({ realtimeInput: { audio: { mimeType: "audio/pcm;rate=16000", data: Te(16e3) } } })), this.socket.send(JSON.stringify({ realtimeInput: { audioStreamEnd: !0 } }))), this.spokeSinceUnmute = !1;
|
|
3104
|
+
}
|
|
3105
|
+
async close() {
|
|
3106
|
+
this.closed = !0, this.greetingHold.release(), this.silence.cancel(), window.clearTimeout(this.goAwayTimer), this.socket?.readyState === WebSocket.OPEN && (this.socket.send(
|
|
3107
|
+
JSON.stringify({ realtimeInput: { audioStreamEnd: !0 } })
|
|
3108
|
+
), this.socket.close(1e3, "session ended")), await this.microphone?.stop(), this.callbacks.status("closed");
|
|
3109
|
+
}
|
|
3110
|
+
}
|
|
3111
|
+
const Y = 24e3, lt = 4;
|
|
3112
|
+
function dt(l) {
|
|
3113
|
+
const e = atob(l), t = new Uint8Array(e.length);
|
|
3114
|
+
for (let s = 0; s < e.length; s++) t[s] = e.charCodeAt(s);
|
|
3115
|
+
return new Int16Array(t.buffer);
|
|
3116
|
+
}
|
|
3117
|
+
function ut(l) {
|
|
3118
|
+
if (l && typeof l == "object") return l;
|
|
3119
|
+
try {
|
|
3120
|
+
const e = JSON.parse(String(l || "{}"));
|
|
3121
|
+
return e && typeof e == "object" ? e : {};
|
|
3122
|
+
} catch {
|
|
3123
|
+
return {};
|
|
3124
|
+
}
|
|
3125
|
+
}
|
|
3126
|
+
class re {
|
|
3127
|
+
constructor(e) {
|
|
3128
|
+
this.callbacks = e, this.sampleCursor = 0, this.generation = 0, this.turn = 0, this.closed = !1, this.gate = new ke(), this.greetingHold = new Ae(), this.silence = new Ce(() => this.checkIn()), this.checkInPrompt = () => Ie, this.spokeSinceUnmute = !1, this.reconnects = 0, this.responding = !1, this.responseId = null, this.replyQueued = !1, this.cancelled = /* @__PURE__ */ new Set(), this.openToolCalls = /* @__PURE__ */ new Set(), this.userText = /* @__PURE__ */ new Map(), this.playbackActive = !1, this.sessionId = crypto.randomUUID();
|
|
3129
|
+
}
|
|
3130
|
+
/** `contract`: one already issued for this conversation (see connectLiveSession). */
|
|
3131
|
+
static async connect(e, t, s) {
|
|
3132
|
+
const i = new re(e);
|
|
3133
|
+
i.options = t;
|
|
3134
|
+
try {
|
|
3135
|
+
return await i.open(s), i;
|
|
3136
|
+
} catch (r) {
|
|
3137
|
+
throw i.closed = !0, i.socket?.close(), await i.microphone?.stop(), r;
|
|
3138
|
+
}
|
|
3139
|
+
}
|
|
3140
|
+
async open(e) {
|
|
3141
|
+
this.callbacks.status("connecting");
|
|
3142
|
+
const t = e ?? await J(this.options);
|
|
3143
|
+
await this.openSocket(t), this.callbacks.status("ready"), this.microphone = await Ee((s) => {
|
|
3144
|
+
this.reconnecting || this.greetingHold.active || (this.spokeSinceUnmute = !0, this.send({ type: "input_audio_buffer.append", audio: s }));
|
|
3145
|
+
}, this.options.suppliedStream, {
|
|
3146
|
+
onSpeechOnset: () => this.localBargeIn(),
|
|
3147
|
+
sampleRate: Y
|
|
3148
|
+
}), this.options.startMuted && await this.setMuted(!0), this.callbacks.status("listening"), t.greeting && (this.greetingHold.start(), this.sendText(t.greeting));
|
|
3149
|
+
}
|
|
3150
|
+
send(e) {
|
|
3151
|
+
return this.socket?.readyState !== WebSocket.OPEN ? !1 : (this.socket.send(JSON.stringify(e)), !0);
|
|
3152
|
+
}
|
|
3153
|
+
openSocket(e) {
|
|
3154
|
+
const t = this.socket = new WebSocket(e.endpoint, ["realtime", `openai-insecure-api-key.${e.credential}`]);
|
|
3155
|
+
return new Promise((s, i) => {
|
|
3156
|
+
let r = !1, n = !1;
|
|
3157
|
+
const a = (o) => {
|
|
3158
|
+
n || (n = !0, window.clearTimeout(c), i(o));
|
|
3159
|
+
}, c = window.setTimeout(() => {
|
|
3160
|
+
a(new Error(`OpenAI Realtime setup timed out (socket state ${t.readyState}).`)), t.close();
|
|
3161
|
+
}, 2e4);
|
|
3162
|
+
t.onerror = () => a(new Error("OpenAI Realtime WebSocket connection failed.")), t.onmessage = (o) => {
|
|
3163
|
+
let h;
|
|
3164
|
+
try {
|
|
3165
|
+
h = JSON.parse(String(o.data));
|
|
3166
|
+
} catch (d) {
|
|
3167
|
+
console.error("[BeVyra OpenAI Realtime] Could not decode server event", d);
|
|
3168
|
+
return;
|
|
3169
|
+
}
|
|
3170
|
+
if (!r) {
|
|
3171
|
+
if (h.type === "error") {
|
|
3172
|
+
a(new Error(`OpenAI Realtime setup failed: ${h.error?.message || h.error?.code || "rejected"}`));
|
|
3173
|
+
return;
|
|
3174
|
+
}
|
|
3175
|
+
(h.type === "session.created" || h.type === "session.updated") && (r = !0, n = !0, window.clearTimeout(c), s());
|
|
3176
|
+
return;
|
|
3177
|
+
}
|
|
3178
|
+
t === this.socket && this.receive(h);
|
|
3179
|
+
}, t.onclose = (o) => {
|
|
3180
|
+
window.clearTimeout(c);
|
|
3181
|
+
const h = o.reason ? `: ${o.reason}` : "";
|
|
3182
|
+
if (!r) {
|
|
3183
|
+
a(new Error(`OpenAI Realtime closed during setup (${o.code})${h}`));
|
|
3184
|
+
return;
|
|
3185
|
+
}
|
|
3186
|
+
if (!(this.closed || t !== this.socket)) {
|
|
3187
|
+
if (this.options.tutor && this.reconnects < lt) {
|
|
3188
|
+
this.reconnect();
|
|
3189
|
+
return;
|
|
3190
|
+
}
|
|
3191
|
+
this.callbacks.status("failed"), this.callbacks.error(new Error(`OpenAI Realtime connection closed (${o.code})${h}`));
|
|
3192
|
+
}
|
|
3193
|
+
};
|
|
3194
|
+
});
|
|
3195
|
+
}
|
|
3196
|
+
/** OpenAI sessions cannot be resumed: the tutor continues in a fresh one. */
|
|
3197
|
+
reconnect() {
|
|
3198
|
+
return this.reconnecting ? this.reconnecting : (this.reconnects += 1, this.reconnecting = (async () => {
|
|
3199
|
+
this.callbacks.status("reconnecting");
|
|
3200
|
+
try {
|
|
3201
|
+
await this.openSocket(await J(this.options)), this.reconnects = 0, this.responding = !1, this.responseId = null, this.replyQueued = !1, this.openToolCalls.clear(), console.info("[BeVyra OpenAI Realtime] reconnected (fresh session)"), this.callbacks.status(this.playbackActive ? "responding" : "listening");
|
|
3202
|
+
} finally {
|
|
3203
|
+
this.reconnecting = void 0;
|
|
3204
|
+
}
|
|
3205
|
+
this.callbacks.sessionRestarted?.();
|
|
3206
|
+
})().catch((e) => {
|
|
3207
|
+
this.closed || (this.callbacks.status("failed"), this.callbacks.error(e instanceof Error ? e : new Error(String(e))));
|
|
3208
|
+
}), this.reconnecting);
|
|
3209
|
+
}
|
|
3210
|
+
/** Asks for a reply now, or once the running response has finished. */
|
|
3211
|
+
reply() {
|
|
3212
|
+
if (this.responseId) {
|
|
3213
|
+
this.replyQueued = !0;
|
|
3214
|
+
return;
|
|
3215
|
+
}
|
|
3216
|
+
this.replyQueued = !1, this.send({ type: "response.create" });
|
|
3217
|
+
}
|
|
3218
|
+
addItem(e) {
|
|
3219
|
+
return this.send({ type: "conversation.item.create", item: { type: "message", role: "user", content: e } });
|
|
3220
|
+
}
|
|
3221
|
+
/** Sends a text turn (lesson instructions or a typed message). */
|
|
3222
|
+
sendOpening(e) {
|
|
3223
|
+
return this.greetingHold.start(), this.sendText(e) ? !0 : (this.greetingHold.release(), !1);
|
|
3224
|
+
}
|
|
3225
|
+
sendText(e) {
|
|
3226
|
+
return this.addItem([{ type: "input_text", text: e }]) ? (this.reply(), !0) : !1;
|
|
3227
|
+
}
|
|
3228
|
+
/** Adds context the model should consider in its next reply, without replying now. */
|
|
3229
|
+
addContext(e) {
|
|
3230
|
+
return this.addItem([{ type: "input_text", text: e }]);
|
|
3231
|
+
}
|
|
3232
|
+
/** Shows the model an image (a JPEG, base64). */
|
|
3233
|
+
sendImage(e) {
|
|
3234
|
+
return this.addItem([{ type: "input_image", image_url: `data:image/jpeg;base64,${e}` }]);
|
|
3235
|
+
}
|
|
3236
|
+
respondToTool(e, t) {
|
|
3237
|
+
this.send({ type: "conversation.item.create", item: { type: "function_call_output", call_id: e.id, output: JSON.stringify(t) } }), this.openToolCalls.delete(e.id), this.openToolCalls.size || this.reply();
|
|
3238
|
+
}
|
|
3239
|
+
/** Decides what the silence check-in says; null skips it. */
|
|
3240
|
+
setCheckIn(e) {
|
|
3241
|
+
this.checkInPrompt = e;
|
|
3242
|
+
}
|
|
3243
|
+
/** Stops the avatar for the app (e.g. the learner picked a gallery image). */
|
|
3244
|
+
interruptForApp() {
|
|
3245
|
+
this.closed || (this.greetingHold.release(), this.gate.localBargeIn(), this.cancelResponse(), this.interrupt());
|
|
3246
|
+
}
|
|
3247
|
+
/** Stops server generation for the response the user no longer wants. */
|
|
3248
|
+
cancelResponse() {
|
|
3249
|
+
this.responseId && (this.cancelled.add(this.responseId), this.send({ type: "response.cancel" }));
|
|
3250
|
+
}
|
|
3251
|
+
receive(e) {
|
|
3252
|
+
switch (e.type) {
|
|
3253
|
+
case "input_audio_buffer.speech_started":
|
|
3254
|
+
this.responding && (this.responseId && this.cancelled.add(this.responseId), this.gate.serverInterrupted() && this.interrupt());
|
|
3255
|
+
return;
|
|
3256
|
+
case "conversation.item.input_audio_transcription.delta": {
|
|
3257
|
+
const t = String(e.item_id || ""), s = (this.userText.get(t) || "") + String(e.delta || "");
|
|
3258
|
+
this.userText.set(t, s), s.trim() && this.userSpoke(), this.callbacks.transcript("user", s, !1);
|
|
3259
|
+
return;
|
|
3260
|
+
}
|
|
3261
|
+
case "conversation.item.input_audio_transcription.completed": {
|
|
3262
|
+
const t = String(e.item_id || "");
|
|
3263
|
+
this.userText.delete(t);
|
|
3264
|
+
const s = String(e.transcript || "");
|
|
3265
|
+
s.trim() && this.userSpoke(), this.callbacks.transcript("user", s, !0);
|
|
3266
|
+
return;
|
|
3267
|
+
}
|
|
3268
|
+
case "response.created":
|
|
3269
|
+
this.responseId = String(e.response?.id || "") || "pending";
|
|
3270
|
+
return;
|
|
3271
|
+
case "response.output_audio_transcript.delta":
|
|
3272
|
+
case "response.audio_transcript.delta":
|
|
3273
|
+
if (this.cancelled.has(String(e.response_id)) || this.gate.audio() === "drop") return;
|
|
3274
|
+
e.delta && this.callbacks.transcript("assistant", String(e.delta), !1);
|
|
3275
|
+
return;
|
|
3276
|
+
case "response.output_audio.delta":
|
|
3277
|
+
case "response.audio.delta":
|
|
3278
|
+
this.audio(e);
|
|
3279
|
+
return;
|
|
3280
|
+
case "response.done":
|
|
3281
|
+
this.greetingHold.turnComplete(), this.responseDone(e.response || {});
|
|
3282
|
+
return;
|
|
3283
|
+
case "error": {
|
|
3284
|
+
const t = String(e.error?.code || "");
|
|
3285
|
+
if (t === "response_cancel_not_active") return;
|
|
3286
|
+
if (t === "conversation_already_has_active_response") {
|
|
3287
|
+
this.replyQueued = !0;
|
|
3288
|
+
return;
|
|
3289
|
+
}
|
|
3290
|
+
console.warn("[BeVyra OpenAI Realtime] server error", e.error);
|
|
3291
|
+
return;
|
|
3292
|
+
}
|
|
3293
|
+
}
|
|
3294
|
+
}
|
|
3295
|
+
userSpoke() {
|
|
3296
|
+
this.silence.userSpoke(), this.callbacks.userSpeech?.();
|
|
3297
|
+
}
|
|
3298
|
+
audio(e) {
|
|
3299
|
+
if (!e.delta || this.cancelled.has(String(e.response_id)) || this.gate.audio() === "drop") return;
|
|
3300
|
+
this.responding = !0, this.silence.assistantActive(), this.callbacks.status("responding");
|
|
3301
|
+
const t = dt(String(e.delta)), s = {
|
|
3302
|
+
sessionId: this.sessionId,
|
|
3303
|
+
turnId: this.turn,
|
|
3304
|
+
responseGenerationId: this.generation,
|
|
3305
|
+
startSample: this.sampleCursor,
|
|
3306
|
+
endSample: this.sampleCursor + t.length,
|
|
3307
|
+
sampleRate: Y,
|
|
3308
|
+
pcm24: t
|
|
3309
|
+
};
|
|
3310
|
+
this.sampleCursor = s.endSample, this.callbacks.assistantAudio(s);
|
|
3311
|
+
}
|
|
3312
|
+
responseDone(e) {
|
|
3313
|
+
const t = String(e.id || this.responseId || ""), s = this.cancelled.delete(t) || e.status === "cancelled";
|
|
3314
|
+
this.responseId = null, this.responding = !1;
|
|
3315
|
+
const i = s ? [] : (e.output || []).filter((r) => r?.type === "function_call").map((r) => ({ id: String(r.call_id || ""), name: String(r.name || ""), args: ut(r.arguments) }));
|
|
3316
|
+
if (this.gate.turnComplete() && !s) {
|
|
3317
|
+
const r = this.generation;
|
|
3318
|
+
Promise.resolve(this.callbacks.turnComplete?.()).finally(() => {
|
|
3319
|
+
this.closed || r !== this.generation || this.callbacks.status("listening");
|
|
3320
|
+
});
|
|
3321
|
+
}
|
|
3322
|
+
this.turn += 1, this.sampleCursor = 0, i.length && (i.forEach((r) => this.openToolCalls.add(r.id)), this.callbacks.toolCall?.(i)), this.replyQueued && !this.openToolCalls.size && this.reply();
|
|
3323
|
+
}
|
|
3324
|
+
/**
|
|
3325
|
+
* The renderer reports when the avatar is audible. Barge-in detection and
|
|
3326
|
+
* the mobile echo guard follow actual playback, which lags generation.
|
|
3327
|
+
*/
|
|
3328
|
+
setAssistantPlayback(e) {
|
|
3329
|
+
this.playbackActive = e, this.greetingHold.playback(e), this.microphone?.setAssistantSpeaking(e), e ? this.silence.assistantActive() : this.silence.assistantFinished();
|
|
3330
|
+
}
|
|
3331
|
+
/** Ask once, gently, whether the user needs anything else. */
|
|
3332
|
+
checkIn() {
|
|
3333
|
+
if (this.closed || this.playbackActive || this.socket?.readyState !== WebSocket.OPEN) return;
|
|
3334
|
+
const e = this.checkInPrompt();
|
|
3335
|
+
e && (console.info("[BeVyra OpenAI Realtime] silence check-in"), this.sendText(e));
|
|
3336
|
+
}
|
|
3337
|
+
/** The user started talking over the audible avatar. */
|
|
3338
|
+
localBargeIn() {
|
|
3339
|
+
this.closed || !this.playbackActive || (console.info("[BeVyra OpenAI Realtime] local barge-in"), this.greetingHold.release(), this.silence.userSpoke(), this.gate.localBargeIn(), this.cancelResponse(), this.interrupt(), this.callbacks.userSpeech?.());
|
|
3340
|
+
}
|
|
3341
|
+
interrupt() {
|
|
3342
|
+
this.callbacks.status("interrupted"), this.setAssistantPlayback(!1), this.generation += 1, this.sampleCursor = 0, this.callbacks.status("listening");
|
|
3343
|
+
}
|
|
3344
|
+
async setMuted(e) {
|
|
3345
|
+
this.silence.setMuted(e), await this.microphone?.setMuted(e), e && this.spokeSinceUnmute && this.socket?.readyState === WebSocket.OPEN && !this.reconnecting && this.send({ type: "input_audio_buffer.append", audio: Te(Y) }), this.spokeSinceUnmute = !1;
|
|
3346
|
+
}
|
|
3347
|
+
async close() {
|
|
3348
|
+
this.closed = !0, this.greetingHold.release(), this.silence.cancel(), this.socket?.readyState === WebSocket.OPEN && this.socket.close(1e3, "session ended"), await this.microphone?.stop(), this.callbacks.status("closed");
|
|
3349
|
+
}
|
|
3350
|
+
}
|
|
3351
|
+
async function ft(l, e) {
|
|
3352
|
+
l.status("connecting");
|
|
3353
|
+
const t = await J(e);
|
|
3354
|
+
return t.provider === "openai" ? re.connect(l, e, t) : ie.connect(l, e, t);
|
|
3355
|
+
}
|
|
3356
|
+
export {
|
|
3357
|
+
Fe as AvatarDevicePerformanceError,
|
|
3358
|
+
X as AvatarDeviceUnsupportedError,
|
|
3359
|
+
G as BeVyraAvatarSdk,
|
|
3360
|
+
j as ClientOnlyRuntime,
|
|
3361
|
+
ie as GeminiLiveClient,
|
|
3362
|
+
re as OpenAIRealtimeClient,
|
|
3363
|
+
ft as connectLiveSession,
|
|
3364
|
+
pt as loadVerifiedAvatarBundle,
|
|
3365
|
+
Ye as loadVerifiedRustAvatarBundle,
|
|
3366
|
+
Ze as loadVerifiedRustRuntime
|
|
3367
|
+
};
|