ava-engine 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +63 -0
- package/CLA.md +43 -0
- package/COMMERCIAL-LICENSE.md +43 -0
- package/LICENSE +73 -0
- package/README.md +240 -2
- package/THIRD_PARTY_NOTICES.md +38 -0
- package/TRADEMARKS.md +31 -0
- package/bin/ava-engine.mjs +12 -0
- package/lib/asr.mjs +147 -0
- package/lib/cli/apply.mjs +58 -0
- package/lib/cli/config.mjs +95 -0
- package/lib/cli/main.mjs +187 -0
- package/lib/cli/setup-server.mjs +195 -0
- package/lib/cli/setup.mjs +233 -0
- package/lib/fetch-models.mjs +104 -0
- package/lib/guard.mjs +92 -0
- package/lib/kokoro-model.mjs +51 -0
- package/lib/models.mjs +47 -0
- package/lib/openrouter.mjs +153 -0
- package/lib/ours.mjs +766 -0
- package/lib/platforms.mjs +201 -0
- package/lib/state.mjs +150 -0
- package/lib/stt.mjs +22 -0
- package/lib/tts.mjs +178 -0
- package/lib/workflow.mjs +18 -0
- package/package.json +80 -4
- package/public/app.js +1708 -0
- package/public/demo.js +420 -0
- package/public/favicon.svg +1 -0
- package/public/fonts/OFL.txt +96 -0
- package/public/fonts/inter-tight-var.woff2 +0 -0
- package/public/fonts/inter-var.woff2 +0 -0
- package/public/fonts/plex-mono-400.woff2 +0 -0
- package/public/fonts/plex-mono-500.woff2 +0 -0
- package/public/img/cta-1600.avif +0 -0
- package/public/img/cta-1600.webp +0 -0
- package/public/img/cta-2400.avif +0 -0
- package/public/img/cta-2400.webp +0 -0
- package/public/img/cta-960.avif +0 -0
- package/public/img/cta-960.webp +0 -0
- package/public/img/flows-1600.avif +0 -0
- package/public/img/flows-1600.webp +0 -0
- package/public/img/flows-2400.avif +0 -0
- package/public/img/flows-2400.webp +0 -0
- package/public/img/flows-960.avif +0 -0
- package/public/img/flows-960.webp +0 -0
- package/public/img/hero-1600.avif +0 -0
- package/public/img/hero-1600.webp +0 -0
- package/public/img/hero-2400.avif +0 -0
- package/public/img/hero-2400.webp +0 -0
- package/public/img/hero-960.avif +0 -0
- package/public/img/hero-960.webp +0 -0
- package/public/img/p-csm-480.avif +0 -0
- package/public/img/p-csm-480.webp +0 -0
- package/public/img/p-csm-800.avif +0 -0
- package/public/img/p-csm-800.webp +0 -0
- package/public/img/p-docs-480.avif +0 -0
- package/public/img/p-docs-480.webp +0 -0
- package/public/img/p-docs-800.avif +0 -0
- package/public/img/p-docs-800.webp +0 -0
- package/public/img/p-quiet-480.avif +0 -0
- package/public/img/p-quiet-480.webp +0 -0
- package/public/img/p-quiet-800.avif +0 -0
- package/public/img/p-quiet-800.webp +0 -0
- package/public/img/p-tickets-480.avif +0 -0
- package/public/img/p-tickets-480.webp +0 -0
- package/public/img/p-tickets-800.avif +0 -0
- package/public/img/p-tickets-800.webp +0 -0
- package/public/img/p-trial-480.avif +0 -0
- package/public/img/p-trial-480.webp +0 -0
- package/public/img/p-trial-800.avif +0 -0
- package/public/img/p-trial-800.webp +0 -0
- package/public/img/sees-1600.avif +0 -0
- package/public/img/sees-1600.webp +0 -0
- package/public/img/sees-2400.avif +0 -0
- package/public/img/sees-2400.webp +0 -0
- package/public/img/sees-960.avif +0 -0
- package/public/img/sees-960.webp +0 -0
- package/public/img/speaks-1600.avif +0 -0
- package/public/img/speaks-1600.webp +0 -0
- package/public/img/speaks-2400.avif +0 -0
- package/public/img/speaks-2400.webp +0 -0
- package/public/img/speaks-960.avif +0 -0
- package/public/img/speaks-960.webp +0 -0
- package/public/img/voice-1600.avif +0 -0
- package/public/img/voice-1600.webp +0 -0
- package/public/img/voice-2400.avif +0 -0
- package/public/img/voice-2400.webp +0 -0
- package/public/img/voice-960.avif +0 -0
- package/public/img/voice-960.webp +0 -0
- package/public/index.html +440 -0
- package/public/landing.html +975 -0
- package/public/pcm-worklet.js +11 -0
- package/public/perceptionWorker.js +103 -0
- package/public/rtc.js +60 -0
- package/public/setup.css +94 -0
- package/public/setup.html +129 -0
- package/public/setup.js +264 -0
- package/public/specialist.html +92 -0
- package/public/specialist.js +132 -0
- package/public/style.css +836 -0
- package/public/vendor/lenis.min.js +3 -0
- package/scripts/fetch-models.sh +15 -0
- package/server.mjs +502 -0
package/public/app.js
ADDED
|
@@ -0,0 +1,1708 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/* global makeRTC -- from rtc.js */
|
|
3
|
+
const $ = (id) => document.getElementById(id);
|
|
4
|
+
// Public-facing label for the talker in the UI — we don't surface the underlying
|
|
5
|
+
// model slug. Change this text if you want a different name shown.
|
|
6
|
+
const TALKER_LABEL = "Realtime";
|
|
7
|
+
const S = {
|
|
8
|
+
sessionId: null, sharing: false, stream: null, joined: false,
|
|
9
|
+
tts: { enabled: false }, voice: "browser", cc: true,
|
|
10
|
+
t0: 0, timer: null, proactive: true,
|
|
11
|
+
turnInFlight: false, obsTimer: null, lastObs: null, observing: false,
|
|
12
|
+
pendingChange: null, lastFrame: null, speaking: false,
|
|
13
|
+
latHistory: [], calls: { vision: 0, intent: 0, router: 0, planner: 0, voice: 0 },
|
|
14
|
+
models: {}, degraded: false,
|
|
15
|
+
mode: "voice", micDevice: "", ringing: false,
|
|
16
|
+
agentMode: "flexible", platform: null, platforms: [], screen: "onboard",
|
|
17
|
+
workflow: null, handoffTimer: null, knowTimer: null,
|
|
18
|
+
specialistPresent: false, bridgeSince: 0, bridgeTimer: null, rtc: null, rtcMic: null,
|
|
19
|
+
};
|
|
20
|
+
const video = $("video"), capcanvas = $("capcanvas"), hashcanvas = $("hashcanvas");
|
|
21
|
+
let pip = null;
|
|
22
|
+
|
|
23
|
+
/* ═══════════ boot ═══════════ */
|
|
24
|
+
(async function boot() {
|
|
25
|
+
try {
|
|
26
|
+
const h = await (await fetch("/api/health")).json();
|
|
27
|
+
S.tts = h.tts || { enabled: false };
|
|
28
|
+
S.stt = !!(h.stt && h.stt.enabled);
|
|
29
|
+
S.sttStream = !!(h.stt && h.stt.streaming); // offline sherpa streaming ASR on the server
|
|
30
|
+
// speech-recognition engine picker (green room)
|
|
31
|
+
const engines = (h.stt && h.stt.engines) || [];
|
|
32
|
+
// Default to multilingual SenseVoice when it's available (the user's pick).
|
|
33
|
+
S.sttEngine = (engines.find((e) => e.id === "sensevoice") ? "sensevoice" : (h.stt && h.stt.defaultEngine)) || (engines[0] && engines[0].id) || "zipformer";
|
|
34
|
+
const ss = $("sttsel");
|
|
35
|
+
if (ss) {
|
|
36
|
+
if (engines.length) {
|
|
37
|
+
ss.innerHTML = engines.map((e) => `<option value="${esc(e.id)}">${esc(e.label)}</option>`).join("");
|
|
38
|
+
ss.value = S.sttEngine;
|
|
39
|
+
const note = () => { $("sttnote").textContent = S.sttEngine === "sensevoice" ? "Multilingual — more accurate on accents + other languages; captions appear when you finish speaking" : "English — fast, with live word-by-word captions as you talk"; };
|
|
40
|
+
ss.onchange = () => { S.sttEngine = ss.value; note(); };
|
|
41
|
+
note();
|
|
42
|
+
} else { ss.innerHTML = "<option>browser</option>"; ss.disabled = true; $("sttnote").textContent = "using the browser's built-in recognizer"; }
|
|
43
|
+
}
|
|
44
|
+
S.models = h.models || {};
|
|
45
|
+
if (S.models.talker) { $("modelchip").textContent = TALKER_LABEL; renderConTurn(0); }
|
|
46
|
+
const provs = (S.tts.providers || [{ id: "browser", label: "System", enabled: true }]).filter((p) => p.enabled);
|
|
47
|
+
const opts = provs.map((p) => `<option value="${p.id}">${esc(p.label)}</option>`).join("");
|
|
48
|
+
S.voice = S.tts.default || "browser";
|
|
49
|
+
for (const id of ["voicesel", "voicesel2"]) {
|
|
50
|
+
const sel = $(id);
|
|
51
|
+
sel.innerHTML = opts; sel.value = S.voice;
|
|
52
|
+
sel.onchange = () => { S.voice = sel.value; $("voicesel").value = S.voice; $("voicesel2").value = S.voice; };
|
|
53
|
+
}
|
|
54
|
+
initOnboard(h);
|
|
55
|
+
} catch {}
|
|
56
|
+
populateMics();
|
|
57
|
+
})();
|
|
58
|
+
|
|
59
|
+
/* ═══════════ onboard: agent mode + platform pickers ═══════════ */
|
|
60
|
+
function initOnboard(h) {
|
|
61
|
+
const modes = (h.modes && h.modes.length) ? h.modes : [
|
|
62
|
+
{ id: "flexible", name: "Flexible agent", tagline: "Sees your screen and helps with anything you're doing." },
|
|
63
|
+
{ id: "onboarding", name: "Onboarding agent", tagline: "Focused on getting a new user into their dashboard on a chosen platform." },
|
|
64
|
+
];
|
|
65
|
+
S.platforms = h.platforms || [];
|
|
66
|
+
$("obModes").innerHTML = modes.map((m) => `
|
|
67
|
+
<button type="button" class="ob-mode${m.id === S.agentMode ? " selected" : ""}" data-mode="${esc(m.id)}">
|
|
68
|
+
<span class="ob-radio"></span>
|
|
69
|
+
<span class="ob-modebody"><span class="ob-modename">${esc(m.name)}</span><span class="ob-modedesc">${esc(m.tagline)}</span></span>
|
|
70
|
+
</button>`).join("");
|
|
71
|
+
document.querySelectorAll(".ob-mode").forEach((el) => { el.onclick = () => selectMode(el.dataset.mode); });
|
|
72
|
+
|
|
73
|
+
$("obPlats").innerHTML = S.platforms.filter((p) => !p.demoOnly).map((p) => `
|
|
74
|
+
<button type="button" class="ob-plat" data-id="${esc(p.id)}">
|
|
75
|
+
<i class="ob-platdot" style="background:${esc(p.accent || "#4F46E5")}"></i>
|
|
76
|
+
<span class="ob-platmeta"><span class="ob-platname">${esc(p.name)}</span><span class="ob-platcat">${esc(p.category || "")}</span></span>
|
|
77
|
+
</button>`).join("");
|
|
78
|
+
document.querySelectorAll(".ob-plat").forEach((el) => { el.onclick = () => selectPlatform(el.dataset.id); });
|
|
79
|
+
|
|
80
|
+
selectMode(S.agentMode);
|
|
81
|
+
}
|
|
82
|
+
function selectMode(id) {
|
|
83
|
+
S.agentMode = id === "onboarding" ? "onboarding" : "flexible";
|
|
84
|
+
document.querySelectorAll(".ob-mode").forEach((el) => el.classList.toggle("selected", el.dataset.mode === S.agentMode));
|
|
85
|
+
const onb = S.agentMode === "onboarding";
|
|
86
|
+
$("obPlatformBlock").hidden = !onb;
|
|
87
|
+
const pickable = S.platforms.filter((p) => !p.demoOnly);
|
|
88
|
+
if (onb && !S.platform && pickable.length) selectPlatform(pickable[0].id);
|
|
89
|
+
}
|
|
90
|
+
function selectPlatform(id) {
|
|
91
|
+
const p = S.platforms.find((x) => x.id === id);
|
|
92
|
+
if (!p) return;
|
|
93
|
+
S.platform = p;
|
|
94
|
+
document.querySelectorAll(".ob-plat").forEach((el) => el.classList.toggle("selected", el.dataset.id === id));
|
|
95
|
+
$("obPlatNote").textContent = `${p.name} — ${p.tagline || ""}`;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/* ═══════════ screen state machine ═══════════ */
|
|
99
|
+
function setScreen(name) {
|
|
100
|
+
S.screen = name;
|
|
101
|
+
document.body.className = "screen-" + name;
|
|
102
|
+
if (name === "connecting") runCnWave();
|
|
103
|
+
}
|
|
104
|
+
// phones and some browsers can't capture a screen — hide the share controls there (class on <html>: setScreen owns body's)
|
|
105
|
+
if (!navigator.mediaDevices || !navigator.mediaDevices.getDisplayMedia) document.documentElement.classList.add("noshare");
|
|
106
|
+
|
|
107
|
+
/* connecting pre-roll: a voice ribbon that wakes up as each step completes */
|
|
108
|
+
function runCnWave() {
|
|
109
|
+
const cv = $("cnWave");
|
|
110
|
+
if (!cv || cv.dataset.running) return;
|
|
111
|
+
cv.dataset.running = "1";
|
|
112
|
+
const g = cv.getContext("2d");
|
|
113
|
+
const accent = getComputedStyle(document.documentElement).getPropertyValue("--brand-a").trim() || "#6366F1";
|
|
114
|
+
let amp = 2;
|
|
115
|
+
const frame = (t) => {
|
|
116
|
+
if (S.screen !== "connecting") { delete cv.dataset.running; return; }
|
|
117
|
+
const r = cv.getBoundingClientRect(), d = Math.min(2, devicePixelRatio || 1);
|
|
118
|
+
if (cv.width !== Math.round(r.width * d)) { cv.width = Math.round(r.width * d); cv.height = Math.round(r.height * d); }
|
|
119
|
+
g.setTransform(d, 0, 0, d, 0, 0);
|
|
120
|
+
const w = r.width, h = r.height, done = document.querySelectorAll("#cnSteps li.done").length;
|
|
121
|
+
amp += (3 + done * 6 + 3 * Math.abs(Math.sin(t * 0.004)) - amp) * 0.05;
|
|
122
|
+
g.clearRect(0, 0, w, h);
|
|
123
|
+
for (let i = 0; i < 3; i++) {
|
|
124
|
+
g.beginPath(); g.globalAlpha = 0.9 - i * 0.28; g.lineWidth = 1.8 - i * 0.4; g.strokeStyle = i ? "#A5B4FC" : accent;
|
|
125
|
+
for (let x = 0; x <= w; x += 3) {
|
|
126
|
+
const n = x / w, env = Math.pow(Math.sin(Math.PI * n), 2);
|
|
127
|
+
const y = h / 2 + env * amp * (1 - i * 0.2) * (Math.sin(n * (8 + i * 2) + t * (0.0022 + i * 0.0006) + i) * 0.7 + Math.sin(n * 17 - t * 0.003 + i) * 0.3);
|
|
128
|
+
x ? g.lineTo(x, y) : g.moveTo(x, y);
|
|
129
|
+
}
|
|
130
|
+
g.stroke();
|
|
131
|
+
}
|
|
132
|
+
requestAnimationFrame(frame);
|
|
133
|
+
};
|
|
134
|
+
requestAnimationFrame(frame);
|
|
135
|
+
}
|
|
136
|
+
function cnStep(step, state) {
|
|
137
|
+
const li = document.querySelector(`#cnSteps li[data-step="${step}"]`);
|
|
138
|
+
if (li) { li.classList.remove("active", "done"); if (state) li.classList.add(state); }
|
|
139
|
+
}
|
|
140
|
+
function resetConnecting(title) {
|
|
141
|
+
$("cnTitle").textContent = title || "Connecting you to Ava…";
|
|
142
|
+
["session", "voice", "greet"].forEach((s) => cnStep(s, null));
|
|
143
|
+
}
|
|
144
|
+
const wait = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
145
|
+
|
|
146
|
+
/* central stage layout: Meet grid (pre-share) ↔ shared screen ↔ compact tiles */
|
|
147
|
+
function updateStageLayout() {
|
|
148
|
+
// the console takes the transcript's slot; the stage and tracker stay put (they just get narrower)
|
|
149
|
+
$("people").hidden = !$("console").hidden;
|
|
150
|
+
$("meetgrid").hidden = !(S.joined && !S.sharing);
|
|
151
|
+
$("minitiles").hidden = !(S.joined && S.sharing);
|
|
152
|
+
$("workflow").hidden = !(S.joined && S.workflow);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/* ═══════════ call sounds (WebAudio — no assets) ═══════════ */
|
|
156
|
+
function unlockAudio() {
|
|
157
|
+
if (!audioCtx) { try { audioCtx = new (window.AudioContext || window.webkitAudioContext)(); } catch { return null; } }
|
|
158
|
+
if (audioCtx.state === "suspended") audioCtx.resume().catch(() => {});
|
|
159
|
+
return audioCtx;
|
|
160
|
+
}
|
|
161
|
+
function blip(freq, at, dur = 0.1, vol = 0.05) {
|
|
162
|
+
const ctx = unlockAudio(); if (!ctx) return;
|
|
163
|
+
const osc = ctx.createOscillator(), g = ctx.createGain();
|
|
164
|
+
osc.type = "sine"; osc.frequency.value = freq;
|
|
165
|
+
const t = ctx.currentTime + at;
|
|
166
|
+
g.gain.setValueAtTime(0, t);
|
|
167
|
+
g.gain.linearRampToValueAtTime(vol, t + 0.02);
|
|
168
|
+
g.gain.exponentialRampToValueAtTime(0.001, t + dur);
|
|
169
|
+
osc.connect(g); g.connect(ctx.destination);
|
|
170
|
+
osc.start(t); osc.stop(t + dur + 0.05);
|
|
171
|
+
}
|
|
172
|
+
let ringTimer = null;
|
|
173
|
+
function startRing() {
|
|
174
|
+
stopRing();
|
|
175
|
+
const ring = () => { blip(600, 0, 0.12, 0.045); blip(760, 0.18, 0.14, 0.045); };
|
|
176
|
+
ring(); ringTimer = setInterval(ring, 1700);
|
|
177
|
+
S.ringing = true;
|
|
178
|
+
}
|
|
179
|
+
function stopRing() { if (ringTimer) clearInterval(ringTimer); ringTimer = null; S.ringing = false; }
|
|
180
|
+
function joinChime() { blip(660, 0, 0.12, 0.06); blip(880, 0.13, 0.18, 0.06); }
|
|
181
|
+
function leaveChime() { blip(660, 0, 0.12, 0.05); blip(440, 0.13, 0.2, 0.05); }
|
|
182
|
+
function ackBlip() { blip(920, 0, 0.05, 0.028); } // "heard you" — after your utterance lands
|
|
183
|
+
function yieldTick() { blip(520, 0, 0.06, 0.03); } // "go ahead" — she steps back when you cut in
|
|
184
|
+
|
|
185
|
+
// Soft room tone while a voice call is live — real calls are never dead silent,
|
|
186
|
+
// and the faint bed makes her pauses feel like presence instead of a crash.
|
|
187
|
+
let roomTone = null;
|
|
188
|
+
function startRoomTone() {
|
|
189
|
+
const ctx = unlockAudio(); if (!ctx || roomTone) return;
|
|
190
|
+
const len = 2 * ctx.sampleRate, buf = ctx.createBuffer(1, len, ctx.sampleRate);
|
|
191
|
+
const d = buf.getChannelData(0);
|
|
192
|
+
let last = 0;
|
|
193
|
+
for (let i = 0; i < len; i++) { const w = Math.random() * 2 - 1; last = (last + 0.02 * w) / 1.02; d[i] = last * 3.5; } // brown-ish noise
|
|
194
|
+
const src = ctx.createBufferSource(); src.buffer = buf; src.loop = true;
|
|
195
|
+
const g = ctx.createGain(); g.gain.value = 0.006;
|
|
196
|
+
src.connect(g); g.connect(ctx.destination); src.start();
|
|
197
|
+
roomTone = { src, g };
|
|
198
|
+
}
|
|
199
|
+
function stopRoomTone() { if (!roomTone) return; try { roomTone.src.stop(); } catch {} roomTone = null; }
|
|
200
|
+
|
|
201
|
+
/* ═══════════ join / leave ═══════════ */
|
|
202
|
+
// Mode defaults: a voice call keeps captions off (you're listening, and the
|
|
203
|
+
// transcript rail has the text); text-only keeps Ava's voice off.
|
|
204
|
+
function applyModeDefaults(mode) {
|
|
205
|
+
const voiceOn = mode === "voice";
|
|
206
|
+
$("speak").checked = voiceOn;
|
|
207
|
+
$("speakbtn").classList.toggle("on", voiceOn); $("speakbtn").setAttribute("aria-pressed", voiceOn);
|
|
208
|
+
S.cc = !voiceOn;
|
|
209
|
+
$("ccbtn").classList.toggle("on", S.cc); $("ccbtn").setAttribute("aria-pressed", S.cc);
|
|
210
|
+
$("capwrap").hidden = !S.cc;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
async function joinCall(mode) {
|
|
214
|
+
if (S.joined) return true;
|
|
215
|
+
S.mode = mode || S.mode || "voice";
|
|
216
|
+
const onb = S.agentMode === "onboarding" && S.platform;
|
|
217
|
+
if (onb && S.platform.accent) applyBrand(S.platform.accent); else resetBrand();
|
|
218
|
+
stopMicTest();
|
|
219
|
+
// the connect ring covers the whole warm-up: session, voice model, greeting
|
|
220
|
+
unlockAudio(); startRing();
|
|
221
|
+
setScreen("connecting");
|
|
222
|
+
resetConnecting(onb ? `Connecting you to your ${S.platform.name} guide…` : "Connecting you to Ava…");
|
|
223
|
+
cnStep("session", "active");
|
|
224
|
+
// prewarm the voice while the ring plays (the first synth after boot is the
|
|
225
|
+
// slow one — spend it on a throwaway "Hi." instead of the greeting)
|
|
226
|
+
if (S.mode === "voice" && (S.voice === "kokoro" || S.voice === "elevenlabs")) {
|
|
227
|
+
fetch("/api/tts", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ text: "Hi.", provider: S.voice }) }).then((r) => r.arrayBuffer()).catch(() => {});
|
|
228
|
+
}
|
|
229
|
+
try {
|
|
230
|
+
const res = await fetch("/api/session", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ mode: S.agentMode, platformId: S.platform ? S.platform.id : null }) });
|
|
231
|
+
const r = await res.json().catch(() => ({}));
|
|
232
|
+
if (!res.ok || !r.sessionId) throw new Error(r.error || `server error ${res.status}`);
|
|
233
|
+
S.sessionId = r.sessionId; S.joined = true;
|
|
234
|
+
} catch (e) {
|
|
235
|
+
stopRing();
|
|
236
|
+
setScreen("onboard");
|
|
237
|
+
$("joinstate").textContent = e instanceof TypeError ? "couldn't connect — is the server running?" : e.message;
|
|
238
|
+
return false;
|
|
239
|
+
}
|
|
240
|
+
cnStep("session", "done"); cnStep("voice", "active");
|
|
241
|
+
await wait(650); // let the voice-warm step read on the connecting animation
|
|
242
|
+
cnStep("voice", "done"); cnStep("greet", "active");
|
|
243
|
+
await wait(350);
|
|
244
|
+
|
|
245
|
+
// land in the room (Google-Meet style tiles until they share)
|
|
246
|
+
$("joinstate").textContent = "";
|
|
247
|
+
setTopbarForMode();
|
|
248
|
+
setScreen("call");
|
|
249
|
+
applyModeDefaults(S.mode);
|
|
250
|
+
updateStageLayout();
|
|
251
|
+
$("avastatusBig").textContent = "joining…";
|
|
252
|
+
$("endcall").disabled = false;
|
|
253
|
+
setPillLive(true);
|
|
254
|
+
setHealth("healthy", "HEALTHY");
|
|
255
|
+
S.t0 = Date.now(); startTimer();
|
|
256
|
+
S.record = []; // fresh session tape for the end-of-call replay
|
|
257
|
+
setStuck(false); // clear any leftover rethinking indicator
|
|
258
|
+
S.workflow = null; renderWorkflow(null); clearTimeout(S.handoffTimer); $("wfKnow").hidden = true;
|
|
259
|
+
S.specialistPresent = false; S.specialistKey = null; S.handoff = null; S.bridgeSince = 0; stopBridge(); stopUserRTC();
|
|
260
|
+
if (S.mode === "voice") { startLiveMic(); startRoomTone(); } // always-on detection + a faint call-presence bed
|
|
261
|
+
greet(); // the ring stops (with a join chime) when the greeting starts
|
|
262
|
+
setTimeout(() => { if (S.ringing) stopRing(); }, 12000); // safety: never ring forever
|
|
263
|
+
return true;
|
|
264
|
+
}
|
|
265
|
+
function setTopbarForMode() {
|
|
266
|
+
const onb = S.agentMode === "onboarding" && S.platform;
|
|
267
|
+
$("apptitle").textContent = onb ? `${S.platform.name} Onboarding` : "Flexible Agent";
|
|
268
|
+
$("topsub").textContent = onb ? `onboarding guide · ${S.platform.category || "platform"}` : "screen-aware assistant";
|
|
269
|
+
$("footmode").textContent = onb ? `onboarding ${S.platform.name} · transcript on` : "recording off · transcript on";
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/* ═══════════ per-platform brand theming ═══════════ */
|
|
273
|
+
function hexToRgb(h) { h = String(h || "").replace("#", ""); if (h.length === 3) h = h.split("").map((c) => c + c).join(""); const n = parseInt(h || "4F46E5", 16); return [(n >> 16) & 255, (n >> 8) & 255, n & 255]; }
|
|
274
|
+
function shade(hex, f) { const [r, g, b] = hexToRgb(hex); const m = (x) => Math.max(0, Math.min(255, Math.round(f < 0 ? x * (1 + f) : x + (255 - x) * f))); return `rgb(${m(r)},${m(g)},${m(b)})`; }
|
|
275
|
+
function applyBrand(hex) {
|
|
276
|
+
const r = document.documentElement, [R, G, B] = hexToRgb(hex);
|
|
277
|
+
const lum = (0.299 * R + 0.587 * G + 0.114 * B) / 255;
|
|
278
|
+
r.style.setProperty("--amber", hex);
|
|
279
|
+
r.style.setProperty("--amber-ink", lum > 0.6 ? "#12141A" : "#FFFFFF");
|
|
280
|
+
r.style.setProperty("--brandtint", `rgba(${R},${G},${B},0.10)`);
|
|
281
|
+
r.style.setProperty("--brandline", `rgba(${R},${G},${B},0.32)`);
|
|
282
|
+
r.style.setProperty("--brand-a", shade(hex, 0.16));
|
|
283
|
+
r.style.setProperty("--brand-b", hex);
|
|
284
|
+
r.style.setProperty("--brand-c", shade(hex, -0.34));
|
|
285
|
+
}
|
|
286
|
+
function resetBrand() { const r = document.documentElement; ["--amber", "--amber-ink", "--brandtint", "--brandline", "--brand-a", "--brand-b", "--brand-c"].forEach((p) => r.style.removeProperty(p)); }
|
|
287
|
+
// Closing the tab mid-call ends the session too (revokes any handoff link).
|
|
288
|
+
window.addEventListener("pagehide", () => {
|
|
289
|
+
if (S.joined && S.sessionId) fetch("/api/end", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sessionId: S.sessionId }), keepalive: true }).catch(() => {});
|
|
290
|
+
});
|
|
291
|
+
$("endcall").onclick = () => {
|
|
292
|
+
if (!S.joined) return;
|
|
293
|
+
// grab what the report needs BEFORE teardown wipes the session id / clock
|
|
294
|
+
const sid = S.sessionId, durMs = Date.now() - S.t0, tape = S.record || [];
|
|
295
|
+
if (S.specialistKey || handoffActive()) fetch("/api/handoff", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sessionId: sid, action: "cancel" }) }).catch(() => {});
|
|
296
|
+
if (S.turnAbort) { S.turnAbort.abort(); S.turnAbort = null; }
|
|
297
|
+
S.turnInFlight = false;
|
|
298
|
+
stopShare(); S.joined = false; S.sessionId = null; stopTimer(); stopRing();
|
|
299
|
+
stopLiveMic(); stopRoomTone(); S.pendingUtterance = null;
|
|
300
|
+
$("endcall").disabled = true;
|
|
301
|
+
$("capwrap").hidden = true;
|
|
302
|
+
updateStageLayout(); // S.joined is false now → hides meet grid + tiles
|
|
303
|
+
$("joinstate").textContent = "";
|
|
304
|
+
setPillLive(false); setHealth("ready", "READY");
|
|
305
|
+
stopStreamPlayback();
|
|
306
|
+
if ("speechSynthesis" in window) speechSynthesis.cancel();
|
|
307
|
+
if (ttsAudio) { try { ttsAudio.pause(); } catch {} ttsAudio = null; }
|
|
308
|
+
leaveChime();
|
|
309
|
+
if (pip) { try { pip.win.close(); } catch {} } // its pagehide handler clears the timer + observer
|
|
310
|
+
resetBrand();
|
|
311
|
+
setStuck(false);
|
|
312
|
+
S.workflow = null; renderWorkflow(null); clearTimeout(S.handoffTimer); $("wfKnow").hidden = true;
|
|
313
|
+
S.specialistPresent = false; S.specialistKey = null; S.handoff = null; S.bridgeSince = 0; stopBridge(); stopUserRTC();
|
|
314
|
+
setAvaState("standby");
|
|
315
|
+
laneIdle("vision", "waiting for a window"); laneIdle("intent", "waiting for input");
|
|
316
|
+
laneIdle("router", "idle"); laneIdle("planner", "no threads"); laneIdle("voice", "muted");
|
|
317
|
+
showReport(sid, durMs, tape);
|
|
318
|
+
};
|
|
319
|
+
function setPillLive(on) {
|
|
320
|
+
const p = $("livepill");
|
|
321
|
+
p.className = "pill " + (on ? "live" : "standby");
|
|
322
|
+
$("livepilltext").textContent = on ? "LIVE" : "STANDBY";
|
|
323
|
+
}
|
|
324
|
+
function setHealth(cls, text) { const h = $("health"); h.className = "health " + cls; $("healthtext").textContent = text; }
|
|
325
|
+
function startTimer() {
|
|
326
|
+
stopTimer();
|
|
327
|
+
S.timer = setInterval(() => {
|
|
328
|
+
const s = Math.floor((Date.now() - S.t0) / 1000);
|
|
329
|
+
$("timer").textContent = [Math.floor(s / 3600), Math.floor((s % 3600) / 60), s % 60].map((x) => String(x).padStart(2, "0")).join(":");
|
|
330
|
+
}, 1000);
|
|
331
|
+
}
|
|
332
|
+
function stopTimer() { if (S.timer) clearInterval(S.timer); S.timer = null; }
|
|
333
|
+
|
|
334
|
+
/* ═══════════ green room (pre-join gear check) ═══════════ */
|
|
335
|
+
$("joinvoice").onclick = () => joinCall("voice");
|
|
336
|
+
$("jointext").onclick = () => joinCall("text");
|
|
337
|
+
$("meetShare").onclick = () => shareToggle();
|
|
338
|
+
$("sharemain").onclick = async () => { if (await joinCall(S.joined ? S.mode : "voice")) { if (!S.sharing) shareToggle(); } };
|
|
339
|
+
|
|
340
|
+
async function populateMics() {
|
|
341
|
+
try {
|
|
342
|
+
const devs = await navigator.mediaDevices.enumerateDevices();
|
|
343
|
+
const ins = devs.filter((d) => d.kind === "audioinput" && d.deviceId && d.deviceId !== "default");
|
|
344
|
+
$("micsel").innerHTML = '<option value="">System default</option>' +
|
|
345
|
+
ins.map((d, i) => `<option value="${esc(d.deviceId)}">${esc(d.label || "Microphone " + (i + 1))}</option>`).join("");
|
|
346
|
+
if (S.micDevice) $("micsel").value = S.micDevice;
|
|
347
|
+
} catch {}
|
|
348
|
+
}
|
|
349
|
+
$("micsel").onchange = () => { S.micDevice = $("micsel").value; };
|
|
350
|
+
|
|
351
|
+
let micTest = null;
|
|
352
|
+
async function toggleMicTest() {
|
|
353
|
+
if (micTest) { stopMicTest(); return; }
|
|
354
|
+
let stream;
|
|
355
|
+
try { stream = await navigator.mediaDevices.getUserMedia({ audio: S.micDevice ? { deviceId: { exact: S.micDevice } } : true }); }
|
|
356
|
+
catch { $("joinstate").textContent = "mic blocked — check browser permissions"; return; }
|
|
357
|
+
populateMics(); // labels become available once permission is granted
|
|
358
|
+
const ctx = unlockAudio(); if (!ctx) { stream.getTracks().forEach((t) => t.stop()); return; }
|
|
359
|
+
const an = ctx.createAnalyser(); an.fftSize = 512;
|
|
360
|
+
ctx.createMediaStreamSource(stream).connect(an);
|
|
361
|
+
const data = new Uint8Array(an.fftSize);
|
|
362
|
+
const bars = [...document.querySelectorAll("#miclevel i")];
|
|
363
|
+
micTest = { stream, raf: 0 };
|
|
364
|
+
const tick = () => {
|
|
365
|
+
if (!micTest) return;
|
|
366
|
+
an.getByteTimeDomainData(data);
|
|
367
|
+
let peak = 0;
|
|
368
|
+
for (let i = 0; i < data.length; i++) { const v = Math.abs(data[i] - 128) / 128; if (v > peak) peak = v; }
|
|
369
|
+
const lit = Math.min(bars.length, Math.round(peak * bars.length * 2.2));
|
|
370
|
+
bars.forEach((b, i) => b.classList.toggle("lit", i < lit));
|
|
371
|
+
micTest.raf = requestAnimationFrame(tick);
|
|
372
|
+
};
|
|
373
|
+
tick();
|
|
374
|
+
$("mictest").classList.add("on"); $("mictest").textContent = "Stop";
|
|
375
|
+
}
|
|
376
|
+
function stopMicTest() {
|
|
377
|
+
if (!micTest) return;
|
|
378
|
+
cancelAnimationFrame(micTest.raf);
|
|
379
|
+
micTest.stream.getTracks().forEach((t) => t.stop());
|
|
380
|
+
micTest = null;
|
|
381
|
+
document.querySelectorAll("#miclevel i").forEach((b) => b.classList.remove("lit"));
|
|
382
|
+
$("mictest").classList.remove("on"); $("mictest").textContent = "Test";
|
|
383
|
+
}
|
|
384
|
+
$("mictest").onclick = toggleMicTest;
|
|
385
|
+
// Preview doubles as the warm-up: it runs the full TTS path end to end.
|
|
386
|
+
$("voicetest").onclick = () => { unlockAudio(); speak("Hi, I'm Ava. This is how I'll sound on the call.", true, true); };
|
|
387
|
+
|
|
388
|
+
/* ═══════════ screen share ═══════════ */
|
|
389
|
+
function shareToggle() {
|
|
390
|
+
if (S.sharing) { stopShare(); return; }
|
|
391
|
+
navigator.mediaDevices.getDisplayMedia({ video: { frameRate: 5 }, audio: false }).then((st) => {
|
|
392
|
+
S.stream = st; video.srcObject = st; video.classList.add("on");
|
|
393
|
+
if (pip && pip.video) { pip.video.srcObject = st; pip.video.play().catch(() => {}); }
|
|
394
|
+
S.sharing = true; updateStageLayout();
|
|
395
|
+
const track = st.getVideoTracks()[0];
|
|
396
|
+
const win = (track.label || "your window").replace(/^(window|screen|web-contents-media-stream.*?):\s*/i, "").slice(0, 42) || "your window";
|
|
397
|
+
$("privacy").hidden = false; $("privwin").textContent = win;
|
|
398
|
+
$("share").classList.add("armed"); $("sharelabel").textContent = "Sharing " + win.slice(0, 24);
|
|
399
|
+
const ps = pipEl("pipShare"); if (ps) ps.classList.add("armed");
|
|
400
|
+
$("stagelabel").textContent = "stage · " + win;
|
|
401
|
+
$("youshare").hidden = false; $("miniShareDot").classList.add("on");
|
|
402
|
+
laneRun("vision", "watching " + win.slice(0, 26), true);
|
|
403
|
+
track.addEventListener("ended", stopShare);
|
|
404
|
+
// Prefer the off-main-thread worker (never throttled). Fall back to the rVFC loop.
|
|
405
|
+
if (!startWorkerDetection(track)) startObserve();
|
|
406
|
+
if (!pip) openPiP(); // pop out WHEN sharing starts (this click is the gesture)
|
|
407
|
+
refreshUserRTC();
|
|
408
|
+
}).catch(() => {});
|
|
409
|
+
}
|
|
410
|
+
function stopShare() {
|
|
411
|
+
const was = S.sharing;
|
|
412
|
+
if (S.stream) S.stream.getTracks().forEach((t) => t.stop());
|
|
413
|
+
S.stream = null; S.sharing = false; video.classList.remove("on");
|
|
414
|
+
updateStageLayout();
|
|
415
|
+
$("privacy").hidden = true;
|
|
416
|
+
$("share").classList.remove("armed"); $("sharelabel").textContent = "Share a window";
|
|
417
|
+
const ps = pipEl("pipShare"); if (ps) ps.classList.remove("armed");
|
|
418
|
+
$("stagelabel").textContent = "stage · nothing shared";
|
|
419
|
+
$("youshare").hidden = true; $("miniShareDot").classList.remove("on");
|
|
420
|
+
$("callout").hidden = true;
|
|
421
|
+
S.pendingChange = null; S.lastFrame = null;
|
|
422
|
+
stopWorkerDetection(); stopObserve();
|
|
423
|
+
if (was) refreshUserRTC();
|
|
424
|
+
if (was && S.sessionId) {
|
|
425
|
+
fetch("/api/observe", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sessionId: S.sessionId, stopped: true }) }).catch(() => {});
|
|
426
|
+
laneIdle("vision", "sharing stopped");
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
$("share").onclick = async () => { if (await joinCall(S.joined ? S.mode : "voice")) shareToggle(); };
|
|
430
|
+
|
|
431
|
+
/* ═══════════ off-main-thread detection (Worker + MediaStreamTrackProcessor) ═══════════ */
|
|
432
|
+
let perceptWorker = null;
|
|
433
|
+
function startWorkerDetection(track) {
|
|
434
|
+
if (!("MediaStreamTrackProcessor" in window) || typeof Worker === "undefined") return false;
|
|
435
|
+
try {
|
|
436
|
+
perceptWorker = new Worker("perceptionWorker.js");
|
|
437
|
+
perceptWorker.onmessage = (e) => { if (e.data && e.data.type === "change") onWorkerChange(e.data); };
|
|
438
|
+
const proc = new MediaStreamTrackProcessor({ track });
|
|
439
|
+
perceptWorker.postMessage({ type: "start", readable: proc.readable }, [proc.readable]);
|
|
440
|
+
return true;
|
|
441
|
+
} catch { perceptWorker = null; return false; }
|
|
442
|
+
}
|
|
443
|
+
function stopWorkerDetection() { if (perceptWorker) { try { perceptWorker.postMessage({ type: "stop" }); perceptWorker.terminate(); } catch {} perceptWorker = null; } }
|
|
444
|
+
function blobToDataURL(b) { return new Promise((res) => { const fr = new FileReader(); fr.onload = () => res(fr.result); fr.readAsDataURL(b); }); }
|
|
445
|
+
// Coalesce a burst of change-frames into ONE read of the SETTLED screen. A single
|
|
446
|
+
// action (click → navigate → content loads) trips the detector several times; if
|
|
447
|
+
// we read each one, the same action shows up as two+ vision reads and Ava reacts
|
|
448
|
+
// twice. So on a change we (re)start a short settle timer and only read once the
|
|
449
|
+
// screen has stopped changing. The screen-share START reads promptly (no settle).
|
|
450
|
+
let obsSettleTimer = null, obsSettleFrame = null;
|
|
451
|
+
function onWorkerChange(m) {
|
|
452
|
+
if (m.announce) { doObserve(m); return; }
|
|
453
|
+
obsSettleFrame = m;
|
|
454
|
+
clearTimeout(obsSettleTimer);
|
|
455
|
+
obsSettleTimer = setTimeout(() => { const mm = obsSettleFrame; obsSettleFrame = null; if (mm) doObserve(mm); }, 550);
|
|
456
|
+
}
|
|
457
|
+
async function doObserve(m) {
|
|
458
|
+
// Busy → queue the change instead of dropping it (the worker's own hash has
|
|
459
|
+
// already advanced, so a dropped change would never re-fire).
|
|
460
|
+
if (S.turnInFlight || S.observing) { S.pendingChange = m; return; }
|
|
461
|
+
S.observing = true;
|
|
462
|
+
laneRun("vision", m.announce ? "you shared — taking a look" : "screen changed — reading", true);
|
|
463
|
+
try {
|
|
464
|
+
const frame = await blobToDataURL(m.blob);
|
|
465
|
+
S.lastFrame = { frame, hash: m.hash };
|
|
466
|
+
// changed:true → the worker's two-signal detector already decided this IS a
|
|
467
|
+
// change; the server must re-read even if the coarse hash gate disagrees
|
|
468
|
+
const r = await fetch("/api/observe", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sessionId: S.sessionId, frame, frameHash: m.hash, changed: true }) }).then((x) => x.json());
|
|
469
|
+
handleObserve(r, m.announce);
|
|
470
|
+
} catch {}
|
|
471
|
+
S.observing = false;
|
|
472
|
+
processPendingChange();
|
|
473
|
+
}
|
|
474
|
+
function processPendingChange() {
|
|
475
|
+
if (!S.pendingChange || S.turnInFlight || S.observing || !S.sharing) return;
|
|
476
|
+
const m = S.pendingChange; S.pendingChange = null;
|
|
477
|
+
doObserve(m); // already a settled/queued change — read it directly, no re-debounce
|
|
478
|
+
}
|
|
479
|
+
function meaningfulActivity(a) { return a && !/^\s*no( real)? change/i.test(a); }
|
|
480
|
+
function handleObserve(r, announce) {
|
|
481
|
+
if (!r || r.error) { laneIdle("vision", "read failed — will retry"); return; }
|
|
482
|
+
const worthIt = announce || r.navChanged || r.stepChanged || meaningfulActivity(r.activity);
|
|
483
|
+
if (r.ran && r.screenState) {
|
|
484
|
+
S.calls.vision += 1;
|
|
485
|
+
laneDone("vision", null, `v${r.screenState.v} · ${r.screenState.app || "?"}`);
|
|
486
|
+
if (worthIt) addSysLine("vision", `${r.screenState.app} · ${r.screenState.page}${r.activity ? " — " + r.activity : ""}`);
|
|
487
|
+
if (meaningfulActivity(r.activity)) showCallout(r.activity);
|
|
488
|
+
} else if (r.discarded) {
|
|
489
|
+
laneIdle("vision", "no real change — skipped");
|
|
490
|
+
}
|
|
491
|
+
if (r.session) renderSession(r.session);
|
|
492
|
+
if (r.workflow) renderWorkflow(r.workflow); // real-time tracker update as the user navigates
|
|
493
|
+
// a real change reaches Ava — SHE decides whether it's a moment to speak
|
|
494
|
+
if (S.proactive && worthIt) scheduleNudge(600);
|
|
495
|
+
}
|
|
496
|
+
let calloutTimer = null;
|
|
497
|
+
function showCallout(text) {
|
|
498
|
+
$("callouttext").textContent = text;
|
|
499
|
+
$("callout").hidden = false;
|
|
500
|
+
clearTimeout(calloutTimer);
|
|
501
|
+
calloutTimer = setTimeout(() => { $("callout").hidden = true; }, 7000);
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
/* proactive nudges — debounced, only when Ava is idle */
|
|
505
|
+
function scheduleNudge(delay) { clearTimeout(S.nudgeTimer); S.nudgeTimer = setTimeout(fireNudge, delay); }
|
|
506
|
+
async function fireNudge() {
|
|
507
|
+
if (!S.sharing || S.turnInFlight || S.specialistPresent) return; // a specialist owns the call → Ava stays quiet
|
|
508
|
+
// Don't drop a moment just because she's mid-sentence — wait, then react to
|
|
509
|
+
// the LATEST screen state (observes keep it current while she talks). This is
|
|
510
|
+
// what let a $10k typo and a modal-close slip by silently before.
|
|
511
|
+
if (S.speaking || S.nudgeBusy) { scheduleNudge(700); return; }
|
|
512
|
+
S.nudgeBusy = true;
|
|
513
|
+
try {
|
|
514
|
+
const r = await fetch("/api/nudge", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sessionId: S.sessionId }) }).then((x) => x.json());
|
|
515
|
+
if (r && r.goalGuessedNow) { addSysLine("planner", "inferred intention: " + r.goal); renderGoal(r.goal); }
|
|
516
|
+
if (r && r.workflow) renderWorkflow(r.workflow);
|
|
517
|
+
if (r && r.text && !S.speaking && !S.turnInFlight) { addAgentProactive(r.text); if (r.session) renderSession(r.session); }
|
|
518
|
+
} catch {}
|
|
519
|
+
S.nudgeBusy = false;
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
/* ═══════════ fallback observe loop (no worker) ═══════════ */
|
|
523
|
+
let lastObsTime = 0;
|
|
524
|
+
const HAS_RVFC = "requestVideoFrameCallback" in HTMLVideoElement.prototype;
|
|
525
|
+
function startObserve() {
|
|
526
|
+
stopObserve(); S.lastObs = null;
|
|
527
|
+
const kick = () => { if (S.sharing && activeVid().videoWidth) observeNow(true); else if (S.sharing) setTimeout(kick, 200); };
|
|
528
|
+
setTimeout(kick, 250);
|
|
529
|
+
scheduleFrame();
|
|
530
|
+
S.obsTimer = setInterval(onFrame, 700);
|
|
531
|
+
}
|
|
532
|
+
function scheduleFrame() { const v = activeVid(); if (S.sharing && HAS_RVFC && v.requestVideoFrameCallback) v.requestVideoFrameCallback(onFrame); }
|
|
533
|
+
function stopObserve() { if (S.obsTimer) clearInterval(S.obsTimer); S.obsTimer = null; }
|
|
534
|
+
function onFrame() {
|
|
535
|
+
if (!S.sharing || perceptWorker) return;
|
|
536
|
+
const now = performance.now(), hash = frameHash();
|
|
537
|
+
if (hash && hash !== S.lastObs && !S.observing && !S.turnInFlight && now - lastObsTime > 600) {
|
|
538
|
+
laneRun("vision", "screen changed — reading", true);
|
|
539
|
+
S.lastObs = hash; lastObsTime = now; observeNow(false);
|
|
540
|
+
}
|
|
541
|
+
scheduleFrame();
|
|
542
|
+
}
|
|
543
|
+
async function observeNow(announce) {
|
|
544
|
+
if (S.observing || S.turnInFlight) return; S.observing = true;
|
|
545
|
+
try {
|
|
546
|
+
const r = await fetch("/api/observe", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sessionId: S.sessionId, frame: captureFrame(), frameHash: frameHash() }) }).then((x) => x.json());
|
|
547
|
+
handleObserve(r, announce);
|
|
548
|
+
} catch {}
|
|
549
|
+
S.observing = false;
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
/* capture helpers */
|
|
553
|
+
function activeVid() {
|
|
554
|
+
if (pip && pip.video && pip.video.videoWidth && pip.video.readyState >= 2) return pip.video;
|
|
555
|
+
return video;
|
|
556
|
+
}
|
|
557
|
+
function captureFrame() {
|
|
558
|
+
const v = activeVid(); if (!S.sharing || !v.videoWidth) return null;
|
|
559
|
+
// 1600px / q0.8: small, low-contrast text (per-plan credits, tiny labels) is
|
|
560
|
+
// legible to the vision model at this fidelity; latency is unaffected by size.
|
|
561
|
+
const t = Math.min(1, 1600 / Math.max(v.videoWidth, v.videoHeight));
|
|
562
|
+
capcanvas.width = Math.round(v.videoWidth * t); capcanvas.height = Math.round(v.videoHeight * t);
|
|
563
|
+
capcanvas.getContext("2d").drawImage(v, 0, 0, capcanvas.width, capcanvas.height);
|
|
564
|
+
return capcanvas.toDataURL("image/jpeg", 0.8);
|
|
565
|
+
}
|
|
566
|
+
function frameHash() {
|
|
567
|
+
const v = activeVid(); if (!S.sharing || !v.videoWidth) return null;
|
|
568
|
+
const N = 33; hashcanvas.width = N; hashcanvas.height = N; // 32×32 dHash — same shape as the worker's
|
|
569
|
+
const c = hashcanvas.getContext("2d"); c.drawImage(v, 0, 0, N, N);
|
|
570
|
+
const d = c.getImageData(0, 0, N, N).data;
|
|
571
|
+
const gray = (x, y) => { const i = (y * N + x) * 4; return d[i] * 0.299 + d[i + 1] * 0.587 + d[i + 2] * 0.114; };
|
|
572
|
+
let bits = "";
|
|
573
|
+
for (let y = 0; y < 32; y++) for (let x = 0; x < 32; x++) bits += gray(x, y) > gray(x + 1, y) ? "1" : "0";
|
|
574
|
+
for (let y = 0; y < 32; y++) for (let x = 0; x < 32; x++) bits += gray(x, y) > gray(x, y + 1) ? "1" : "0";
|
|
575
|
+
let hex = ""; for (let i = 0; i < bits.length; i += 4) hex += parseInt(bits.slice(i, i + 4), 2).toString(16);
|
|
576
|
+
return hex;
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
/* ═══════════ console drawer ═══════════ */
|
|
580
|
+
function consoleToggle(force) {
|
|
581
|
+
const open = force != null ? force : $("console").hidden;
|
|
582
|
+
$("console").hidden = !open;
|
|
583
|
+
$("consolebtn").classList.toggle("on", open);
|
|
584
|
+
updateStageLayout();
|
|
585
|
+
}
|
|
586
|
+
$("consolebtn").onclick = () => consoleToggle();
|
|
587
|
+
$("conclose").onclick = () => consoleToggle(false);
|
|
588
|
+
document.addEventListener("keydown", (e) => {
|
|
589
|
+
if ((e.metaKey || e.ctrlKey) && e.key === "\\") { e.preventDefault(); consoleToggle(); }
|
|
590
|
+
if ((e.metaKey || e.ctrlKey) && e.key.toLowerCase() === "k") { e.preventDefault(); joinCall(S.joined ? S.mode : "text").then(() => $("input").focus()); }
|
|
591
|
+
});
|
|
592
|
+
|
|
593
|
+
/* crew lanes */
|
|
594
|
+
function laneRun(lane, state, on) {
|
|
595
|
+
const row = $("row-" + lane); if (!row) return;
|
|
596
|
+
row.classList.toggle("run", !!on);
|
|
597
|
+
if (state != null) $("st-" + lane).textContent = state;
|
|
598
|
+
}
|
|
599
|
+
function laneDone(lane, ms, state) {
|
|
600
|
+
const row = $("row-" + lane);
|
|
601
|
+
row.classList.remove("run");
|
|
602
|
+
if (state != null) $("st-" + lane).textContent = state;
|
|
603
|
+
if (ms != null) $("ms-" + lane).textContent = ms + "ms";
|
|
604
|
+
if (S.calls[lane]) $("ca-" + lane).textContent = S.calls[lane] + (S.calls[lane] === 1 ? " call" : " calls");
|
|
605
|
+
renderCrewCount();
|
|
606
|
+
}
|
|
607
|
+
function laneIdle(lane, state) { const row = $("row-" + lane); row.classList.remove("run"); if (state != null) $("st-" + lane).textContent = state; renderCrewCount(); }
|
|
608
|
+
function renderCrewCount() {
|
|
609
|
+
const n = document.querySelectorAll(".crewrow.run").length;
|
|
610
|
+
$("crewcount").textContent = n + " / 5 RUNNING";
|
|
611
|
+
}
|
|
612
|
+
function renderConTurn(turns) {
|
|
613
|
+
$("conturn").textContent = "turn " + (turns || 0) + " · " + (S.models.talker ? TALKER_LABEL : "—");
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
/* trace */
|
|
617
|
+
const LANE_OF_ROLE = { perception: "vision", intent: "intent", planner: "planner", talker: "voice" };
|
|
618
|
+
function renderTrace(perModel, hotMs, avgMs) {
|
|
619
|
+
// big number: the hot-path (reply) latency; delta vs session average
|
|
620
|
+
$("bigLat").textContent = (hotMs / 1000).toFixed(2);
|
|
621
|
+
$("bigLat").classList.remove("null");
|
|
622
|
+
$("tracelabel").textContent = "END-TO-END";
|
|
623
|
+
const d = $("latDelta");
|
|
624
|
+
if (avgMs > 0 && Math.abs(hotMs - avgMs) > 40) {
|
|
625
|
+
const diff = (hotMs - avgMs) / 1000;
|
|
626
|
+
d.textContent = (diff > 0 ? "+" : "−") + Math.abs(diff).toFixed(2) + "s vs avg";
|
|
627
|
+
d.className = "mono delta " + (diff > 0 ? "bad" : "good");
|
|
628
|
+
} else d.textContent = "";
|
|
629
|
+
|
|
630
|
+
// sparkline of reply latencies
|
|
631
|
+
S.latHistory.push(hotMs); if (S.latHistory.length > 24) S.latHistory.shift();
|
|
632
|
+
const max = Math.max(...S.latHistory, 1);
|
|
633
|
+
$("spark").innerHTML = S.latHistory.map((v, i) => {
|
|
634
|
+
const h = Math.max(3, Math.round((v / max) * 30));
|
|
635
|
+
const cls = i === S.latHistory.length - 1 ? "now" : v > max * 0.75 ? "hi" : "";
|
|
636
|
+
return `<span class="${cls}" style="height:${h}px"></span>`;
|
|
637
|
+
}).join("");
|
|
638
|
+
|
|
639
|
+
// lane-striped bar of the REPLY path (the planner runs async, off the hot
|
|
640
|
+
// path — its latency lives in its crew row, not in this bar)
|
|
641
|
+
const lanes = (perModel || []).filter((u) => u.latencyMs > 0 && u.role !== "planner").map((u) => ({ lane: LANE_OF_ROLE[u.role] || "voice", ms: u.latencyMs }));
|
|
642
|
+
const total = lanes.reduce((s, l) => s + l.ms, 0);
|
|
643
|
+
if (total > 0) {
|
|
644
|
+
$("lanebar").innerHTML = lanes.map((l) => {
|
|
645
|
+
const w = (l.ms / total) * 100;
|
|
646
|
+
const label = w > 17 ? `<span class="mono">${l.lane.toUpperCase()} ${l.ms}ms</span>` : "";
|
|
647
|
+
return `<div class="seg ${l.lane}" style="width:${w}%">${label}</div>`;
|
|
648
|
+
}).join("");
|
|
649
|
+
const t = total / 1000;
|
|
650
|
+
$("ax1").textContent = (t / 3).toFixed(1) + "s"; $("ax2").textContent = (2 * t / 3).toFixed(1) + "s"; $("ax3").textContent = t.toFixed(1) + "s";
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
/* compiled context */
|
|
655
|
+
function renderContext(compiled, tokens) {
|
|
656
|
+
$("ctxtok").innerHTML = `<b>${(tokens / 1000).toFixed(1)}k</b> <span class="meta">/ 32k</span>`;
|
|
657
|
+
const total = compiled.length || 1;
|
|
658
|
+
const screenM = compiled.match(/SCREEN:[\s\S]*?(?=\nRECENT ACTIVITY|\nPLAN:|$)/);
|
|
659
|
+
const talkM = compiled.match(/RECENT:[\s\S]*$/);
|
|
660
|
+
const screenLen = screenM ? screenM[0].length : 0;
|
|
661
|
+
const talkLen = talkM ? talkM[0].length : 0;
|
|
662
|
+
const sysLen = Math.max(0, total - screenLen - talkLen);
|
|
663
|
+
const scale = (tokens / 32000) * 100;
|
|
664
|
+
const meter = $("ctxmeter");
|
|
665
|
+
meter.querySelector(".sys").style.width = ((sysLen / total) * scale) + "%";
|
|
666
|
+
meter.querySelector(".screen").style.width = ((screenLen / total) * scale) + "%";
|
|
667
|
+
meter.querySelector(".talk").style.width = ((talkLen / total) * scale) + "%";
|
|
668
|
+
$("ctxtext").textContent = compiled;
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
/* session metrics */
|
|
672
|
+
function renderSession(m) {
|
|
673
|
+
if (!m) return;
|
|
674
|
+
$("sTurns").textContent = m.turns;
|
|
675
|
+
$("sTok").textContent = fmt(m.totalTokens);
|
|
676
|
+
$("sAvg").textContent = m.avgLatencyMs ? (m.avgLatencyMs / 1000).toFixed(2) + "s" : "—";
|
|
677
|
+
$("sVision").textContent = `${m.perception.analysed}/${m.perception.captured}`;
|
|
678
|
+
const sp = $("spend");
|
|
679
|
+
sp.textContent = "$" + m.cost.toFixed(4);
|
|
680
|
+
sp.classList.toggle("null", m.cost === 0);
|
|
681
|
+
const hrs = Math.max((Date.now() - S.t0) / 3600e3, 1 / 120);
|
|
682
|
+
$("spendrate").textContent = "$" + (m.cost / hrs).toFixed(3) + " / hr";
|
|
683
|
+
renderConTurn(m.turns);
|
|
684
|
+
}
|
|
685
|
+
function renderGoal(goal) { if (goal && goal !== "—") $("goal").textContent = goal; }
|
|
686
|
+
|
|
687
|
+
/* ═══════════ turn loop (SSE) ═══════════ */
|
|
688
|
+
async function submitComposer() {
|
|
689
|
+
const v = $("input").value.trim();
|
|
690
|
+
if (!v) return;
|
|
691
|
+
$("input").value = "";
|
|
692
|
+
// typing before the call starts joins text-only (voice stays off, CC on)
|
|
693
|
+
if (await joinCall(S.joined ? S.mode : "text")) sendTurn(v);
|
|
694
|
+
}
|
|
695
|
+
$("composer").addEventListener("submit", (e) => { e.preventDefault(); submitComposer(); });
|
|
696
|
+
$("input").addEventListener("keydown", (e) => { if (e.key === "Enter" && !e.shiftKey) { e.preventDefault(); submitComposer(); } });
|
|
697
|
+
|
|
698
|
+
function parseSSE(chunk) {
|
|
699
|
+
let ev = "message", data = "";
|
|
700
|
+
for (const line of chunk.split("\n")) { if (line.startsWith("event:")) ev = line.slice(6).trim(); else if (line.startsWith("data:")) data += line.slice(5).trim(); }
|
|
701
|
+
let d = {}; try { d = JSON.parse(data); } catch {}
|
|
702
|
+
return { ev, d };
|
|
703
|
+
}
|
|
704
|
+
async function streamTurn(body) {
|
|
705
|
+
setAvaState("thinking"); S.turnInFlight = true; clearTimeout(S.nudgeTimer);
|
|
706
|
+
$("cap").textContent = ""; $("cap").classList.remove("err");
|
|
707
|
+
const ctx = { streamMsg: null, full: "" };
|
|
708
|
+
const ac = new AbortController(); S.turnAbort = ac; // hanging up cancels the turn in flight
|
|
709
|
+
try {
|
|
710
|
+
const res = await fetch("/api/turn", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body), signal: ac.signal });
|
|
711
|
+
if (!res.ok) { const j = await res.json().catch(() => ({})); throw new Error(j.error || `server error ${res.status}`); }
|
|
712
|
+
const reader = res.body.getReader(); const dec = new TextDecoder(); let buf = "";
|
|
713
|
+
for (;;) { const { value, done } = await reader.read(); if (done) break; buf += dec.decode(value, { stream: true }); let i; while ((i = buf.indexOf("\n\n")) >= 0) { handleEvent(parseSSE(buf.slice(0, i)), ctx); buf = buf.slice(i + 2); } }
|
|
714
|
+
} catch (e) { if (!ac.signal.aborted) agentError(e.message); }
|
|
715
|
+
const latest = S.turnAbort === ac;
|
|
716
|
+
if (latest) S.turnAbort = null;
|
|
717
|
+
if (ac.signal.aborted || !latest) return; // hung up, or a newer turn owns the flags now
|
|
718
|
+
S.turnInFlight = false;
|
|
719
|
+
S.lastObs = frameHash();
|
|
720
|
+
processPendingChange();
|
|
721
|
+
if (S.pendingUtterance) { const u = S.pendingUtterance; S.pendingUtterance = null; sendTurn(u); } // speech that arrived mid-turn
|
|
722
|
+
}
|
|
723
|
+
// Ask the worker for the CURRENT frame straight off the track.
|
|
724
|
+
function grabFrame(timeout = 800) {
|
|
725
|
+
return new Promise((resolve) => {
|
|
726
|
+
if (!perceptWorker) return resolve(null);
|
|
727
|
+
let done = false;
|
|
728
|
+
const h = async (e) => { if (done || !e.data || e.data.type !== "grabbed") return; done = true; perceptWorker.removeEventListener("message", h); resolve(e.data.blob ? { frame: await blobToDataURL(e.data.blob), hash: e.data.hash } : null); };
|
|
729
|
+
perceptWorker.addEventListener("message", h);
|
|
730
|
+
perceptWorker.postMessage({ type: "grab" });
|
|
731
|
+
setTimeout(() => { if (!done) { done = true; if (perceptWorker) perceptWorker.removeEventListener("message", h); resolve(null); } }, timeout);
|
|
732
|
+
});
|
|
733
|
+
}
|
|
734
|
+
function voiceModeOn() { return S.mode === "voice" && $("speak").checked; }
|
|
735
|
+
async function sendTurn(text) {
|
|
736
|
+
addLine("you", text);
|
|
737
|
+
logAgent("user", text);
|
|
738
|
+
// A specialist is on the call → the user is talking to the human, not Ava.
|
|
739
|
+
if (S.specialistPresent) {
|
|
740
|
+
try { await fetch("/api/bridge", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sessionId: S.sessionId, action: "usersay", text }) }); } catch {}
|
|
741
|
+
return;
|
|
742
|
+
}
|
|
743
|
+
// If the worker flagged a change that hasn't been read yet (it fired while a
|
|
744
|
+
// turn was in flight), this turn MUST re-read the screen even when the
|
|
745
|
+
// coarse hash gate would say "no change" — the fresh grab below supersedes
|
|
746
|
+
// the queued change.
|
|
747
|
+
const changed = !!S.pendingChange;
|
|
748
|
+
let frame = null, hash = null;
|
|
749
|
+
if (S.sharing && perceptWorker) { const g = await grabFrame(); if (g) { frame = g.frame; hash = g.hash; } else if (S.lastFrame) { frame = S.lastFrame.frame; hash = S.lastFrame.hash; } }
|
|
750
|
+
if (!frame && S.sharing) { frame = captureFrame(); hash = frameHash(); }
|
|
751
|
+
if (changed && frame) S.pendingChange = null;
|
|
752
|
+
await streamTurn({ sessionId: S.sessionId, userText: text, frame, screenActive: S.sharing, frameHash: hash, voice: voiceModeOn(), changed: changed && !!frame });
|
|
753
|
+
}
|
|
754
|
+
async function greet() { await streamTurn({ sessionId: S.sessionId, greet: true, voice: voiceModeOn() }); }
|
|
755
|
+
|
|
756
|
+
function handleEvent({ ev, d }, ctx) {
|
|
757
|
+
if (ev === "perceive") {
|
|
758
|
+
if (d.ran && d.screenState) {
|
|
759
|
+
S.calls.vision += 1;
|
|
760
|
+
laneDone("vision", d.latencyMs || null, `v${d.screenState.v} · ${d.screenState.app || "?"}`);
|
|
761
|
+
if (meaningfulActivity(d.screenState.activity)) addSysLine("vision", `${d.screenState.app} · ${d.screenState.page} — ${d.screenState.activity}`, { ms: d.latencyMs, tok: d.tokens, model: d.model });
|
|
762
|
+
}
|
|
763
|
+
else if (d.discarded) laneIdle("vision", "no change — reused last read");
|
|
764
|
+
else if (d.reason) laneIdle("vision", d.reason.slice(0, 42));
|
|
765
|
+
} else if (ev === "intent") {
|
|
766
|
+
S.calls.intent += 1;
|
|
767
|
+
laneDone("intent", d.latencyMs || null, d.intent === "unavailable" ? "unavailable — regex route" : `parsed · ${d.intent}${d.frustrated ? " · frustrated" : ""}`);
|
|
768
|
+
if (d.intent && d.intent !== "unavailable") addSysLine("intent", `${d.intent}${d.frustrated ? " · frustrated" : ""}${d.request ? " — wants: " + d.request : ""}`, { ms: d.latencyMs, tok: d.tokens, model: d.model });
|
|
769
|
+
} else if (ev === "gate") {
|
|
770
|
+
S.calls.router += 1;
|
|
771
|
+
laneRun("router", null, false);
|
|
772
|
+
$("st-router").textContent = `→ ${d.mode}${d.app ? " · " + d.app : ""}`;
|
|
773
|
+
$("ms-router").textContent = "0ms";
|
|
774
|
+
$("ca-router").textContent = "deterministic";
|
|
775
|
+
addSysLine("router", `→ ${d.mode}${d.app ? " · " + d.app : ""}${d.willPlan ? " · will re-plan" : ""}`, { ms: 0 });
|
|
776
|
+
} else if (ev === "thinking") {
|
|
777
|
+
if (d.thought) {
|
|
778
|
+
const tags = (d.stuck ? " · stuck" : "") + (d.resolved ? " · resolved" : "");
|
|
779
|
+
const extra = (d.blocker ? " — blocker: " + d.blocker : "") + (d.nextTactic ? " → try: " + d.nextTactic : "");
|
|
780
|
+
addSysLine("thinking", d.thought + tags + extra, { ms: d.latencyMs, tok: d.tokens, model: d.model });
|
|
781
|
+
}
|
|
782
|
+
setStuck(d.stuck); // light up (or clear) the "rethinking" indicator on Ava's tile
|
|
783
|
+
} else if (ev === "workflow") {
|
|
784
|
+
renderWorkflow(d);
|
|
785
|
+
} else if (ev === "knowledge") {
|
|
786
|
+
if (d.count) { addSysLine("knowledge", `retrieved ${d.count} ${d.source || "product"} doc${d.count > 1 ? "s" : ""} to answer`); showKnow(d); }
|
|
787
|
+
} else if (ev === "context") {
|
|
788
|
+
renderContext(d.compiled, d.tokens);
|
|
789
|
+
} else if (ev === "talkerStart") {
|
|
790
|
+
if (S.ringing) { stopRing(); joinChime(); } // the call has connected
|
|
791
|
+
laneRun("voice", "composing…", true);
|
|
792
|
+
setAvaState("speaking");
|
|
793
|
+
$("cap").textContent = ""; ctx.streamMsg = addLine("ava", ""); ctx.full = "";
|
|
794
|
+
} else if (ev === "delta") {
|
|
795
|
+
ctx.full += d.text;
|
|
796
|
+
if (S.cc) $("cap").textContent = clean(ctx.full);
|
|
797
|
+
if (ctx.streamMsg) { ctx.streamMsg.querySelector("p").textContent = clean(ctx.full); $("transcript").scrollTop = $("transcript").scrollHeight; }
|
|
798
|
+
} else if (ev === "talkerDone") {
|
|
799
|
+
S.calls.voice += 1;
|
|
800
|
+
laneDone("voice", d.latencyMs, "answered · " + (S.voice === "browser" ? "system voice" : S.voice));
|
|
801
|
+
ctx.full = humanize(clean(d.text));
|
|
802
|
+
if (S.cc) $("cap").textContent = ctx.full;
|
|
803
|
+
if (ctx.streamMsg) ctx.streamMsg.querySelector("p").textContent = ctx.full;
|
|
804
|
+
logAgent("voice", ctx.full, { ms: d.latencyMs, tok: d.tokens, model: d.model });
|
|
805
|
+
speak(ctx.full, true).then((spoke) => {
|
|
806
|
+
// no audio for this reply (voice off, or superseded) — don't stay "speaking"
|
|
807
|
+
if (!spoke && !streamState && !ttsFetch && !(ttsAudio && !ttsAudio.paused)) setAvaState("listening");
|
|
808
|
+
});
|
|
809
|
+
if (S.degraded) { S.degraded = false; setHealth("healthy", "HEALTHY"); setPillLive(true); }
|
|
810
|
+
// reply done — unblock observation now, don't wait for the planner tail
|
|
811
|
+
S.turnInFlight = false; processPendingChange();
|
|
812
|
+
} else if (ev === "planner") {
|
|
813
|
+
if (d.goal) renderGoal(d.goal);
|
|
814
|
+
if (d.ran) {
|
|
815
|
+
S.calls.planner += 1;
|
|
816
|
+
laneDone("planner", d.latencyMs, d.understanding ? "re-planned · " + d.understanding : "re-planned");
|
|
817
|
+
addSysLine("planner", `goal: ${d.goal}${d.plan && d.plan.length ? " · plan: " + d.plan.join(" → ") : ""}`, { ms: d.latencyMs, tok: d.tokens, model: d.model });
|
|
818
|
+
}
|
|
819
|
+
else if (d.error) laneIdle("planner", "failed — keeping last plan");
|
|
820
|
+
else laneIdle("planner", d.understanding && d.understanding !== "—" ? "watching · " + d.understanding : "watching");
|
|
821
|
+
} else if (ev === "metrics") {
|
|
822
|
+
if (d.goal) renderGoal(d.goal);
|
|
823
|
+
renderSession(d.session);
|
|
824
|
+
if (d.perModel && d.hotLatencyMs) renderTrace(d.perModel, d.hotLatencyMs, d.session ? d.session.avgLatencyMs : 0);
|
|
825
|
+
} else if (ev === "error") {
|
|
826
|
+
agentError(d.error);
|
|
827
|
+
} else if (ev === "done") {
|
|
828
|
+
if (!(ttsAudio && !ttsAudio.paused && !ttsAudio.ended) && !S.speaking) setAvaState("listening");
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
/* ═══════════ Ava state / transcript / caption ═══════════ */
|
|
833
|
+
function setAvaState(state) {
|
|
834
|
+
// state: standby | listening | thinking | speaking
|
|
835
|
+
const sp = state === "speaking";
|
|
836
|
+
["avatile", "minitileAva", "avaBig"].forEach((id) => { const el = $(id); if (el) el.classList.toggle("speaking", sp); });
|
|
837
|
+
const t = $("avatile"); if (t) t.classList.toggle("thinking", state === "thinking");
|
|
838
|
+
const label = sp ? "speaking · " + (S.voice === "browser" ? "system" : S.voice)
|
|
839
|
+
: state === "thinking" ? "thinking…"
|
|
840
|
+
: state === "listening" ? "listening" : "standby";
|
|
841
|
+
["avastatus", "avastatusBig", "wfAgentState"].forEach((id) => { const el = $(id); if (el) el.textContent = label; });
|
|
842
|
+
["eqCap", "eqTile", "eqMini", "eqBig", "wfEq"].forEach((id) => { const el = $(id); if (el) el.classList.toggle("live", sp); });
|
|
843
|
+
if (sp) laneRun("voice", null, true); else laneRun("voice", null, false);
|
|
844
|
+
if (pip && pip.status) pip.status.textContent = $("avastatus").textContent;
|
|
845
|
+
if (pip && pip.orb) pip.orb.classList.toggle("speaking", state === "speaking");
|
|
846
|
+
S.speaking = state === "speaking";
|
|
847
|
+
}
|
|
848
|
+
function setYouStatus(t) { for (const id of ["youmic", "youstatusBig"]) { const el = $(id); if (el) el.textContent = t; } }
|
|
849
|
+
// Visible "rethinking" indicator — driven by the reasoning agent's stuck verdict.
|
|
850
|
+
function setStuck(stuck) {
|
|
851
|
+
S.stuck = !!stuck;
|
|
852
|
+
["avaBig", "minitileAva", "avatile"].forEach((id) => { const el = $(id); if (el) el.classList.toggle("stuck", S.stuck); });
|
|
853
|
+
const r = $("avaRethink"); if (r) r.hidden = !S.stuck;
|
|
854
|
+
const pr = pipEl("pipRethink"); if (pr) pr.hidden = !S.stuck;
|
|
855
|
+
if (S.stuck) { // immediate cue the moment we notice, before the reply's "speaking" overwrites it
|
|
856
|
+
["avastatusBig", "avastatus", "wfAgentState"].forEach((id) => { const el = $(id); if (el) el.textContent = "rethinking…"; });
|
|
857
|
+
if (pip && pip.status) pip.status.textContent = "rethinking…";
|
|
858
|
+
}
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
/* ═══════════ workflow tracker + human handoff + RAG indicator ═══════════ */
|
|
862
|
+
function renderWorkflow(wf) {
|
|
863
|
+
S.workflow = wf || null;
|
|
864
|
+
if (wf && wf.handoff) S.handoff = wf.handoff;
|
|
865
|
+
if (!wf) {
|
|
866
|
+
// A re-plan can clear the tracker mid-handoff; keep the panel (and its
|
|
867
|
+
// "End handoff" control) up until the handoff is over.
|
|
868
|
+
if (!(handoffActive() || S.specialistPresent)) { $("workflow").hidden = true; return; }
|
|
869
|
+
$("wfTitle").textContent = "With a specialist"; $("wfCount").textContent = ""; $("wfFill").style.width = "0%"; $("wfEta").textContent = ""; $("wfSteps").innerHTML = "";
|
|
870
|
+
renderHandoff(S.handoff); updateStageLayout(); return;
|
|
871
|
+
}
|
|
872
|
+
$("wfTitle").textContent = wf.title || "Getting you set up";
|
|
873
|
+
$("wfCount").textContent = `${wf.done} / ${wf.total}`;
|
|
874
|
+
$("wfFill").style.width = (wf.total ? Math.round((wf.done / wf.total) * 100) : 0) + "%";
|
|
875
|
+
$("wfEta").textContent = wf.done >= wf.total ? "All done — nicely handled." : (wf.etaMin != null ? `About ${wf.etaMin} min${wf.etaMin === 1 ? "" : "s"} left` : "");
|
|
876
|
+
$("wfSteps").innerHTML = wf.steps.map((s) => {
|
|
877
|
+
const sub = s.status === "done" ? "Done" : s.status === "active" ? ("Happening now" + (wf.note ? " · " + esc(wf.note) : "")) : "";
|
|
878
|
+
return `<div class="wf-step ${s.status}"><span class="wf-mark"></span><div class="wf-stepbody"><span class="wf-steptitle">${esc(s.title)}</span>${sub ? `<span class="wf-substatus">${sub}</span>` : ""}</div></div>`;
|
|
879
|
+
}).join("");
|
|
880
|
+
renderHandoff(S.handoff);
|
|
881
|
+
updateStageLayout();
|
|
882
|
+
}
|
|
883
|
+
function handoffActive() { return !!(S.handoff && S.handoff.state && S.handoff.state !== "none"); }
|
|
884
|
+
// The specialist gets their own revocable key, never the user's session id.
|
|
885
|
+
function specialistURL() { return S.specialistKey ? `${location.origin}/specialist?key=${encodeURIComponent(S.specialistKey)}` : ""; }
|
|
886
|
+
function renderHandoff(h) {
|
|
887
|
+
const banner = $("wfHandoffBanner"), btn = $("wfHandoff"), txt = $("wfHandoffText"), link = $("wfHandoffLink"), spin = $("wfSpin");
|
|
888
|
+
const on = (h && h.state && h.state !== "none") || S.specialistPresent;
|
|
889
|
+
banner.hidden = !on;
|
|
890
|
+
btn.textContent = on ? "End handoff" : "Talk to a person";
|
|
891
|
+
btn.classList.toggle("active", !!on);
|
|
892
|
+
if (!on) { link.hidden = true; return; }
|
|
893
|
+
if (S.specialistPresent || (h && h.state === "joined")) {
|
|
894
|
+
spin.style.display = "none";
|
|
895
|
+
txt.innerHTML = "<b>A specialist is on the call.</b> They can see your screen and progress, and Ava has stepped back.";
|
|
896
|
+
link.hidden = true;
|
|
897
|
+
} else {
|
|
898
|
+
spin.style.display = "";
|
|
899
|
+
txt.textContent = "Share this link with a specialist so they can join:";
|
|
900
|
+
link.hidden = !S.specialistKey; link.textContent = specialistURL();
|
|
901
|
+
}
|
|
902
|
+
}
|
|
903
|
+
$("wfHandoffLink").onclick = () => { try { navigator.clipboard.writeText(specialistURL()); const l = $("wfHandoffLink"); const t = l.textContent; l.textContent = "Copied to clipboard ✓"; setTimeout(() => { l.textContent = t; }, 1400); } catch {} };
|
|
904
|
+
$("wfHandoff").onclick = async () => {
|
|
905
|
+
if (!S.joined) return;
|
|
906
|
+
const on = handoffActive() || S.specialistPresent;
|
|
907
|
+
const action = on ? "cancel" : "request";
|
|
908
|
+
try {
|
|
909
|
+
const r = await fetch("/api/handoff", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sessionId: S.sessionId, action }) }).then((x) => x.json());
|
|
910
|
+
if (r.handoff) S.handoff = r.handoff;
|
|
911
|
+
S.specialistKey = action === "request" ? (r.specialistKey || null) : null;
|
|
912
|
+
if (action === "request") { addSysLine("router", "requested a human specialist"); startBridge(); renderHandoff(r.handoff || { state: "waiting" }); }
|
|
913
|
+
else { stopBridge(); stopUserRTC(); S.specialistPresent = false; renderHandoff({ state: "none" }); addSysLine("router", "handoff ended — Ava continues"); }
|
|
914
|
+
} catch {}
|
|
915
|
+
};
|
|
916
|
+
// Poll the shared bridge while a handoff is active: pick up the specialist
|
|
917
|
+
// joining, their messages, and system lines; the user's own + Ava's lines are
|
|
918
|
+
// already on screen so we skip them.
|
|
919
|
+
function startBridge() { stopBridge(); bridgeTick(); }
|
|
920
|
+
function stopBridge() { clearTimeout(S.bridgeTimer); S.bridgeTimer = null; }
|
|
921
|
+
async function bridgeTick() {
|
|
922
|
+
if (!S.joined || !S.sessionId) return;
|
|
923
|
+
try {
|
|
924
|
+
const d = await fetch(`/api/bridge?sessionId=${encodeURIComponent(S.sessionId)}&since=${S.bridgeSince || 0}`).then((x) => x.json());
|
|
925
|
+
if (d && d.seq != null) {
|
|
926
|
+
S.bridgeSince = d.seq;
|
|
927
|
+
(d.events || []).forEach((e) => {
|
|
928
|
+
if (e.role === "specialist") { addLine("specialist", e.text); if (voiceModeOn() && !S.rtc) speak(e.text, true, true); } // if live voice is up, they hear it for real
|
|
929
|
+
else if (e.role === "system") addSysLine("router", e.text);
|
|
930
|
+
});
|
|
931
|
+
if (d.context) {
|
|
932
|
+
if (d.context.handoff) S.handoff = d.context.handoff;
|
|
933
|
+
const was = S.specialistPresent;
|
|
934
|
+
S.specialistPresent = !!d.context.specialist;
|
|
935
|
+
if (S.specialistPresent && !was) { setAvaState("listening"); startUserRTC(); } // Ava steps back, open the live A/V link
|
|
936
|
+
if (!S.specialistPresent && was) { stopUserRTC(); S.specialistKey = null; }
|
|
937
|
+
renderHandoff(d.context.handoff || (S.specialistPresent ? { state: "joined" } : { state: "waiting" }));
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
} catch {}
|
|
941
|
+
if (handoffActive() || S.specialistPresent) S.bridgeTimer = setTimeout(bridgeTick, 1400);
|
|
942
|
+
}
|
|
943
|
+
// Live A/V link to the specialist: send the shared screen + the user's mic,
|
|
944
|
+
// receive (and play) the specialist's voice.
|
|
945
|
+
async function startUserRTC() {
|
|
946
|
+
if (S.rtc || !window.makeRTC || !S.sessionId) return;
|
|
947
|
+
const streams = [];
|
|
948
|
+
if (S.sharing && S.stream) streams.push(S.stream); // the real screen the user is sharing
|
|
949
|
+
// mic: reuse the live-call mic if we have it, else grab one for the handoff
|
|
950
|
+
let micStream = liveMic && liveMic.stream;
|
|
951
|
+
if (!micStream) { try { micStream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true } }); S.rtcMic = micStream; } catch {} }
|
|
952
|
+
if (micStream) streams.push(micStream);
|
|
953
|
+
try {
|
|
954
|
+
S.rtc = makeRTC({ sessionId: S.sessionId, role: "user", localStreams: streams,
|
|
955
|
+
onRemote: (stream) => { const a = $("rtcAudio"); if (a) { a.srcObject = stream; a.play().catch(() => {}); } },
|
|
956
|
+
onStatus: (s) => { if (s === "connected") addSysLine("router", "live voice + screen connected with the specialist"); } });
|
|
957
|
+
} catch {}
|
|
958
|
+
}
|
|
959
|
+
// The peer link carries the tracks it started with; when the shared window
|
|
960
|
+
// changes mid-handoff, reconnect so the specialist sees the current screen.
|
|
961
|
+
function refreshUserRTC() { if (S.rtc && S.specialistPresent && S.joined) { stopUserRTC(); startUserRTC(); } }
|
|
962
|
+
function stopUserRTC() {
|
|
963
|
+
if (S.rtc) { try { S.rtc.close(); } catch {} S.rtc = null; }
|
|
964
|
+
if (S.rtcMic) { try { S.rtcMic.getTracks().forEach((t) => t.stop()); } catch {} S.rtcMic = null; }
|
|
965
|
+
const a = $("rtcAudio"); if (a) { try { a.srcObject = null; } catch {} }
|
|
966
|
+
}
|
|
967
|
+
function showKnow(d) {
|
|
968
|
+
const el = $("wfKnow"); if (!el) return;
|
|
969
|
+
el.textContent = `Referenced ${d.count} ${d.source || "product"} doc${d.count > 1 ? "s" : ""}`;
|
|
970
|
+
el.hidden = false;
|
|
971
|
+
clearTimeout(S.knowTimer); S.knowTimer = setTimeout(() => { el.hidden = true; }, 9000);
|
|
972
|
+
}
|
|
973
|
+
function nowTime() { const d = new Date(); return String(d.getHours()).padStart(2, "0") + ":" + String(d.getMinutes()).padStart(2, "0"); }
|
|
974
|
+
function addLine(who, text) {
|
|
975
|
+
const el = document.createElement("div");
|
|
976
|
+
const cls = who === "ava" ? "ava" : who === "specialist" ? "specialist" : "you";
|
|
977
|
+
const label = who === "ava" ? "AVA" : who === "specialist" ? "SPECIALIST" : "YOU";
|
|
978
|
+
el.className = "tline " + cls;
|
|
979
|
+
el.innerHTML = `<span class="twho">${label}<span class="ttime mono">${nowTime()}</span></span><p></p>`;
|
|
980
|
+
el.querySelector("p").textContent = text;
|
|
981
|
+
$("transcript").appendChild(el); $("transcript").scrollTop = $("transcript").scrollHeight;
|
|
982
|
+
return el;
|
|
983
|
+
}
|
|
984
|
+
// agent-lane log entries (VISION / INTENT / ROUTER / PLANNER) — evaluation trail
|
|
985
|
+
function addSysLine(lane, text, meta) {
|
|
986
|
+
const el = document.createElement("div");
|
|
987
|
+
el.className = "tline sys " + lane;
|
|
988
|
+
el.innerHTML = `<span class="twho">${lane.toUpperCase()}<span class="ttime mono">${nowTime()}</span></span><p></p>`;
|
|
989
|
+
el.querySelector("p").textContent = text;
|
|
990
|
+
$("transcript").appendChild(el); $("transcript").scrollTop = $("transcript").scrollHeight;
|
|
991
|
+
logAgent(lane, text, meta);
|
|
992
|
+
}
|
|
993
|
+
$("agentlog").onclick = () => {
|
|
994
|
+
const off = $("transcript").classList.toggle("noagents");
|
|
995
|
+
$("agentlog").textContent = off ? "agents off" : "agents on";
|
|
996
|
+
$("agentlog").classList.toggle("on", !off);
|
|
997
|
+
};
|
|
998
|
+
function addAgentProactive(text) {
|
|
999
|
+
const t = humanize(clean(text));
|
|
1000
|
+
setAvaState("speaking");
|
|
1001
|
+
clearTimeout(S.speakSafety); S.speakSafety = setTimeout(() => { if (S.speaking) setAvaState("listening"); }, Math.min(9000, 1800 + t.length * 45));
|
|
1002
|
+
addLine("ava", t);
|
|
1003
|
+
logAgent("voice", t);
|
|
1004
|
+
if (S.cc) $("cap").textContent = t;
|
|
1005
|
+
speak(t);
|
|
1006
|
+
}
|
|
1007
|
+
/* ═══════════ session report — recap + animated agent replay ═══════════ */
|
|
1008
|
+
// One append-only tape of every agent action, timestamped from call start.
|
|
1009
|
+
// The end-of-call report plays it back node-by-node so a reviewer can watch
|
|
1010
|
+
// perception → intent → router → planner → talker collaborate in real order.
|
|
1011
|
+
function logAgent(lane, text, meta) {
|
|
1012
|
+
if (!S.joined || !Array.isArray(S.record)) return;
|
|
1013
|
+
if (!REPORT_LANES[lane]) return;
|
|
1014
|
+
S.record.push({ t: Date.now() - S.t0, lane, text: String(text || "").slice(0, 180), ms: Math.round((meta && meta.ms) || 0), tok: Math.round((meta && meta.tok) || 0), model: (meta && meta.model) || "" });
|
|
1015
|
+
}
|
|
1016
|
+
const REPORT_LANES = { vision: 1, intent: 1, thinking: 1, router: 1, planner: 1, voice: 1, user: 1 };
|
|
1017
|
+
const LANE_FN = { vision: "fn-vision", intent: "fn-intent", router: "fn-router", planner: "fn-planner", voice: "fn-voice", user: "fn-user" };
|
|
1018
|
+
const LANE_NAME = { vision: "Perception", intent: "Intent", thinking: "Thinking", router: "Router", planner: "Planner", voice: "Talker", user: "You" };
|
|
1019
|
+
const PLAY_SVG = '<svg width="14" height="14" viewBox="0 0 24 24" fill="currentColor"><path d="M8 5v14l11-7z"/></svg>';
|
|
1020
|
+
const PAUSE_SVG = '<svg width="14" height="14" viewBox="0 0 24 24" fill="currentColor"><path d="M7 5h4v14H7zM13 5h4v14h-4z"/></svg>';
|
|
1021
|
+
|
|
1022
|
+
function fmtClock(ms) {
|
|
1023
|
+
const s = Math.max(0, Math.floor(ms / 1000));
|
|
1024
|
+
const h = Math.floor(s / 3600), m = Math.floor((s % 3600) / 60), sec = s % 60;
|
|
1025
|
+
const pad = (x) => String(x).padStart(2, "0");
|
|
1026
|
+
return (h ? pad(h) + ":" : "") + pad(m) + ":" + pad(sec);
|
|
1027
|
+
}
|
|
1028
|
+
function laneCounts(tape) {
|
|
1029
|
+
const c = { vision: 0, intent: 0, router: 0, planner: 0, voice: 0, user: 0 };
|
|
1030
|
+
for (const e of tape) if (c[e.lane] != null) c[e.lane]++;
|
|
1031
|
+
return c;
|
|
1032
|
+
}
|
|
1033
|
+
function setCounts(c) { for (const k of ["vision", "intent", "router", "planner", "voice"]) { const el = $("fc-" + k); if (el) el.textContent = c[k] || 0; } }
|
|
1034
|
+
|
|
1035
|
+
async function showReport(sid, durMs, rawTape) {
|
|
1036
|
+
stopReplay();
|
|
1037
|
+
const tape = rawTape.slice().sort((a, b) => a.t - b.t);
|
|
1038
|
+
S.reportTape = tape;
|
|
1039
|
+
S.reportDur = durMs;
|
|
1040
|
+
$("repDur").textContent = fmtClock(durMs);
|
|
1041
|
+
$("repHeadline").textContent = "Session complete";
|
|
1042
|
+
$("repSummary").textContent = "Preparing your recap…";
|
|
1043
|
+
$("repHighlights").innerHTML = "";
|
|
1044
|
+
const c = laneCounts(tape);
|
|
1045
|
+
setCounts(c);
|
|
1046
|
+
renderStats(durMs, tape, c);
|
|
1047
|
+
renderGantt(tape, durMs);
|
|
1048
|
+
$("repProgress").style.width = "0%"; $("repPlayhead").style.left = "0%";
|
|
1049
|
+
$("repNow").textContent = tape.length ? "Press play to watch the session unfold, agent by agent." : "No agent activity was recorded this session.";
|
|
1050
|
+
$("repPlay").disabled = !tape.length;
|
|
1051
|
+
$("report").hidden = false;
|
|
1052
|
+
|
|
1053
|
+
if (!sid) { $("repSummary").textContent = "Recap unavailable — no active session."; return; }
|
|
1054
|
+
try {
|
|
1055
|
+
const s = await fetch("/api/summary", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sessionId: sid }) }).then((r) => r.json())
|
|
1056
|
+
.finally(() => { // the recap was the last use — free the session slot now
|
|
1057
|
+
fetch("/api/end", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sessionId: sid }), keepalive: true }).catch(() => {});
|
|
1058
|
+
});
|
|
1059
|
+
if (s && !s.error) {
|
|
1060
|
+
$("repHeadline").textContent = s.headline || "Session complete";
|
|
1061
|
+
$("repSummary").textContent = s.summary || "—";
|
|
1062
|
+
$("repHighlights").innerHTML = (s.highlights || []).map((h) => `<li>${esc(h)}</li>`).join("");
|
|
1063
|
+
} else {
|
|
1064
|
+
$("repSummary").textContent = "Recap unavailable for this session.";
|
|
1065
|
+
}
|
|
1066
|
+
} catch {
|
|
1067
|
+
$("repSummary").textContent = "Recap unavailable — the session may have ended.";
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
function renderStats(durMs, tape, c) {
|
|
1071
|
+
const stats = [
|
|
1072
|
+
{ n: fmtClock(durMs), l: "Duration" },
|
|
1073
|
+
{ n: c.vision, l: "Screen reads" },
|
|
1074
|
+
{ n: c.voice, l: "Ava replies" },
|
|
1075
|
+
{ n: tape.length, l: "Agent events" },
|
|
1076
|
+
];
|
|
1077
|
+
$("repStats").innerHTML = stats.map((s) => `<div class="rep-stat"><b>${s.n}</b><span>${s.l}</span></div>`).join("");
|
|
1078
|
+
}
|
|
1079
|
+
// detailed, timestamped Gantt — one row per agent, bars placed by real time,
|
|
1080
|
+
// width by how long that call took. Shows where the wall-clock actually went.
|
|
1081
|
+
function renderGantt(tape, durMs) {
|
|
1082
|
+
const rows = $("ganttRows"); if (!rows) return;
|
|
1083
|
+
rows.innerHTML = ""; S.ganttBars = [];
|
|
1084
|
+
const span = Math.max(1000, durMs || 0, ...tape.map((e) => e.t + (e.ms || 0)), 1);
|
|
1085
|
+
const dotColor = (l) => l === "user" ? "var(--green)" : `var(--${l})`;
|
|
1086
|
+
["vision", "intent", "thinking", "router", "planner", "voice", "user"].forEach((lane) => {
|
|
1087
|
+
const evs = tape.map((e, i) => ({ e, i })).filter((x) => x.e.lane === lane);
|
|
1088
|
+
if (!evs.length) return;
|
|
1089
|
+
const row = document.createElement("div"); row.className = "g-row";
|
|
1090
|
+
const label = document.createElement("span"); label.className = "g-label";
|
|
1091
|
+
label.innerHTML = `<i class="gdotc" style="background:${dotColor(lane)}"></i>${LANE_NAME[lane]}`;
|
|
1092
|
+
const track = document.createElement("div"); track.className = "g-track";
|
|
1093
|
+
evs.forEach(({ e, i }) => {
|
|
1094
|
+
const bar = document.createElement("div"); bar.className = "g-bar " + lane;
|
|
1095
|
+
const left = (e.t / span) * 100;
|
|
1096
|
+
bar.style.left = left + "%";
|
|
1097
|
+
bar.style.width = Math.min(100 - left, Math.max((e.ms || 0) / span * 100, 1.4)) + "%";
|
|
1098
|
+
bar.title = `${fmtClock(e.t)} · ${e.ms ? e.ms + "ms" : "instant"}${e.tok ? " · " + e.tok + " tok" : ""}`;
|
|
1099
|
+
track.appendChild(bar); S.ganttBars[i] = bar;
|
|
1100
|
+
});
|
|
1101
|
+
row.appendChild(label); row.appendChild(track); rows.appendChild(row);
|
|
1102
|
+
});
|
|
1103
|
+
const ax = $("ganttAxis"); if (ax) { ax.innerHTML = ""; for (let k = 0; k <= 4; k++) { const s = document.createElement("span"); s.textContent = fmtClock(span * k / 4); ax.appendChild(s); } }
|
|
1104
|
+
}
|
|
1105
|
+
function ganttOnUpto(n) { (S.ganttBars || []).forEach((b, i) => { if (b) b.classList.toggle("on", i < n); }); }
|
|
1106
|
+
function stepLabel(e) {
|
|
1107
|
+
const t = e.ms ? ` · ${e.ms}ms${e.tok ? ` · ${e.tok} tok` : ""}` : "";
|
|
1108
|
+
return `${fmtClock(e.t)} · ${LANE_NAME[e.lane] || e.lane}${t} — ${e.text || "…"}`;
|
|
1109
|
+
}
|
|
1110
|
+
|
|
1111
|
+
function fireNode(lane) {
|
|
1112
|
+
const el = $(LANE_FN[lane]); if (el) { el.classList.add("fire"); setTimeout(() => el.classList.remove("fire"), 520); }
|
|
1113
|
+
const bb = $("fn-bb"); if (bb) { bb.classList.add("fire"); setTimeout(() => bb.classList.remove("fire"), 520); } // every action reads/writes the blackboard
|
|
1114
|
+
}
|
|
1115
|
+
let replay = null;
|
|
1116
|
+
function stopReplay() {
|
|
1117
|
+
if (replay && replay.timer) clearTimeout(replay.timer);
|
|
1118
|
+
replay = null;
|
|
1119
|
+
$("repPlay").innerHTML = PLAY_SVG + " Play replay";
|
|
1120
|
+
}
|
|
1121
|
+
function playReplay() {
|
|
1122
|
+
const tape = (S.reportTape || []).slice().sort((a, b) => a.t - b.t);
|
|
1123
|
+
if (!tape.length) return;
|
|
1124
|
+
if (replay && replay.playing) { // pause
|
|
1125
|
+
clearTimeout(replay.timer); replay.playing = false;
|
|
1126
|
+
$("repPlay").innerHTML = PLAY_SVG + " Resume";
|
|
1127
|
+
return;
|
|
1128
|
+
}
|
|
1129
|
+
if (replay && !replay.playing) { $("repPlay").innerHTML = PAUSE_SVG + " Pause"; replay.playing = true; stepReplay(); return; } // resume
|
|
1130
|
+
// fresh play
|
|
1131
|
+
const span = Math.max(1, tape[tape.length - 1].t);
|
|
1132
|
+
replay = { tape, i: 0, span, counts: { vision: 0, intent: 0, router: 0, planner: 0, voice: 0, user: 0 }, timer: null, playing: true };
|
|
1133
|
+
setCounts(replay.counts); ganttOnUpto(0);
|
|
1134
|
+
$("repPlay").innerHTML = PAUSE_SVG + " Pause";
|
|
1135
|
+
stepReplay();
|
|
1136
|
+
}
|
|
1137
|
+
function stepReplay() {
|
|
1138
|
+
if (!replay || !replay.playing) return;
|
|
1139
|
+
const { tape } = replay;
|
|
1140
|
+
if (replay.i >= tape.length) { finishReplay(); return; }
|
|
1141
|
+
const e = tape[replay.i];
|
|
1142
|
+
fireNode(e.lane);
|
|
1143
|
+
const gb = (S.ganttBars || [])[replay.i]; if (gb) gb.classList.add("on");
|
|
1144
|
+
if (replay.counts[e.lane] != null) { replay.counts[e.lane]++; const fc = $("fc-" + e.lane); if (fc) fc.textContent = replay.counts[e.lane]; }
|
|
1145
|
+
const pct = ((replay.i + 1) / tape.length) * 100;
|
|
1146
|
+
$("repProgress").style.width = pct + "%"; $("repPlayhead").style.left = pct + "%";
|
|
1147
|
+
$("repNow").textContent = stepLabel(e);
|
|
1148
|
+
replay.i++;
|
|
1149
|
+
// compress the real gaps into a watchable ~14s replay, but keep the rhythm
|
|
1150
|
+
let gap = 650;
|
|
1151
|
+
if (replay.i < tape.length) {
|
|
1152
|
+
const raw = tape[replay.i].t - e.t;
|
|
1153
|
+
gap = Math.min(1500, Math.max(280, raw * (14000 / replay.span)));
|
|
1154
|
+
}
|
|
1155
|
+
replay.timer = setTimeout(stepReplay, gap);
|
|
1156
|
+
}
|
|
1157
|
+
function finishReplay() {
|
|
1158
|
+
const tape = S.reportTape || [];
|
|
1159
|
+
$("repProgress").style.width = "100%"; $("repPlayhead").style.left = "100%";
|
|
1160
|
+
setCounts(laneCounts(tape)); ganttOnUpto(tape.length);
|
|
1161
|
+
$("repNow").textContent = `Replay complete — ${tape.length} agent actions over ${fmtClock(S.reportDur || 0)}.`;
|
|
1162
|
+
replay = null;
|
|
1163
|
+
$("repPlay").innerHTML = PLAY_SVG + " Replay again";
|
|
1164
|
+
}
|
|
1165
|
+
function closeReport() { stopReplay(); $("report").hidden = true; setScreen("onboard"); }
|
|
1166
|
+
$("repPlay").onclick = playReplay;
|
|
1167
|
+
$("repClose").onclick = closeReport;
|
|
1168
|
+
$("repNew").onclick = closeReport;
|
|
1169
|
+
$("repTimeline").onclick = (ev) => {
|
|
1170
|
+
const tape = (S.reportTape || []).slice().sort((a, b) => a.t - b.t);
|
|
1171
|
+
if (!tape.length) return;
|
|
1172
|
+
const rc = $("repTimeline").getBoundingClientRect();
|
|
1173
|
+
const pct = Math.min(1, Math.max(0, (ev.clientX - rc.left) / rc.width));
|
|
1174
|
+
const idx = Math.max(0, Math.min(tape.length, Math.round(pct * tape.length)));
|
|
1175
|
+
if (replay && replay.timer) clearTimeout(replay.timer);
|
|
1176
|
+
const counts = laneCounts(tape.slice(0, idx));
|
|
1177
|
+
setCounts(counts); ganttOnUpto(idx);
|
|
1178
|
+
const e = tape[Math.max(0, idx - 1)];
|
|
1179
|
+
$("repProgress").style.width = pct * 100 + "%"; $("repPlayhead").style.left = pct * 100 + "%";
|
|
1180
|
+
if (e) { fireNode(e.lane); $("repNow").textContent = stepLabel(e); }
|
|
1181
|
+
replay = { tape, i: idx, span: Math.max(1, tape[tape.length - 1].t), counts, timer: null, playing: false };
|
|
1182
|
+
$("repPlay").innerHTML = PLAY_SVG + (idx >= tape.length ? " Replay again" : " Resume");
|
|
1183
|
+
};
|
|
1184
|
+
|
|
1185
|
+
function agentError(msg) {
|
|
1186
|
+
$("cap").textContent = "⚠ " + msg; $("cap").classList.add("err");
|
|
1187
|
+
S.degraded = true;
|
|
1188
|
+
setHealth("degraded", "DEGRADED");
|
|
1189
|
+
const p = $("livepill"); p.className = "pill degraded"; $("livepilltext").textContent = "DEGRADED";
|
|
1190
|
+
setAvaState("listening");
|
|
1191
|
+
}
|
|
1192
|
+
const fmt = (n) => (n || 0).toLocaleString("en-US");
|
|
1193
|
+
// Safe for element text AND quoted attribute values.
|
|
1194
|
+
function esc(s) { return String(s == null ? "" : s).replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c])); }
|
|
1195
|
+
|
|
1196
|
+
/* text cleanup — strip markdown & robotic openers so replies read like speech */
|
|
1197
|
+
function humanize(t) {
|
|
1198
|
+
let s = String(t || "")
|
|
1199
|
+
.replace(/^\s*(it looks like|i see that|i see|looks like|i can see|i notice)\s+(?:that\s+)?you(?:'re| are)?\s*(?:currently\s+)?(?:on|in|at|viewing|looking at|working in|working on)\b[^.!?]*[.!?]\s*/i, "")
|
|
1200
|
+
.replace(/^\s*(it looks like|it seems(?: like)?|it appears(?: that)?|i can see that|i notice(?: that)?)\s+/i, "");
|
|
1201
|
+
s = s.trim();
|
|
1202
|
+
return s.length >= 8 ? s.charAt(0).toUpperCase() + s.slice(1) : t;
|
|
1203
|
+
}
|
|
1204
|
+
// Cut a reply at the first sign the model echoed its own compiled-context prompt.
|
|
1205
|
+
const LEAK_RE = /(SCREEN CHANGED|\bGOAL\s*[:(]|\bSCREEN:\s|\bvisible:\s|\bstate:\s|USER (INTENT|JUST SAID)|RECENT ACTIVITY|\bMODE:\s+[A-Z]|COMPILED CONTEXT|ONBOARDING MODE)/;
|
|
1206
|
+
function clean(t) {
|
|
1207
|
+
let s = String(t || "");
|
|
1208
|
+
const m = s.search(LEAK_RE); if (m >= 0) s = s.slice(0, m);
|
|
1209
|
+
return s
|
|
1210
|
+
.replace(/^\s*(ava|assistant|ai|agent)\s*[:>\-–]\s*/i, "") // strip leaked speaker label
|
|
1211
|
+
.replace(/\*\*(.*?)\*\*/g, "$1").replace(/(^|[^*])\*(?!\*)([^*]+)\*/g, "$1$2")
|
|
1212
|
+
.replace(/`{1,3}([^`]*)`{1,3}/g, "$1").replace(/^#{1,6}\s+/gm, "").replace(/^\s*[-•]\s+/gm, "")
|
|
1213
|
+
.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1").replace(/[_~]/g, "")
|
|
1214
|
+
.replace(/[\u{1F000}-\u{1FAFF}\u{2600}-\u{27BF}\u{2190}-\u{21FF}\u{2B00}-\u{2BFF}]/gu, "")
|
|
1215
|
+
.replace(/[ \t]{2,}/g, " ").trim();
|
|
1216
|
+
}
|
|
1217
|
+
function simText(a, b) {
|
|
1218
|
+
const w = (s) => new Set(String(s || "").toLowerCase().replace(/[^a-z0-9 ]/g, " ").split(/\s+/).filter((x) => x.length > 2));
|
|
1219
|
+
const A = w(a), B = w(b); if (!A.size || !B.size) return 0;
|
|
1220
|
+
let i = 0; for (const x of A) if (B.has(x)) i++;
|
|
1221
|
+
return i / (A.size + B.size - i);
|
|
1222
|
+
}
|
|
1223
|
+
|
|
1224
|
+
/* ═══════════ toggles ═══════════ */
|
|
1225
|
+
$("ccbtn").onclick = () => {
|
|
1226
|
+
S.cc = !S.cc;
|
|
1227
|
+
$("ccbtn").classList.toggle("on", S.cc); $("ccbtn").setAttribute("aria-pressed", S.cc);
|
|
1228
|
+
$("capwrap").hidden = !S.cc || !S.joined;
|
|
1229
|
+
};
|
|
1230
|
+
$("speakbtn").onclick = () => {
|
|
1231
|
+
const c = $("speak"); c.checked = !c.checked;
|
|
1232
|
+
$("speakbtn").classList.toggle("on", c.checked); $("speakbtn").setAttribute("aria-pressed", c.checked);
|
|
1233
|
+
if (!c.checked) {
|
|
1234
|
+
if ("speechSynthesis" in window) speechSynthesis.cancel();
|
|
1235
|
+
stopStreamPlayback();
|
|
1236
|
+
stopRoomTone();
|
|
1237
|
+
if (ttsAudio) { try { ttsAudio.pause(); } catch {} ttsAudio = null; }
|
|
1238
|
+
setAvaState("listening");
|
|
1239
|
+
} else if (S.joined && S.mode === "voice") startRoomTone();
|
|
1240
|
+
};
|
|
1241
|
+
|
|
1242
|
+
/* ═══════════ voice in (mic) ═══════════ */
|
|
1243
|
+
let rec = null, mediaRec = null, micChunks = [];
|
|
1244
|
+
const INPUT_PLACEHOLDER = "Tell Ava what you're working on…";
|
|
1245
|
+
function pipEl(id) { try { return pip && pip.win ? pip.win.document.getElementById(id) : null; } catch { return null; } }
|
|
1246
|
+
function micButtons(on) {
|
|
1247
|
+
["mic", "mic2"].forEach((id) => $(id).classList.toggle("miclive", on));
|
|
1248
|
+
const pm = pipEl("pipMic"); if (pm) pm.classList.toggle("miclive", on);
|
|
1249
|
+
setYouStatus(on ? "mic open" : "mic off");
|
|
1250
|
+
}
|
|
1251
|
+
function micToggle() {
|
|
1252
|
+
if (S.mode === "voice") { liveMic ? stopLiveMic() : startLiveMic(); return; } // voice call → toggle always-on detection
|
|
1253
|
+
if (BROWSER_STT) browserMic();
|
|
1254
|
+
else if (S.stt) (mediaRec ? stopMic() : startMic());
|
|
1255
|
+
else laneIdle("intent", "no mic STT available");
|
|
1256
|
+
}
|
|
1257
|
+
|
|
1258
|
+
/* ── live mic: always-on voice detection + barge-in (voice mode) ──
|
|
1259
|
+
An echo-cancelled stream feeds a small RMS VAD. Speech onset while Ava is
|
|
1260
|
+
talking BARGES IN — her audio stops instantly. Continuous browser speech
|
|
1261
|
+
recognition turns each utterance into a turn (VAD-gated Deepgram segments
|
|
1262
|
+
without it). Two guards keep her from hearing herself through speakers:
|
|
1263
|
+
finals that look like her own last line are dropped, and finals with no
|
|
1264
|
+
recent VAD activity are dropped. */
|
|
1265
|
+
let liveMic = null, liveMicPending = false;
|
|
1266
|
+
async function startLiveMic() {
|
|
1267
|
+
if (liveMic || liveMicPending) return true; // a double click mustn't open two mics
|
|
1268
|
+
let stream;
|
|
1269
|
+
liveMicPending = true;
|
|
1270
|
+
try {
|
|
1271
|
+
stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true, ...(S.micDevice ? { deviceId: { exact: S.micDevice } } : {}) } });
|
|
1272
|
+
} catch {
|
|
1273
|
+
$("joinstate").textContent = "mic blocked — voice detection off";
|
|
1274
|
+
laneIdle("intent", "mic blocked — type instead");
|
|
1275
|
+
return false;
|
|
1276
|
+
} finally { liveMicPending = false; }
|
|
1277
|
+
// the call may have ended while the permission prompt was up
|
|
1278
|
+
if (!S.joined || liveMic) { stream.getTracks().forEach((t) => t.stop()); return !!liveMic; }
|
|
1279
|
+
const ctx = unlockAudio(); if (!ctx) { stream.getTracks().forEach((t) => t.stop()); return false; }
|
|
1280
|
+
const an = ctx.createAnalyser(); an.fftSize = 1024;
|
|
1281
|
+
const src = ctx.createMediaStreamSource(stream);
|
|
1282
|
+
src.connect(an);
|
|
1283
|
+
liveMic = { stream, src, an, data: new Uint8Array(an.fftSize), raf: 0, speech: false, above: 0, below: 0, noise: 0.008, lastVoiceAt: 0, sttMode: "none", rec: null, recorder: null, chunks: [], ws: null, tap: null };
|
|
1284
|
+
stream.getAudioTracks()[0].addEventListener("ended", () => { // device unplugged → recover
|
|
1285
|
+
stopLiveMic();
|
|
1286
|
+
setTimeout(() => { if (S.joined && S.mode === "voice") startLiveMic(); }, 1000);
|
|
1287
|
+
});
|
|
1288
|
+
vadTick();
|
|
1289
|
+
// Transcription: prefer the offline sherpa stream on the server — it hears
|
|
1290
|
+
// the SAME echo-cancelled audio as the VAD, so it can't transcribe Ava's
|
|
1291
|
+
// own voice the way the browser's separate SpeechRecognition capture did.
|
|
1292
|
+
if (S.sttStream && await startSherpaStream()) liveMic.sttMode = "sherpa";
|
|
1293
|
+
else if (BROWSER_STT) { liveMic.sttMode = "webspeech"; startLiveRec(); }
|
|
1294
|
+
else if (S.stt) liveMic.sttMode = "segments";
|
|
1295
|
+
micButtons(true);
|
|
1296
|
+
setYouStatus("mic live · voice detection on");
|
|
1297
|
+
return true;
|
|
1298
|
+
}
|
|
1299
|
+
|
|
1300
|
+
/* stream 16kHz Int16 PCM to the server's offline recognizer over /asr */
|
|
1301
|
+
async function startSherpaStream() {
|
|
1302
|
+
if (!liveMic) return false;
|
|
1303
|
+
try {
|
|
1304
|
+
if (!audioCtx.pcmTapLoaded) { await audioCtx.audioWorklet.addModule("pcm-worklet.js"); audioCtx.pcmTapLoaded = true; }
|
|
1305
|
+
} catch { return false; }
|
|
1306
|
+
if (!liveMic) return false;
|
|
1307
|
+
return new Promise((resolve) => {
|
|
1308
|
+
const ws = new WebSocket((location.protocol === "https:" ? "wss://" : "ws://") + location.host + "/asr?engine=" + encodeURIComponent(S.sttEngine || "zipformer"));
|
|
1309
|
+
ws.binaryType = "arraybuffer";
|
|
1310
|
+
let settled = false;
|
|
1311
|
+
const settle = (ok) => { if (!settled) { settled = true; resolve(ok); } };
|
|
1312
|
+
ws.onopen = () => {
|
|
1313
|
+
if (!liveMic) { ws.close(); return settle(false); }
|
|
1314
|
+
liveMic.asrRetries = 0;
|
|
1315
|
+
if (liveMic.tap) { try { liveMic.tap.port.onmessage = null; liveMic.tap.disconnect(); liveMic.src.disconnect(liveMic.tap); } catch {} } // from a dropped socket
|
|
1316
|
+
const tap = new AudioWorkletNode(audioCtx, "pcm-tap");
|
|
1317
|
+
liveMic.src.connect(tap); tap.connect(audioCtx.destination); // silent output — keeps the node pulled
|
|
1318
|
+
const ratio = 16000 / audioCtx.sampleRate;
|
|
1319
|
+
let acc = new Int16Array(0), frac = 0;
|
|
1320
|
+
tap.port.onmessage = (e) => {
|
|
1321
|
+
if (!liveMic || ws.readyState !== 1) return;
|
|
1322
|
+
const f = e.data; // Float32 @ ctx rate → linear-resample to 16k Int16
|
|
1323
|
+
const outLen = Math.floor(f.length * ratio + frac);
|
|
1324
|
+
frac = f.length * ratio + frac - outLen;
|
|
1325
|
+
const out = new Int16Array(outLen);
|
|
1326
|
+
for (let i = 0; i < outLen; i++) {
|
|
1327
|
+
const pos = i / ratio, i0 = Math.floor(pos), i1 = Math.min(f.length - 1, i0 + 1), t = pos - i0;
|
|
1328
|
+
const v = f[i0] * (1 - t) + f[i1] * t;
|
|
1329
|
+
out[i] = Math.max(-32768, Math.min(32767, Math.round(v * 32767)));
|
|
1330
|
+
}
|
|
1331
|
+
const merged = new Int16Array(acc.length + out.length); merged.set(acc); merged.set(out, acc.length); acc = merged;
|
|
1332
|
+
if (acc.length >= 1600) { ws.send(acc.buffer.slice(0, acc.length * 2)); acc = new Int16Array(0); } // ~100ms chunks
|
|
1333
|
+
};
|
|
1334
|
+
liveMic.ws = ws; liveMic.tap = tap;
|
|
1335
|
+
settle(true);
|
|
1336
|
+
};
|
|
1337
|
+
ws.onmessage = (e) => {
|
|
1338
|
+
let m; try { m = JSON.parse(e.data); } catch { return; }
|
|
1339
|
+
if (m.type === "partial" && m.text) {
|
|
1340
|
+
if (document.activeElement !== $("input")) $("input").placeholder = "“" + prettyUtterance(m.text).slice(0, 56) + "…”";
|
|
1341
|
+
} else if (m.type === "final" && m.text) {
|
|
1342
|
+
$("input").placeholder = INPUT_PLACEHOLDER;
|
|
1343
|
+
acceptUtterance(prettyUtterance(m.text));
|
|
1344
|
+
} else if (m.type === "error") {
|
|
1345
|
+
settle(false);
|
|
1346
|
+
}
|
|
1347
|
+
};
|
|
1348
|
+
ws.onerror = () => settle(false);
|
|
1349
|
+
ws.onclose = () => {
|
|
1350
|
+
settle(false);
|
|
1351
|
+
if (liveMic && liveMic.ws === ws) { // dropped mid-call → reconnect, backing off up to 15s
|
|
1352
|
+
liveMic.ws = null;
|
|
1353
|
+
const n = liveMic.asrRetries = (liveMic.asrRetries || 0) + 1;
|
|
1354
|
+
setTimeout(() => { if (liveMic && liveMic.sttMode === "sherpa" && !liveMic.ws) startSherpaStream(); }, Math.min(15000, 1000 * 2 ** (n - 1)));
|
|
1355
|
+
}
|
|
1356
|
+
};
|
|
1357
|
+
});
|
|
1358
|
+
}
|
|
1359
|
+
// zipformer emits UPPERCASE with no punctuation — normalize to sentence case.
|
|
1360
|
+
// SenseVoice already returns cased + punctuated text, so leave it as-is.
|
|
1361
|
+
function prettyUtterance(t) {
|
|
1362
|
+
const raw = String(t || "").trim();
|
|
1363
|
+
if (S.sttEngine === "sensevoice") return raw;
|
|
1364
|
+
const s = raw.toLowerCase().replace(/\bi\b/g, "I");
|
|
1365
|
+
return s.charAt(0).toUpperCase() + s.slice(1);
|
|
1366
|
+
}
|
|
1367
|
+
function stopLiveMic() {
|
|
1368
|
+
if (!liveMic) return;
|
|
1369
|
+
cancelAnimationFrame(liveMic.raf);
|
|
1370
|
+
if (liveMic.rec) { const r = liveMic.rec; liveMic.rec = null; try { r.onend = null; r.stop(); } catch {} }
|
|
1371
|
+
if (liveMic.recorder && liveMic.recorder.state !== "inactive") { try { liveMic.recorder.onstop = null; liveMic.recorder.stop(); } catch {} }
|
|
1372
|
+
if (liveMic.ws) { const w = liveMic.ws; liveMic.ws = null; try { w.onclose = null; w.close(); } catch {} }
|
|
1373
|
+
if (liveMic.tap) { try { liveMic.tap.port.onmessage = null; liveMic.tap.disconnect(); } catch {} }
|
|
1374
|
+
liveMic.stream.getTracks().forEach((t) => t.stop());
|
|
1375
|
+
liveMic = null;
|
|
1376
|
+
$("input").placeholder = INPUT_PLACEHOLDER;
|
|
1377
|
+
micButtons(false);
|
|
1378
|
+
}
|
|
1379
|
+
function vadTick() {
|
|
1380
|
+
if (!liveMic) return;
|
|
1381
|
+
liveMic.an.getByteTimeDomainData(liveMic.data);
|
|
1382
|
+
let sum = 0;
|
|
1383
|
+
for (let i = 0; i < liveMic.data.length; i++) { const v = (liveMic.data[i] - 128) / 128; sum += v * v; }
|
|
1384
|
+
const rms = Math.sqrt(sum / liveMic.data.length);
|
|
1385
|
+
// live mic-level ring on the You tile — makes the call feel alive
|
|
1386
|
+
const yl = $("youLevel");
|
|
1387
|
+
if (yl) { const lvl = Math.min(1, rms * 8); yl.style.opacity = lvl > 0.06 ? String(0.25 + lvl * 0.75) : "0"; yl.style.transform = `scale(${1 + lvl * 0.22})`; }
|
|
1388
|
+
// adaptive threshold over the noise floor; harder to trip while Ava talks
|
|
1389
|
+
// (imperfect echo cancellation with open speakers)
|
|
1390
|
+
const thresh = Math.max(0.015, liveMic.noise * 3) * (S.speaking ? 1.8 : 1);
|
|
1391
|
+
const now = performance.now();
|
|
1392
|
+
if (rms > thresh) {
|
|
1393
|
+
if (!liveMic.above) liveMic.above = now;
|
|
1394
|
+
liveMic.below = 0;
|
|
1395
|
+
if (!liveMic.speech && now - liveMic.above > 140) onSpeechStart();
|
|
1396
|
+
if (liveMic.speech) liveMic.lastVoiceAt = Date.now();
|
|
1397
|
+
} else {
|
|
1398
|
+
liveMic.above = 0;
|
|
1399
|
+
liveMic.noise = liveMic.noise * 0.995 + rms * 0.005;
|
|
1400
|
+
if (liveMic.speech) {
|
|
1401
|
+
if (!liveMic.below) liveMic.below = now;
|
|
1402
|
+
if (now - liveMic.below > 800) onSpeechEnd();
|
|
1403
|
+
} else liveMic.below = 0;
|
|
1404
|
+
}
|
|
1405
|
+
liveMic.raf = requestAnimationFrame(vadTick);
|
|
1406
|
+
}
|
|
1407
|
+
function onSpeechStart() {
|
|
1408
|
+
liveMic.speech = true; liveMic.lastVoiceAt = Date.now();
|
|
1409
|
+
setYouStatus("speaking…");
|
|
1410
|
+
if (S.speaking) bargeIn();
|
|
1411
|
+
if (liveMic.sttMode === "segments") startSegmentRecorder();
|
|
1412
|
+
}
|
|
1413
|
+
function onSpeechEnd() {
|
|
1414
|
+
liveMic.speech = false; liveMic.below = 0;
|
|
1415
|
+
setYouStatus("mic live · voice detection on");
|
|
1416
|
+
if (liveMic.recorder && liveMic.recorder.state === "recording") liveMic.recorder.stop();
|
|
1417
|
+
}
|
|
1418
|
+
function bargeIn() {
|
|
1419
|
+
// What did they actually HEAR before cutting in? (compute before stopping)
|
|
1420
|
+
let heard = null;
|
|
1421
|
+
if (streamState && audioCtx && streamState.sentences.length) {
|
|
1422
|
+
const now = audioCtx.currentTime;
|
|
1423
|
+
// a sentence counts as heard once ~60% of it has played — starting a
|
|
1424
|
+
// sentence's first word doesn't mean they got it
|
|
1425
|
+
const all = streamState.sentences;
|
|
1426
|
+
const heardSent = all.filter((s) => now >= s.start + 0.6 * (s.end - s.start));
|
|
1427
|
+
heard = heardSent.map((s) => s.text).join(" ");
|
|
1428
|
+
if (heardSent.length === all.length && streamState.endTimer) heard = null; // everything landed — nothing to cut
|
|
1429
|
+
} else if (ttsAudio && ttsAudio.duration > 0 && S.curSpokenText) {
|
|
1430
|
+
const words = S.curSpokenText.split(/\s+/);
|
|
1431
|
+
heard = words.slice(0, Math.floor(words.length * Math.min(1, ttsAudio.currentTime / ttsAudio.duration))).join(" ");
|
|
1432
|
+
}
|
|
1433
|
+
stopStreamPlayback(true); // fade her out over ~120ms instead of chopping
|
|
1434
|
+
yieldTick(); // soft "go ahead" cue
|
|
1435
|
+
if ("speechSynthesis" in window) speechSynthesis.cancel();
|
|
1436
|
+
if (ttsAudio) { try { ttsAudio.pause(); } catch {} ttsAudio = null; }
|
|
1437
|
+
laneIdle("voice", "interrupted — listening");
|
|
1438
|
+
setAvaState("listening");
|
|
1439
|
+
if (heard === null) return;
|
|
1440
|
+
// Keep her memory honest: history holds only what was heard, and the echo
|
|
1441
|
+
// guard compares against the heard part, not sentences that never played.
|
|
1442
|
+
lastSpoken = heard;
|
|
1443
|
+
fetch("/api/truncate", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sessionId: S.sessionId, heard }) }).catch(() => {});
|
|
1444
|
+
const line = [...document.querySelectorAll(".tline.ava p")].pop();
|
|
1445
|
+
const cut = heard ? heard + " —" : "—";
|
|
1446
|
+
if (line) line.textContent = cut;
|
|
1447
|
+
if (S.cc) $("cap").textContent = cut;
|
|
1448
|
+
}
|
|
1449
|
+
function startLiveRec() {
|
|
1450
|
+
if (!liveMic || liveMic.rec) return;
|
|
1451
|
+
const R = window.SpeechRecognition || window.webkitSpeechRecognition;
|
|
1452
|
+
const r = new R(); r.lang = "en-US"; r.interimResults = true; r.continuous = true; r.maxAlternatives = 1;
|
|
1453
|
+
liveMic.rec = r;
|
|
1454
|
+
r.onresult = (e) => {
|
|
1455
|
+
let interim = "";
|
|
1456
|
+
for (let i = e.resultIndex; i < e.results.length; i++) {
|
|
1457
|
+
const res = e.results[i];
|
|
1458
|
+
if (res.isFinal) acceptUtterance(res[0].transcript.trim());
|
|
1459
|
+
else interim += res[0].transcript;
|
|
1460
|
+
}
|
|
1461
|
+
if (interim && document.activeElement !== $("input")) $("input").placeholder = "“" + interim.trim().slice(0, 56) + "…”";
|
|
1462
|
+
};
|
|
1463
|
+
r.onerror = (e) => { if (e.error === "not-allowed" || e.error === "service-not-allowed") stopLiveMic(); };
|
|
1464
|
+
r.onend = () => { // Chrome ends recognition on silence — always-on means always restart
|
|
1465
|
+
if (liveMic && liveMic.rec === r) { liveMic.rec = null; setTimeout(() => { if (liveMic) startLiveRec(); }, 250); }
|
|
1466
|
+
};
|
|
1467
|
+
try { r.start(); } catch {}
|
|
1468
|
+
}
|
|
1469
|
+
function acceptUtterance(t) {
|
|
1470
|
+
$("input").placeholder = INPUT_PLACEHOLDER;
|
|
1471
|
+
if (!liveMic) return;
|
|
1472
|
+
if (!t || t.replace(/[^a-z0-9]/gi, "").length < 2) return;
|
|
1473
|
+
// Echo guards apply only while Ava is (or just was) audible — when she's
|
|
1474
|
+
// silent there is nothing to echo, so every transcript is the user's.
|
|
1475
|
+
const avaAudible = S.speaking || (streamState && streamState.sentences.length) || (ttsAudio && !ttsAudio.paused);
|
|
1476
|
+
if (avaAudible) {
|
|
1477
|
+
if (simText(t, lastSpoken) > 0.55) return; // she heard herself
|
|
1478
|
+
if (Date.now() - liveMic.lastVoiceAt > 4000) return; // no human speech confirmed by the VAD
|
|
1479
|
+
}
|
|
1480
|
+
ackBlip(); // soft "heard you" — the gap before her reply no longer feels dead
|
|
1481
|
+
if (S.turnInFlight) { S.pendingUtterance = ((S.pendingUtterance || "") + " " + t).trim(); return; }
|
|
1482
|
+
sendTurn(t);
|
|
1483
|
+
}
|
|
1484
|
+
function startSegmentRecorder() {
|
|
1485
|
+
if (!liveMic || (liveMic.recorder && liveMic.recorder.state === "recording")) return;
|
|
1486
|
+
const mime = pickMime();
|
|
1487
|
+
const seg = new MediaRecorder(liveMic.stream, mime ? { mimeType: mime } : undefined);
|
|
1488
|
+
liveMic.chunks = [];
|
|
1489
|
+
seg.ondataavailable = (e) => { if (e.data && e.data.size) liveMic.chunks.push(e.data); };
|
|
1490
|
+
seg.onstop = async () => {
|
|
1491
|
+
const blob = new Blob(liveMic ? liveMic.chunks : [], { type: seg.mimeType || "audio/webm" });
|
|
1492
|
+
if (blob.size < 4000) return; // too short to be a word
|
|
1493
|
+
try {
|
|
1494
|
+
const r = await fetch("/api/stt", { method: "POST", headers: { "Content-Type": blob.type }, body: blob }).then((x) => x.json());
|
|
1495
|
+
if (r.text) acceptUtterance(r.text.trim());
|
|
1496
|
+
} catch {}
|
|
1497
|
+
};
|
|
1498
|
+
liveMic.recorder = seg; seg.start();
|
|
1499
|
+
}
|
|
1500
|
+
$("mic").onclick = async () => { if (await joinCall(S.joined ? S.mode : "voice")) micToggle(); };
|
|
1501
|
+
$("mic2").onclick = async () => { if (await joinCall(S.joined ? S.mode : "voice")) micToggle(); };
|
|
1502
|
+
function pickMime() { for (const m of ["audio/webm;codecs=opus", "audio/webm", "audio/mp4", "audio/ogg"]) if (window.MediaRecorder && MediaRecorder.isTypeSupported(m)) return m; return ""; }
|
|
1503
|
+
async function startMic() {
|
|
1504
|
+
let stream; try { stream = await navigator.mediaDevices.getUserMedia({ audio: S.micDevice ? { deviceId: { exact: S.micDevice } } : true }); } catch { return; }
|
|
1505
|
+
const mime = pickMime();
|
|
1506
|
+
mediaRec = new MediaRecorder(stream, mime ? { mimeType: mime } : undefined); micChunks = [];
|
|
1507
|
+
mediaRec.ondataavailable = (e) => { if (e.data && e.data.size) micChunks.push(e.data); };
|
|
1508
|
+
mediaRec.onstop = async () => {
|
|
1509
|
+
stream.getTracks().forEach((t) => t.stop());
|
|
1510
|
+
const blob = new Blob(micChunks, { type: mediaRec.mimeType || "audio/webm" });
|
|
1511
|
+
try {
|
|
1512
|
+
const r = await fetch("/api/stt", { method: "POST", headers: { "Content-Type": blob.type }, body: blob }).then((x) => x.json());
|
|
1513
|
+
if (r.text && r.text.trim()) sendTurn(r.text.trim());
|
|
1514
|
+
else if (r.error) agentError("STT: " + r.error);
|
|
1515
|
+
} catch { agentError("STT failed"); }
|
|
1516
|
+
mediaRec = null;
|
|
1517
|
+
};
|
|
1518
|
+
mediaRec.start(); micButtons(true);
|
|
1519
|
+
}
|
|
1520
|
+
function stopMic() { if (mediaRec && mediaRec.state !== "inactive") mediaRec.stop(); micButtons(false); }
|
|
1521
|
+
const BROWSER_STT = "SpeechRecognition" in window || "webkitSpeechRecognition" in window;
|
|
1522
|
+
function browserMic() {
|
|
1523
|
+
const R = window.SpeechRecognition || window.webkitSpeechRecognition;
|
|
1524
|
+
if (rec) { rec.stop(); return; }
|
|
1525
|
+
rec = new R(); rec.lang = "en-US"; rec.interimResults = true; rec.continuous = false; rec.maxAlternatives = 1;
|
|
1526
|
+
let finalT = "";
|
|
1527
|
+
rec.onresult = (e) => {
|
|
1528
|
+
let interim = "";
|
|
1529
|
+
for (let i = e.resultIndex; i < e.results.length; i++) { const r = e.results[i]; if (r.isFinal) finalT += r[0].transcript + " "; else interim += r[0].transcript; }
|
|
1530
|
+
$("input").value = (finalT + interim).trim();
|
|
1531
|
+
};
|
|
1532
|
+
rec.onerror = () => {};
|
|
1533
|
+
rec.onend = () => { micButtons(false); const t = (finalT || $("input").value || "").trim(); rec = null; if (t) { $("input").value = ""; sendTurn(t); } };
|
|
1534
|
+
rec.start(); micButtons(true);
|
|
1535
|
+
}
|
|
1536
|
+
|
|
1537
|
+
/* ═══════════ voice out (TTS) ═══════════ */
|
|
1538
|
+
let VOICE = null;
|
|
1539
|
+
function pickVoice() {
|
|
1540
|
+
if (!("speechSynthesis" in window)) return; const vs = speechSynthesis.getVoices(); if (!vs.length) return;
|
|
1541
|
+
const en = vs.filter((v) => /^en[-_]/i.test(v.lang));
|
|
1542
|
+
const rank = (v) => (/(premium|enhanced|natural|neural)/i.test(v.name) ? 3 : 0) + (/(Samantha|Ava|Zoe|Aria|Jenny|Google US English)/i.test(v.name) ? 2 : 0) + (/en[-_]US/i.test(v.lang) ? 1 : 0);
|
|
1543
|
+
VOICE = (en.length ? en : vs).slice().sort((a, b) => rank(b) - rank(a))[0];
|
|
1544
|
+
}
|
|
1545
|
+
if ("speechSynthesis" in window) { pickVoice(); speechSynthesis.onvoiceschanged = pickVoice; }
|
|
1546
|
+
let ttsAudio = null, lastSpoken = "";
|
|
1547
|
+
|
|
1548
|
+
/* Streaming playback (Kokoro): frames of 4-byte length + WAV arrive per
|
|
1549
|
+
sentence; each is decoded and scheduled back-to-back on an AudioContext, so
|
|
1550
|
+
speech starts after the FIRST sentence is synthesized instead of the last. */
|
|
1551
|
+
let audioCtx = null, streamGen = 0, streamState = null, ttsBus = null;
|
|
1552
|
+
let ttsFetch = null; // AbortController of the /api/tts request still loading (before any audio exists)
|
|
1553
|
+
function ttsOut() { // all TTS audio flows through one gain bus so barge-ins can FADE, not chop
|
|
1554
|
+
if (!ttsBus) { ttsBus = audioCtx.createGain(); ttsBus.connect(audioCtx.destination); }
|
|
1555
|
+
return ttsBus;
|
|
1556
|
+
}
|
|
1557
|
+
function stopStreamPlayback(fade) {
|
|
1558
|
+
streamGen += 1;
|
|
1559
|
+
if (ttsFetch) { try { ttsFetch.abort(); } catch {} ttsFetch = null; } // a barge-in during the fetch must cancel it too
|
|
1560
|
+
const st = streamState; streamState = null;
|
|
1561
|
+
if (!st) return;
|
|
1562
|
+
try { st.abort.abort(); } catch {}
|
|
1563
|
+
clearTimeout(st.endTimer);
|
|
1564
|
+
const kill = () => { for (const s of st.sources) { try { s.stop(); } catch {} } };
|
|
1565
|
+
if (fade && ttsBus && audioCtx) {
|
|
1566
|
+
const g = ttsBus.gain, now = audioCtx.currentTime;
|
|
1567
|
+
g.cancelScheduledValues(now); g.setValueAtTime(g.value, now); g.linearRampToValueAtTime(0.0001, now + 0.12);
|
|
1568
|
+
setTimeout(() => { kill(); const t = audioCtx.currentTime; g.cancelScheduledValues(t); g.setValueAtTime(1, t); }, 160);
|
|
1569
|
+
} else kill();
|
|
1570
|
+
}
|
|
1571
|
+
async function playWavStream(res, abort) {
|
|
1572
|
+
if (!audioCtx) audioCtx = new (window.AudioContext || window.webkitAudioContext)();
|
|
1573
|
+
const gen = ++streamGen;
|
|
1574
|
+
const st = streamState = { abort, sources: [], sentences: [], nextTime: 0, started: false, endTimer: null };
|
|
1575
|
+
if (audioCtx.state === "suspended") { try { await audioCtx.resume(); } catch {} }
|
|
1576
|
+
if (gen !== streamGen) return true; // barged in while resuming
|
|
1577
|
+
const reader = res.body.getReader();
|
|
1578
|
+
let buf = new Uint8Array(0);
|
|
1579
|
+
const schedule = (audioBuf, sentence) => {
|
|
1580
|
+
if (gen !== streamGen) return;
|
|
1581
|
+
const src = audioCtx.createBufferSource();
|
|
1582
|
+
src.buffer = audioBuf; src.connect(ttsOut());
|
|
1583
|
+
const t = Math.max(audioCtx.currentTime + 0.06, st.nextTime);
|
|
1584
|
+
src.start(t); st.nextTime = t + audioBuf.duration; st.sources.push(src);
|
|
1585
|
+
st.sentences.push({ text: sentence, start: t, end: t + audioBuf.duration });
|
|
1586
|
+
if (!st.started) { st.started = true; setAvaState("speaking"); }
|
|
1587
|
+
};
|
|
1588
|
+
const dec = new TextDecoder();
|
|
1589
|
+
for (;;) {
|
|
1590
|
+
let value, done;
|
|
1591
|
+
// an abort here is deliberate (barge-in, or a newer reply took over) —
|
|
1592
|
+
// report success so the caller never "falls back" to the browser voice
|
|
1593
|
+
try { ({ value, done } = await reader.read()); } catch { return true; }
|
|
1594
|
+
if (gen !== streamGen) return true;
|
|
1595
|
+
if (done) break;
|
|
1596
|
+
const merged = new Uint8Array(buf.length + value.length); merged.set(buf); merged.set(value, buf.length); buf = merged;
|
|
1597
|
+
// frame: [4-byte meta length][meta JSON {t: sentence, n: wav bytes}][wav]
|
|
1598
|
+
for (;;) {
|
|
1599
|
+
if (buf.length < 4) break;
|
|
1600
|
+
const metaLen = new DataView(buf.buffer, buf.byteOffset).getUint32(0);
|
|
1601
|
+
if (buf.length < 4 + metaLen) break;
|
|
1602
|
+
let meta; try { meta = JSON.parse(dec.decode(buf.slice(4, 4 + metaLen))); } catch { meta = { t: "", n: 0 }; }
|
|
1603
|
+
if (buf.length < 4 + metaLen + meta.n) break;
|
|
1604
|
+
const wav = buf.slice(4 + metaLen, 4 + metaLen + meta.n);
|
|
1605
|
+
buf = buf.slice(4 + metaLen + meta.n);
|
|
1606
|
+
try { schedule(await audioCtx.decodeAudioData(wav.buffer), meta.t || ""); } catch {}
|
|
1607
|
+
}
|
|
1608
|
+
}
|
|
1609
|
+
if (gen !== streamGen) return true;
|
|
1610
|
+
if (!st.started) return false; // nothing decoded — let the caller fall back
|
|
1611
|
+
const remaining = Math.max(0, (st.nextTime - audioCtx.currentTime) * 1000) + 120;
|
|
1612
|
+
st.endTimer = setTimeout(() => { if (gen === streamGen) { streamState = null; setAvaState("listening"); } }, remaining);
|
|
1613
|
+
return true;
|
|
1614
|
+
}
|
|
1615
|
+
|
|
1616
|
+
async function speak(text, force, ignoreMute) {
|
|
1617
|
+
if (!ignoreMute && !$("speak").checked) return false;
|
|
1618
|
+
if (!force && lastSpoken && simText(text, lastSpoken) > 0.7) { lastSpoken = text; return false; }
|
|
1619
|
+
lastSpoken = text; S.curSpokenText = text;
|
|
1620
|
+
const prov = S.voice || "browser";
|
|
1621
|
+
if (prov !== "browser") {
|
|
1622
|
+
stopStreamPlayback(); // also cancels an older reply's request still in flight
|
|
1623
|
+
if (ttsAudio) { try { ttsAudio.pause(); } catch {} ttsAudio = null; }
|
|
1624
|
+
const abort = ttsFetch = new AbortController();
|
|
1625
|
+
try {
|
|
1626
|
+
const res = await fetch("/api/tts", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ text, provider: prov }), signal: abort.signal });
|
|
1627
|
+
if (!res.ok) throw new Error("tts " + res.status);
|
|
1628
|
+
if ((res.headers.get("Content-Type") || "").includes("x-wav-stream")) {
|
|
1629
|
+
if (ttsFetch === abort) ttsFetch = null; // playWavStream owns the abort from here
|
|
1630
|
+
if (await playWavStream(res, abort)) return true;
|
|
1631
|
+
throw new Error("stream produced no audio");
|
|
1632
|
+
}
|
|
1633
|
+
const blob = await res.blob();
|
|
1634
|
+
if (ttsFetch === abort) ttsFetch = null;
|
|
1635
|
+
if (abort.signal.aborted) return false;
|
|
1636
|
+
const url = URL.createObjectURL(blob);
|
|
1637
|
+
ttsAudio = new Audio(url);
|
|
1638
|
+
ttsAudio.onplay = () => setAvaState("speaking");
|
|
1639
|
+
ttsAudio.onended = ttsAudio.onerror = () => { URL.revokeObjectURL(url); setAvaState("listening"); };
|
|
1640
|
+
await ttsAudio.play(); return true;
|
|
1641
|
+
} catch (e) {
|
|
1642
|
+
// an aborted request means a newer speak/barge-in superseded this one —
|
|
1643
|
+
// stay silent instead of re-speaking the text in the browser voice
|
|
1644
|
+
if (abort.signal.aborted || (e && e.name === "AbortError")) return false;
|
|
1645
|
+
}
|
|
1646
|
+
}
|
|
1647
|
+
if ("speechSynthesis" in window) {
|
|
1648
|
+
const u = new SpeechSynthesisUtterance(text); if (VOICE) u.voice = VOICE; u.rate = 1.03;
|
|
1649
|
+
u.onstart = () => setAvaState("speaking"); u.onend = () => setAvaState("listening");
|
|
1650
|
+
speechSynthesis.cancel(); speechSynthesis.speak(u); return true;
|
|
1651
|
+
}
|
|
1652
|
+
return false;
|
|
1653
|
+
}
|
|
1654
|
+
|
|
1655
|
+
/* ═══════════ pop-out (Document Picture-in-Picture) ═══════════ */
|
|
1656
|
+
async function openPiP() {
|
|
1657
|
+
if (pip) { try { pip.win.close(); } catch {} return; }
|
|
1658
|
+
if (!("documentPictureInPicture" in window)) return;
|
|
1659
|
+
let pw; try { pw = await documentPictureInPicture.requestWindow({ width: 360, height: 540 }); } catch { return; }
|
|
1660
|
+
document.querySelectorAll('link[rel="stylesheet"],link[rel="preconnect"],style').forEach((n) => pw.document.head.appendChild(n.cloneNode(true)));
|
|
1661
|
+
// carry the current (possibly platform-themed) brand palette into the pop-out
|
|
1662
|
+
const rootStyle = getComputedStyle(document.documentElement);
|
|
1663
|
+
["--amber", "--amber-ink", "--brand-a", "--brand-b", "--brand-c", "--brandtint", "--brandline"].forEach((v) => { const val = rootStyle.getPropertyValue(v).trim(); if (val) pw.document.documentElement.style.setProperty(v, val); });
|
|
1664
|
+
const MIC_SVG = '<svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.7" stroke-linecap="round"><rect x="9" y="3" width="6" height="11" rx="3"/><path d="M5.5 11.5a6.5 6.5 0 0 0 13 0M12 18v3"/></svg>';
|
|
1665
|
+
const SHARE_SVG = '<svg width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.7" stroke-linecap="round" stroke-linejoin="round"><rect x="2.5" y="4" width="19" height="13" rx="2"/><path d="M8 21h8M12 17v4M12 12.5V7.5M9.5 10 12 7.5l2.5 2.5"/></svg>';
|
|
1666
|
+
pw.document.body.innerHTML = `<div class="pipwrap">
|
|
1667
|
+
<video id="pipVideo" autoplay muted playsinline style="position:absolute;width:2px;height:2px;opacity:0;pointer-events:none;top:0;left:0"></video>
|
|
1668
|
+
<header class="pip-head">
|
|
1669
|
+
<svg class="wm" viewBox="0 0 66 27.8" role="img" aria-label="Ava"><path transform="translate(1.9 1.9)" d="M0 24L9.5 0L19 24M21.6 0L31.1 24L40.6 0M43.2 24L52.7 0L62.2 24" fill="none" stroke="currentColor" stroke-width="3.8" stroke-linecap="round" stroke-linejoin="round"/></svg>
|
|
1670
|
+
<span class="grow"></span>
|
|
1671
|
+
<button id="pipEnd" class="iconbtn xs" title="Close pop-out" aria-label="Close pop-out"><svg width="13" height="13" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.9" stroke-linecap="round"><path d="m7 7 10 10M17 7 7 17"/></svg></button>
|
|
1672
|
+
</header>
|
|
1673
|
+
<div class="pip-stage">
|
|
1674
|
+
<div class="pip-tile" id="pipOrb">
|
|
1675
|
+
<span class="pip-rethink" id="pipRethink" hidden><i></i>rethinking</span>
|
|
1676
|
+
<span class="pip-orb" aria-hidden="true"><i></i><i></i><span class="pip-orbcore"><svg width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="#FFFFFF" stroke-width="2.5" stroke-linecap="round"><path transform="translate(2.67 8.4) scale(.3)" d="M0 24L9.5 0L19 24M21.6 0L31.1 24L40.6 0M43.2 24L52.7 0L62.2 24" stroke="#FFFFFF" fill="none" stroke-width="3.8" stroke-linecap="round" stroke-linejoin="round"/></svg></span></span>
|
|
1677
|
+
<div class="pip-foot">
|
|
1678
|
+
<span class="pip-name">Ava</span>
|
|
1679
|
+
<span class="mono pip-status" id="pipStatus">listening</span>
|
|
1680
|
+
<span class="eq big" id="pipEq"><i></i><i></i><i></i><i></i><i></i></span>
|
|
1681
|
+
</div>
|
|
1682
|
+
</div>
|
|
1683
|
+
<div class="pip-capwrap"><p class="pip-cap" id="pipCap"></p></div>
|
|
1684
|
+
</div>
|
|
1685
|
+
<form class="pip-composer" id="pipForm">
|
|
1686
|
+
<input id="pipInput" placeholder="Message Ava…" autocomplete="off">
|
|
1687
|
+
<button type="submit" aria-label="Send"><svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 20V5M5.5 11.5 12 5l6.5 6.5"/></svg></button>
|
|
1688
|
+
</form>
|
|
1689
|
+
<div class="pip-dock">
|
|
1690
|
+
<button id="pipMic" class="ctlbtn" title="Talk" aria-label="Talk to Ava">${MIC_SVG}</button>
|
|
1691
|
+
<button id="pipShare" class="ctlbtn" title="Share screen" aria-label="Share screen">${SHARE_SVG}</button>
|
|
1692
|
+
</div></div>`;
|
|
1693
|
+
pip = { win: pw, cap: pw.document.getElementById("pipCap"), status: pw.document.getElementById("pipStatus"), orb: pw.document.getElementById("pipOrb"), video: pw.document.getElementById("pipVideo") };
|
|
1694
|
+
pip.cap.textContent = $("cap").textContent;
|
|
1695
|
+
if (liveMic) pw.document.getElementById("pipMic").classList.add("miclive");
|
|
1696
|
+
if (S.sharing) pw.document.getElementById("pipShare").classList.add("armed");
|
|
1697
|
+
if (S.stream) { pip.video.srcObject = S.stream; pip.video.play().catch(() => {}); }
|
|
1698
|
+
pip.timer = pw.setInterval(onFrame, 450); // foreground window → frames not throttled
|
|
1699
|
+
pw.document.getElementById("pipMic").onclick = () => micToggle();
|
|
1700
|
+
pw.document.getElementById("pipShare").onclick = () => shareToggle();
|
|
1701
|
+
pw.document.getElementById("pipEnd").onclick = () => { try { pw.close(); } catch {} };
|
|
1702
|
+
pw.document.getElementById("pipForm").addEventListener("submit", (e) => { e.preventDefault(); const i = pw.document.getElementById("pipInput"); const v = i.value.trim(); if (v) { i.value = ""; sendTurn(v); } });
|
|
1703
|
+
pip.obs = new MutationObserver(() => { if (pip && pip.cap) pip.cap.textContent = $("cap").textContent; });
|
|
1704
|
+
pip.obs.observe($("cap"), { childList: true, characterData: true, subtree: true });
|
|
1705
|
+
pw.addEventListener("pagehide", () => { if (pip) { if (pip.obs) pip.obs.disconnect(); if (pip.timer) pip.win.clearInterval(pip.timer); } pip = null; $("pipbtn").classList.remove("on"); });
|
|
1706
|
+
$("pipbtn").classList.add("on");
|
|
1707
|
+
}
|
|
1708
|
+
$("pipbtn").onclick = openPiP;
|