convoai 1.5.3 → 1.6.1

package/src/starters/web/frontend/app.js

@@ -0,0 +1,352 @@
+// ConvoAI Starter — Frontend Logic
+// Customize: modify the UI, add features, change interaction flow.
+
+const SERVER = window.location.origin;
+
+let client = null;
+let localTrack = null;
+let currentSession = null;
+let isMuted = false;
+let analyserNode = null;
+let freqAnimFrame = null;
+let historyTimer = null;
+let lastHistoryCount = 0;
+
+// Subtitle state — track partial (ephemeral) and final messages
+let subtitleEntries = [];  // { role, text, final, id }
+let ephemeral = null;      // current partial message
+
+const $ = (id) => document.getElementById(id);
+
+// ─── Agent Status ──────────────────────────────────────────────────────────
+
+function setAgentStatus(text, state) {
+  const el = $('agent-status');
+  el.textContent = text;
+  el.className = 'agent-status' + (state ? ' ' + state : '');
+}
+
+function setAgentRing(state) {
+  $('agent-ring').className = 'agent-ring' + (state ? ' ' + state : '');
+}
+
+// ─── Subtitles ─────────────────────────────────────────────────────────────
+
+// Incremental rendering — only update what changed, no full rebuild
+let renderedCount = 0;
+let ephemeralEl = null;
+
+function renderSubtitles() {
+  const container = $('subtitles');
+
+  // Append new final entries only
+  while (renderedCount < subtitleEntries.length) {
+    const entry = subtitleEntries[renderedCount];
+    // Remove ephemeral before adding final
+    if (ephemeralEl) { ephemeralEl.remove(); ephemeralEl = null; }
+    container.appendChild(createSubLine(entry.role, entry.text, true));
+    renderedCount++;
+  }
+
+  // Update or create ephemeral element
+  if (ephemeral) {
+    if (!ephemeralEl) {
+      ephemeralEl = createSubLine(ephemeral.role, ephemeral.text, false);
+      container.appendChild(ephemeralEl);
+    } else {
+      // Just update text, don't recreate
+      ephemeralEl.querySelector('.sub-body').textContent = ephemeral.text;
+    }
+  } else if (ephemeralEl) {
+    ephemeralEl.remove();
+    ephemeralEl = null;
+  }
+
+  container.scrollTop = container.scrollHeight;
+}
+
+function createSubLine(role, text, isFinal) {
+  const div = document.createElement('div');
+  div.className = 'sub-line ' + (role === 'user' ? 'user' : 'agent') + (isFinal ? '' : ' partial');
+
+  const label = document.createElement('div');
+  label.className = 'label';
+  label.textContent = role === 'user' ? 'You' : 'AI';
+
+  const body = document.createElement('div');
+  body.className = 'sub-body';
+  body.textContent = text;
+
+  div.appendChild(label);
+  div.appendChild(body);
+  return div;
+}
+
+function handleTranscript(msg) {
+  const obj = msg.object;
+  const text = msg.text || '';
+
+  console.log('[subtitle]', JSON.stringify(msg));
+
+  if (obj === 'user.transcription') {
+    if (msg.final) {
+      subtitleEntries.push({ role: 'user', text });
+      ephemeral = null;
+    } else {
+      ephemeral = { role: 'user', text };
+    }
+  } else if (obj === 'assistant.transcription') {
+    const status = msg.turn_status;
+    // turn_status: 0=IN_PROGRESS, 1=END, 2=INTERRUPTED
+    // If turn_status is missing, use final field as fallback
+    if (status === 1 || (status === undefined && msg.final === true)) {
+      subtitleEntries.push({ role: 'assistant', text });
+      ephemeral = null;
+    } else if (status === 2) {
+      subtitleEntries.push({ role: 'assistant', text: text + ' [interrupted]' });
+      ephemeral = null;
+    } else {
+      // In progress (status === 0, undefined, or any other value)
+      ephemeral = { role: 'assistant', text };
+    }
+  } else if (obj && text) {
+    // Unknown message type — show it anyway
+    const role = obj.includes('user') ? 'user' : 'assistant';
+    subtitleEntries.push({ role, text });
+  }
+
+  renderSubtitles();
+}
+
+// ─── Buttons ───────────────────────────────────────────────────────────────
+
+function setMainBtn(text, danger, disabled) {
+  const btn = $('btn-start');
+  btn.textContent = text;
+  btn.className = 'btn-main' + (danger ? ' danger' : '');
+  btn.disabled = disabled;
+}
+
+function updateMicIcon() {
+  $('mic-icon').style.display = isMuted ? 'none' : 'block';
+  $('mic-off-icon').style.display = isMuted ? 'block' : 'none';
+  $('btn-mic').className = 'btn-circle' + (isMuted ? ' muted' : '');
+}
+
+function showInterrupt(show) {
+  $('btn-interrupt').style.visibility = show ? 'visible' : 'hidden';
+}
+
+// ─── Frequency Bars ────────────────────────────────────────────────────────
+
+function startFreqBars() {
+  $('freq-bars').classList.add('active');
+  if (!localTrack || !localTrack._mediaStreamTrack) return;
+
+  try {
+    const ctx = new AudioContext();
+    const src = ctx.createMediaStreamSource(new MediaStream([localTrack._mediaStreamTrack]));
+    analyserNode = ctx.createAnalyser();
+    analyserNode.fftSize = 64;
+    src.connect(analyserNode);
+    const data = new Uint8Array(analyserNode.frequencyBinCount);
+    const spans = $('freq-bars').querySelectorAll('span');
+
+    (function draw() {
+      analyserNode.getByteFrequencyData(data);
+      [data[1], data[3], data[6], data[9], data[13]].forEach((v, i) => {
+        spans[i].style.height = Math.max(4, (v / 255) * 28) + 'px';
+      });
+      freqAnimFrame = requestAnimationFrame(draw);
+    })();
+  } catch {
+    const spans = $('freq-bars').querySelectorAll('span');
+    (function rand() {
+      spans.forEach(s => { s.style.height = (4 + Math.random() * 18) + 'px'; });
+      freqAnimFrame = requestAnimationFrame(rand);
+    })();
+  }
+}
+
+function stopFreqBars() {
+  $('freq-bars').classList.remove('active');
+  if (freqAnimFrame) { cancelAnimationFrame(freqAnimFrame); freqAnimFrame = null; }
+  analyserNode = null;
+}
+
+// ─── Mic / Interrupt ───────────────────────────────────────────────────────
+
+function toggleMic() {
+  if (!localTrack) return;
+  isMuted = !isMuted;
+  localTrack.setMuted(isMuted);
+  updateMicIcon();
+  if (isMuted) { stopFreqBars(); setAgentStatus('Muted', ''); }
+  else { startFreqBars(); setAgentStatus('Listening', 'active'); }
+}
+
+async function interruptAgent() {
+  if (!currentSession) return;
+  try {
+    await fetch(SERVER + '/session/interrupt', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ agentId: currentSession.agentId }),
+    });
+  } catch {}
+}
+
+// ─── Session ───────────────────────────────────────────────────────────────
+
+async function toggleConversation() {
+  if (currentSession) await stopConversation();
+  else await startConversation();
+}
+
+async function startConversation() {
+  setMainBtn('Connecting...', false, true);
+  setAgentStatus('Connecting...', '');
+  subtitleEntries = [];
+  ephemeral = null;
+  renderedCount = 0;
+  if (ephemeralEl) { ephemeralEl.remove(); ephemeralEl = null; }
+  $('subtitles').innerHTML = '';
+
+  try {
+    const res = await fetch(SERVER + '/session/start', { method: 'POST' });
+    if (!res.ok) {
+      const err = await res.json().catch(() => ({}));
+      throw new Error(err.error || 'Failed to start');
+    }
+    currentSession = await res.json();
+
+    client = AgoraRTC.createClient({ mode: 'rtc', codec: 'vp8' });
+
+    // Agent audio
+    client.on('user-published', async (user, mediaType) => {
+      await client.subscribe(user, mediaType);
+      if (mediaType === 'audio') {
+        user.audioTrack.play();
+        setAgentRing('speaking');
+        setAgentStatus('Agent speaking', 'active');
+        showInterrupt(true);
+      }
+    });
+
+    client.on('user-unpublished', (user, mediaType) => {
+      if (mediaType === 'audio') {
+        setAgentRing('listening');
+        setAgentStatus('Listening', 'active');
+        showInterrupt(false);
+      }
+    });
+
+    client.on('user-left', () => {
+      setAgentRing('');
+      showInterrupt(false);
+    });
+
+    await client.join(currentSession.appId, currentSession.channel, currentSession.token, currentSession.uid);
+
+    // ─── Real-time subtitles via DataStream ────────────────────────────
+    // User transcription comes through DataStream; agent transcription
+    // falls back to history API polling (DataStream doesn't carry it).
+    client.on('stream-message', (_uid, data) => {
+      try {
+        let text = data instanceof Uint8Array
+          ? new TextDecoder().decode(data) : String(data);
+
+        // Direct JSON (v2)
+        try {
+          const msg = JSON.parse(text);
+          if (msg.object) { handleTranscript(msg); return; }
+        } catch {}
+
+        // Chunked: message_id|part_idx|part_sum|base64
+        if (text.includes('|')) {
+          const parts = text.split('|');
+          if (parts.length >= 4 && parseInt(parts[2], 10) === 1) {
+            try {
+              const msg = JSON.parse(atob(parts.slice(3).join('|')));
+              if (msg.object) handleTranscript(msg);
+            } catch {}
+          }
+        }
+      } catch {}
+    });
+
+    localTrack = await AgoraRTC.createMicrophoneAudioTrack({ AEC: true, ANS: true, AGC: true });
+    await client.publish([localTrack]);
+
+    isMuted = false;
+    updateMicIcon();
+    $('btn-mic').disabled = false;
+    setMainBtn('End', true, false);
+    setAgentStatus('Listening', 'active');
+    setAgentRing('listening');
+    startFreqBars();
+    startHistoryFallback();
+
+  } catch (err) {
+    console.error('Start failed:', err);
+    setAgentStatus(err.message, 'error');
+    setMainBtn('Start', false, false);
+    setAgentRing('');
+    currentSession = null;
+  }
+}
+
+async function stopConversation() {
+  setMainBtn('Stopping...', true, true);
+  stopFreqBars();
+  stopHistoryFallback();
+  showInterrupt(false);
+
+  try {
+    if (localTrack) { localTrack.stop(); localTrack.close(); localTrack = null; }
+    if (client) { await client.leave(); client = null; }
+    await fetch(SERVER + '/session/stop', { method: 'POST' });
+  } catch (err) { console.error('Stop:', err); }
+
+  currentSession = null;
+  isMuted = false;
+  updateMicIcon();
+  $('btn-mic').disabled = true;
+  setMainBtn('Start', false, false);
+  setAgentStatus('Ready to talk', '');
+  setAgentRing('');
+}
+
+// ─── History Polling Fallback ──────────────────────────────────────────────
+// Agent transcription doesn't come through DataStream, so poll history API.
+
+function startHistoryFallback() {
+  lastHistoryCount = 0;
+  historyTimer = setInterval(async () => {
+    if (!currentSession) return;
+    try {
+      const res = await fetch(SERVER + '/history?agentId=' + currentSession.agentId);
+      if (!res.ok) return;
+      const data = await res.json();
+      const entries = data.history || data.chat_history || data.messages || [];
+
+      for (let i = lastHistoryCount; i < entries.length; i++) {
+        const entry = entries[i];
+        const role = (entry.role === 'assistant' || entry.role === 'agent') ? 'assistant' : 'user';
+        const text = entry.content || entry.text || '';
+        if (!text) continue;
+        // Only add agent messages from history (user comes via DataStream)
+        if (role === 'assistant') {
+          const isDup = subtitleEntries.some(s => s.text === text && s.role === role);
+          if (!isDup) { subtitleEntries.push({ role, text }); renderSubtitles(); }
+        }
+      }
+      lastHistoryCount = entries.length;
+    } catch {}
+  }, 2000);
+}
+
+function stopHistoryFallback() {
+  if (historyTimer) { clearInterval(historyTimer); historyTimer = null; }
+}
+

package/src/starters/web/frontend/index.html

@@ -0,0 +1,54 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>ConvoAI Starter</title>
+  <link rel="stylesheet" href="/style.css">
+</head>
+<body>
+
+  <!-- Agent — always vertically centered -->
+  <div class="center-stage">
+    <div id="agent-ring" class="agent-ring">
+      <div class="agent-avatar">AI</div>
+    </div>
+    <div id="freq-bars" class="freq-bars">
+      <span></span><span></span><span></span><span></span><span></span>
+    </div>
+    <div id="agent-status" class="agent-status">Ready to talk</div>
+  </div>
+
+  <!-- Subtitles — bottom overlay, like real subtitles -->
+  <div id="subtitles" class="subtitles"></div>
+
+  <!-- Controls — fixed bottom -->
+  <div class="control-bar">
+    <button id="btn-mic" class="btn-circle" title="Mute" onclick="toggleMic()" disabled>
+      <svg id="mic-icon" width="22" height="22" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
+        <path d="M12 2a3 3 0 0 0-3 3v7a3 3 0 0 0 6 0V5a3 3 0 0 0-3-3Z"/>
+        <path d="M19 10v2a7 7 0 0 1-14 0v-2"/>
+        <line x1="12" y1="19" x2="12" y2="22"/>
+      </svg>
+      <svg id="mic-off-icon" width="22" height="22" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" style="display:none">
+        <line x1="2" y1="2" x2="22" y2="22"/>
+        <path d="M18.89 13.23A7.12 7.12 0 0 0 19 12v-2"/>
+        <path d="M5 10v2a7 7 0 0 0 12 5"/>
+        <path d="M15 9.34V5a3 3 0 0 0-5.68-1.33"/>
+        <path d="M9 9v3a3 3 0 0 0 5.12 2.12"/>
+        <line x1="12" y1="19" x2="12" y2="22"/>
+      </svg>
+    </button>
+    <button id="btn-start" class="btn-main" onclick="toggleConversation()">Start</button>
+    <button id="btn-interrupt" class="btn-circle" title="Interrupt" onclick="interruptAgent()" style="visibility:hidden">
+      <svg width="22" height="22" viewBox="0 0 24 24" fill="currentColor">
+        <rect x="6" y="4" width="4" height="16" rx="1"/>
+        <rect x="14" y="4" width="4" height="16" rx="1"/>
+      </svg>
+    </button>
+  </div>
+
+  <script src="/agora-sdk.js"></script>
+  <script src="/app.js"></script>
+</body>
+</html>

package/src/starters/web/frontend/style.css

@@ -0,0 +1,241 @@
+/* ConvoAI Starter — Dark Geek Theme
+ * Palette: Happy Hues #4
+ * Customize: change theme colors here */
+
+:root {
+  --bg: #16161a;
+  --surface: #242629;
+  --text: #fffffe;
+  --dim: #94a1b2;
+  --accent: #7f5af0;
+  --green: #2cb67d;
+  --gray: #72757e;
+  --red: #e74c3c;
+}
+
+* { margin: 0; padding: 0; box-sizing: border-box; }
+
+body {
+  font-family: -apple-system, BlinkMacSystemFont, 'Inter', 'Segoe UI', system-ui, sans-serif;
+  background: var(--bg);
+  color: var(--text);
+  height: 100vh;
+  width: 100vw;
+  overflow: hidden;
+  -webkit-font-smoothing: antialiased;
+  position: relative;
+}
+
+/* ── Center Stage — agent always centered ───────────────────────────────── */
+
+.center-stage {
+  position: absolute;
+  top: 50%;
+  left: 50%;
+  transform: translate(-50%, -60%);
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  gap: 20px;
+}
+
+/* Avatar ring */
+
+.agent-ring {
+  width: 18vh;
+  height: 18vh;
+  min-width: 100px;
+  min-height: 100px;
+  max-width: 180px;
+  max-height: 180px;
+  border-radius: 50%;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  border: 2px solid rgba(255, 255, 255, 0.06);
+  transition: border-color 0.5s ease, box-shadow 0.5s ease;
+}
+
+.agent-ring.listening {
+  border-color: rgba(127, 90, 240, 0.4);
+  box-shadow: 0 0 60px rgba(127, 90, 240, 0.1);
+}
+
+.agent-ring.speaking {
+  border-color: rgba(44, 182, 125, 0.4);
+  box-shadow: 0 0 60px rgba(44, 182, 125, 0.1);
+  animation: breathe 3s ease-in-out infinite;
+}
+
+.agent-avatar {
+  width: calc(100% - 10px);
+  height: calc(100% - 10px);
+  border-radius: 50%;
+  background: var(--surface);
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  font-size: 3.5vh;
+  font-weight: 700;
+  color: var(--dim);
+  letter-spacing: 2px;
+}
+
+/* Frequency bars */
+
+.freq-bars {
+  display: flex;
+  align-items: center;
+  gap: 4px;
+  height: 28px;
+  opacity: 0;
+  transition: opacity 0.3s ease;
+}
+
+.freq-bars.active { opacity: 1; }
+
+.freq-bars span {
+  width: 4px;
+  min-height: 4px;
+  height: 4px;
+  background: var(--accent);
+  border-radius: 2px;
+  transition: height 0.08s ease-out;
+}
+
+/* Status text */
+
+.agent-status {
+  font-size: 14px;
+  color: var(--dim);
+  transition: color 0.3s ease;
+}
+
+.agent-status.active { color: var(--green); }
+.agent-status.error { color: var(--red); }
+
+/* ── Subtitles — bottom overlay ─────────────────────────────────────────── */
+
+.subtitles {
+  position: fixed;
+  bottom: 100px;
+  left: 50%;
+  transform: translateX(-50%);
+  width: 90%;
+  max-width: 640px;
+  max-height: 35vh;
+  overflow-y: auto;
+  display: flex;
+  flex-direction: column;
+  gap: 6px;
+  padding: 8px 0;
+  scroll-behavior: smooth;
+}
+
+.sub-line {
+  padding: 8px 16px;
+  border-radius: 10px;
+  font-size: 14px;
+  line-height: 1.5;
+  animation: fadeUp 0.2s ease-out;
+}
+
+.sub-line.user {
+  align-self: flex-end;
+  background: rgba(127, 90, 240, 0.1);
+  color: var(--text);
+  max-width: 80%;
+}
+
+.sub-line.agent {
+  align-self: flex-start;
+  background: rgba(255, 255, 255, 0.05);
+  color: var(--text);
+  max-width: 80%;
+}
+
+.sub-line.partial {
+  opacity: 0.5;
+}
+
+.sub-line .label {
+  font-size: 10px;
+  text-transform: uppercase;
+  letter-spacing: 1px;
+  color: var(--dim);
+  margin-bottom: 2px;
+}
+
+/* ── Control Bar — fixed bottom ─────────────────────────────────────────── */
+
+.control-bar {
+  position: fixed;
+  bottom: 0;
+  left: 0;
+  right: 0;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  gap: 16px;
+  padding: 20px 24px 32px;
+  background: linear-gradient(transparent, var(--bg) 40%);
+}
+
+.btn-main {
+  height: 52px;
+  min-width: 140px;
+  padding: 0 32px;
+  border: none;
+  border-radius: 26px;
+  font-size: 15px;
+  font-weight: 600;
+  cursor: pointer;
+  transition: all 0.2s ease;
+  background: var(--accent);
+  color: var(--text);
+}
+
+.btn-main:hover { filter: brightness(1.1); }
+.btn-main:active { transform: scale(0.97); }
+.btn-main:disabled { opacity: 0.4; cursor: not-allowed; transform: none; }
+
+.btn-main.danger {
+  background: rgba(231, 76, 60, 0.12);
+  color: var(--red);
+}
+
+.btn-main.danger:hover { background: rgba(231, 76, 60, 0.2); }
+
+.btn-circle {
+  width: 52px;
+  height: 52px;
+  border: none;
+  border-radius: 50%;
+  background: var(--surface);
+  color: var(--dim);
+  cursor: pointer;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  transition: all 0.2s ease;
+}
+
+.btn-circle:hover { background: #2e3035; color: var(--text); }
+.btn-circle:disabled { opacity: 0.25; cursor: not-allowed; }
+.btn-circle.muted { background: rgba(231, 76, 60, 0.12); color: var(--red); }
+
+/* ── Scrollbar ──────────────────────────────────────────────────────────── */
+
+.subtitles::-webkit-scrollbar { width: 0; }
+
+/* ── Animations ─────────────────────────────────────────────────────────── */
+
+@keyframes fadeUp {
+  from { opacity: 0; transform: translateY(6px); }
+  to { opacity: 1; transform: translateY(0); }
+}
+
+@keyframes breathe {
+  0%, 100% { box-shadow: 0 0 60px rgba(44, 182, 125, 0.1); }
+  50% { box-shadow: 0 0 80px rgba(44, 182, 125, 0.2), 0 0 120px rgba(44, 182, 125, 0.06); }
+}

package/src/starters/web/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "my-convoai-app",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "convoai-starter": true,
+  "scripts": {
+    "dev": "tsx server/index.ts",
+    "build": "tsc -p server/tsconfig.json",
+    "start": "node dist/server/index.js"
+  },
+  "dependencies": {
+    "agora-rtc-sdk-ng": "^4.24.0",
+    "agora-token": "^2.0.5",
+    "dotenv": "^16.4.0",
+    "express": "^4.21.0"
+  },
+  "devDependencies": {
+    "@types/express": "^5.0.0",
+    "tsx": "^4.19.0",
+    "typescript": "^5.6.0"
+  }
+}

package/src/starters/web/python-server/.env.example

@@ -0,0 +1,23 @@
+# Agora Credentials (from https://console.agora.io)
+AGORA_APP_ID=
+AGORA_APP_CERTIFICATE=
+AGORA_CUSTOMER_ID=
+AGORA_CUSTOMER_SECRET=
+
+# Region: "global" or "cn"
+AGORA_REGION=global
+
+# LLM Configuration
+LLM_VENDOR=openai
+LLM_MODEL=gpt-4o-mini
+LLM_API_KEY=
+LLM_URL=
+LLM_STYLE=openai
+
+# TTS Configuration
+TTS_VENDOR=microsoft
+TTS_API_KEY=
+
+# ASR Configuration
+ASR_VENDOR=ares
+ASR_LANGUAGE=en-US