@tencent-rtc/trtc-agent-skills 0.1.1 → 0.1.4

package/skills/trtc-ai-service/capabilities/conversation-core/web-demo/app.js

@@ -0,0 +1,415 @@
+/* TRTC Conversational AI · Web Demo
+ * Health check → request room credentials (get_config) → TRTC enterRoom + start AI
+ * Subtitles come from TRTC custom messages: type=10000 (subtitle), type=10001 (AI state)
+ * User text messages sent to the AI bot: sendCustomMessage(cmdId:2, type:20000)
+ *   → AI treats it as "user speech" and responds (voice + subtitle)
+ */
+(function () {
+  'use strict';
+
+  const elIndicators = document.querySelectorAll('.indicator');
+  const btnRecheck = document.getElementById('btn-recheck');
+  const btnStart = document.getElementById('btn-start');
+  const btnMic = document.getElementById('btn-mic');
+  const btnStop = document.getElementById('btn-stop');
+  const btnSend = document.getElementById('btn-send');
+  const txt = document.getElementById('text-input');
+  const conv = document.getElementById('conversation');
+  const micLabel = document.getElementById('mic-label');
+  const micIcon = document.getElementById('mic-icon');
+  const agentStatusEl = document.getElementById('agent-status');
+
+  const state = {
+    healthy: false,
+    sessionId: null,
+    sdkAppId: 0,
+    roomId: null,
+    userId: null,
+    userSig: null,
+    agentUserId: null,
+    taskId: null,
+    trtcClient: null,
+    micEnabled: false,
+    aiRoundText: {},      // roundid -> 累积文本（可能是增量也可能累积）
+    aiRoundLast: {},      // roundid -> 上一帧
+    aiRoundBubbleId: {},  // roundid -> dom id
+    userTurnText: '',
+    userBubbleId: null,
+  };
+
+  // ====================== HTTP helpers ======================
+  async function api(path, options = {}) {
+    const resp = await fetch(path, {
+      method: options.method || 'GET',
+      headers: { 'Content-Type': 'application/json' },
+      body: options.body ? JSON.stringify(options.body) : undefined,
+    });
+    let data;
+    try { data = await resp.json(); } catch { data = {}; }
+    if (!resp.ok) {
+      const msg = (data && (data.detail?.message || data.detail || data.msg)) || resp.statusText;
+      throw new Error(typeof msg === 'string' ? msg : JSON.stringify(msg));
+    }
+    return data;
+  }
+
+  // ====================== UI helpers ======================
+  function setIndicator(key, ok, latency) {
+    elIndicators.forEach((el) => {
+      if (el.dataset.key !== key) return;
+      const led = el.querySelector('.led');
+      const lat = el.querySelector('.latency');
+      led.classList.remove('led-unknown', 'led-ok', 'led-fail', 'led-pending');
+      if (ok === 'pending') {
+        led.classList.add('led-pending');
+        lat.textContent = '...';
+      } else if (ok) {
+        led.classList.add('led-ok');
+        lat.textContent = latency != null ? `${latency}ms` : 'ok';
+      } else {
+        led.classList.add('led-fail');
+        lat.textContent = latency != null ? `${latency}ms` : 'fail';
+      }
+    });
+  }
+
+  function clearPlaceholder() {
+    const p = conv.querySelector('.placeholder');
+    if (p) p.remove();
+  }
+
+  function makeBubble(role, text) {
+    clearPlaceholder();
+    const div = document.createElement('div');
+    div.className = `bubble ${role}`;
+    div.textContent = text || '';
+    conv.appendChild(div);
+    div.scrollIntoView({ behavior: 'smooth', block: 'end' });
+    return div;
+  }
+
+  function updateBubble(el, text) {
+    if (!el) return;
+    el.textContent = text;
+    el.scrollIntoView({ behavior: 'smooth', block: 'end' });
+  }
+
+  function setAgentStatus(label, cls = '') {
+    agentStatusEl.innerHTML = `<span class="agent-state agent-state-${cls || 'idle'}">${label}</span>`;
+  }
+
+  function updateMicUI() {
+    btnMic.classList.toggle('btn-active', state.micEnabled);
+    micLabel.textContent = state.micEnabled ? 'Mic on' : 'Mic off';
+    micIcon.textContent = state.micEnabled ? '🔴' : '🎙️';
+  }
+
+  // ====================== Health ======================
+  async function runHealthCheck() {
+    ['tencent_cloud', 'trtc', 'llm'].forEach((k) => setIndicator(k, 'pending'));
+    btnStart.disabled = true;
+    try {
+      const data = await api('/api/v1/health');
+      const checks = data.checks || {};
+      let allOk = true;
+      for (const k of ['tencent_cloud', 'trtc', 'llm']) {
+        const c = checks[k] || {};
+        const ok = c.status === 'ok';
+        if (!ok) allOk = false;
+        setIndicator(k, ok, c.latency_ms);
+        if (!ok && c.detail) console.warn(`[health] ${k}:`, c.error_code, c.detail);
+      }
+      state.healthy = allOk;
+      btnStart.disabled = !allOk;
+      if (!data.configured) {
+        clearPlaceholder();
+        makeBubble(
+          'agent',
+          `Credentials missing: ${(data.missing || []).join(', ')}\nRun: python scripts/setup-credentials.py`
+        );
+      }
+    } catch (err) {
+      ['tencent_cloud', 'trtc', 'llm'].forEach((k) => setIndicator(k, false));
+      console.error('[health] error', err);
+    }
+  }
+
+  // ====================== TRTC custom message handling ======================
+  // type:10000 字幕（user / ai），type:10001 AI 状态
+  function handleCustomMessage(data, eventUserId) {
+    if (!data || typeof data !== 'object') return;
+
+    if (data.type === 10001) {
+      const stateCode = data.payload && data.payload.state;
+      const map = {
+        1: ['listening', 'listening'],
+        2: ['thinking',  'thinking'],
+        3: ['speaking',  'speaking'],
+        4: ['interrupted', 'idle'],
+      };
+      const m = map[stateCode] || ['idle', 'idle'];
+      setAgentStatus(m[0], m[1]);
+      return;
+    }
+
+    if (data.type !== 10000) return;
+    const text = (data.payload && data.payload.text) || '';
+    const sender = data.sender || eventUserId || '';
+    const end = data.payload && data.payload.end === true;
+    const roundid = (data.payload && data.payload.roundid) || '';
+    const isUser = sender === state.userId;
+
+    if (isUser) {
+      // 用户语音 ASR：仅 end=true 才落入气泡，避免半句乱码
+      if (end && text.trim()) {
+        if (!state.userBubbleId) {
+          state.userBubbleId = makeBubble('user', text.trim());
+        } else {
+          const cur = state.userBubbleId.textContent || '';
+          updateBubble(state.userBubbleId, cur ? `${cur} ${text.trim()}` : text.trim());
+        }
+        // 一次 user finalize 后，下一次新 ASR 进入新气泡
+        setTimeout(() => { state.userBubbleId = null; }, 1500);
+      }
+      return;
+    }
+
+    // AI 字幕：增量/累积自适应，按 roundid 聚合到同一气泡
+    if (text.trim() && roundid) {
+      const last = state.aiRoundLast[roundid] || '';
+      const cur  = state.aiRoundText[roundid] || '';
+      const isAccumulative = last && text.startsWith(last);
+      state.aiRoundText[roundid] = isAccumulative ? text : (cur + text);
+      state.aiRoundLast[roundid] = text;
+
+      let bubble = state.aiRoundBubbleId[roundid];
+      if (!bubble) {
+        bubble = makeBubble('agent', state.aiRoundText[roundid]);
+        state.aiRoundBubbleId[roundid] = bubble;
+      } else {
+        updateBubble(bubble, state.aiRoundText[roundid]);
+      }
+    }
+
+    if (end && roundid) {
+      const bubble = state.aiRoundBubbleId[roundid];
+      const finalText = state.aiRoundText[roundid] || text;
+      if (bubble) updateBubble(bubble, finalText);
+      delete state.aiRoundText[roundid];
+      delete state.aiRoundLast[roundid];
+      delete state.aiRoundBubbleId[roundid];
+    }
+  }
+
+  // ====================== Start / Stop ======================
+  async function startConversation() {
+    btnStart.disabled = true;
+    setAgentStatus('connecting...', 'thinking');
+    try {
+      // 1) get_config
+      const cfg = await api('/api/v1/get_config', { method: 'POST', body: {} });
+      const data = cfg.data || {};
+      Object.assign(state, {
+        sessionId: data.session_id,
+        sdkAppId: data.sdk_app_id,
+        roomId: parseInt(data.room_id, 10),  // TRTC enterRoom 要求 roomId 是 number（数字房间号）
+        userId: data.user_id,
+        userSig: data.user_sig,
+        agentUserId: data.agent_user_id,
+      });
+
+      // 2) TRTC create + enterRoom
+      if (typeof TRTC === 'undefined') throw new Error('TRTC Web SDK not loaded');
+      state.trtcClient = TRTC.create();
+
+      state.trtcClient.on(TRTC.EVENT.CUSTOM_MESSAGE, (event) => {
+        try {
+          const txt = new TextDecoder().decode(event.data);
+          const parsed = JSON.parse(txt);
+          handleCustomMessage(parsed, event.userId);
+        } catch (e) { console.warn('parse custom msg failed', e); }
+      });
+
+      state.trtcClient.on(TRTC.EVENT.ERROR, (err) => {
+        console.error('[trtc] error', err);
+      });
+
+      await state.trtcClient.enterRoom({
+        roomId: state.roomId,
+        scene: 'rtc',
+        sdkAppId: state.sdkAppId,
+        userId: state.userId,
+        userSig: state.userSig,
+      });
+
+      // 3) start local audio (默认静音；用户点 Mic 后才开)
+      try {
+        await state.trtcClient.startLocalAudio();
+        await state.trtcClient.updateLocalAudio({ mute: true });
+        state.micEnabled = false;
+      } catch (e) {
+        console.warn('mic init failed (continue text-only):', e);
+      }
+
+      // 4) StartAIConversation
+      await api('/api/v1/agent/start', {
+        method: 'POST',
+        body: { session_id: state.sessionId, language: 'en' },
+      });
+
+      btnStop.disabled = false;
+      btnSend.disabled = false;
+      btnMic.disabled = false;
+      txt.disabled = false;
+      txt.focus();
+      updateMicUI();
+      setAgentStatus('ready', 'idle');
+    } catch (err) {
+      console.error('[start] error', err);
+      makeBubble('agent', `Start failed: ${err.message || err}`);
+      setAgentStatus('error', 'idle');
+      await safeExitRoom();
+      btnStart.disabled = !state.healthy;
+    }
+  }
+
+  async function stopConversation() {
+    btnStop.disabled = true;
+    btnMic.disabled = true;
+    btnSend.disabled = true;
+    txt.disabled = true;
+    txt.value = '';
+    try {
+      if (state.sessionId) {
+        await api('/api/v1/agent/stop', {
+          method: 'POST',
+          body: { session_id: state.sessionId },
+        });
+      }
+    } catch (e) {
+      console.warn('[stop] api error', e);
+    }
+    await safeExitRoom();
+    state.sessionId = null;
+    state.taskId = null;
+    state.micEnabled = false;
+    updateMicUI();
+    setAgentStatus('idle', 'idle');
+    btnStart.disabled = !state.healthy;
+  }
+
+  async function safeExitRoom() {
+    try {
+      if (state.trtcClient) {
+        await state.trtcClient.exitRoom();
+        state.trtcClient.destroy();
+      }
+    } catch (e) {
+      console.warn('exitRoom failed', e);
+    } finally {
+      state.trtcClient = null;
+    }
+  }
+
+  // ====================== Mic toggle ======================
+  async function toggleMic() {
+    if (!state.trtcClient) return;
+    state.micEnabled = !state.micEnabled;
+    try {
+      await state.trtcClient.updateLocalAudio({ mute: !state.micEnabled });
+      // 用户刚开 mic 即将说话 → 立刻打断 AI 当前 TTS（智能打断）
+      if (state.micEnabled) sendInterrupt();
+    } catch (e) {
+      console.error('toggleMic failed', e);
+      state.micEnabled = !state.micEnabled; // revert
+    }
+    updateMicUI();
+  }
+
+  // ====================== Text injection (端侧 → AI bot) ======================
+  // 协议参考：https://cloud.tencent.com/document/product/647/115412
+  //   - type:20000 = 把文字当作"用户说的话"喂给 AI bot，触发一轮 LLM → TTS + 字幕
+  //     payload 字段名必须是 `message`（不是 text）
+  //   - type:20001 = 立即打断 AI 当前 TTS（用户开始新一轮输入时触发智能打断）
+  // 通过 TRTC sendCustomMessage(cmdId:2) 端到端发送。
+
+  function uuid() {
+    return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
+      const r = (Math.random() * 16) | 0;
+      const v = c === 'x' ? r : (r & 0x3) | 0x8;
+      return v.toString(16);
+    });
+  }
+
+  async function sendCustomToAgent(message) {
+    if (!state.trtcClient) return false;
+    try {
+      await state.trtcClient.sendCustomMessage({
+        cmdId: 2,
+        data: new TextEncoder().encode(JSON.stringify(message)).buffer,
+      });
+      return true;
+    } catch (e) {
+      console.warn('sendCustomMessage failed', e);
+      return false;
+    }
+  }
+
+  function sendInterrupt() {
+    return sendCustomToAgent({
+      type: 20001,
+      sender: state.userId,
+      receiver: [state.agentUserId],
+      payload: { id: uuid(), timestamp: Date.now() },
+    });
+  }
+
+  async function sendText() {
+    const text = (txt.value || '').trim();
+    if (!text || !state.trtcClient) return;
+    txt.value = '';
+    // 立即渲染用户气泡
+    makeBubble('user', text);
+
+    // 1) 先打断 AI 当前 TTS（智能打断）
+    sendInterrupt();
+
+    // 2) 略延后一点点，确保 interrupt 生效，再推用户回合给 AI
+    //    AI bot 收到 type:20000 后会跳过 ASR、直接走 LLM → TTS + 字幕回放
+    setTimeout(() => {
+      sendCustomToAgent({
+        type: 20000,
+        sender: state.userId,
+        receiver: [state.agentUserId],
+        payload: {
+          id: uuid(),
+          message: text,           // ← 协议规范字段名（不是 text）
+          timestamp: Date.now(),
+        },
+      });
+    }, 120);
+  }
+
+  // ====================== Bindings ======================
+  btnRecheck.addEventListener('click', runHealthCheck);
+  btnStart.addEventListener('click', startConversation);
+  btnStop.addEventListener('click', stopConversation);
+  btnMic.addEventListener('click', toggleMic);
+  btnSend.addEventListener('click', sendText);
+
+  // IME（中文输入法）兼容：拼音上屏时按空格/回车不应触发发送
+  //   - compositionstart / compositionend 跟踪输入法是否在编辑中
+  //   - event.isComposing 是 W3C 标准属性
+  //   - event.keyCode === 229 是 IME 状态下 Enter 的兜底信号（旧浏览器）
+  let imeComposing = false;
+  txt.addEventListener('compositionstart', () => { imeComposing = true; });
+  txt.addEventListener('compositionend',   () => { imeComposing = false; });
+  txt.addEventListener('keydown', (e) => {
+    if (e.key !== 'Enter') return;
+    if (imeComposing || e.isComposing || e.keyCode === 229) return;
+    e.preventDefault();
+    sendText();
+  });
+
+  runHealthCheck();
+})();

package/skills/trtc-ai-service/capabilities/conversation-core/web-demo/index.html

@@ -0,0 +1,68 @@
+<!DOCTYPE html>
+<html lang="zh-CN">
+<head>
+  <meta charset="UTF-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>TRTC Conversational AI</title>
+  <link rel="stylesheet" href="/static/styles.css" />
+</head>
+<body>
+  <header class="status-bar" id="status-bar">
+    <div class="brand">
+      <span class="dot"></span>
+      <strong>TRTC Conversational AI</strong>
+    </div>
+    <div class="indicators" role="status" aria-live="polite">
+      <div class="indicator" data-key="tencent_cloud">
+        <span class="led led-unknown"></span>
+        <span class="label">Tencent Cloud</span>
+        <span class="latency">--</span>
+      </div>
+      <div class="indicator" data-key="trtc">
+        <span class="led led-unknown"></span>
+        <span class="label">TRTC</span>
+        <span class="latency">--</span>
+      </div>
+      <div class="indicator" data-key="llm">
+        <span class="led led-unknown"></span>
+        <span class="label">LLM</span>
+        <span class="latency">--</span>
+      </div>
+      <button id="btn-recheck" class="btn btn-ghost" type="button">Recheck</button>
+    </div>
+  </header>
+
+  <main>
+    <section class="conversation" id="conversation">
+      <div class="placeholder">
+        <h1>Voice Agent Demo</h1>
+        <p>All three indicators must be green before starting.</p>
+        <ul class="hint-list">
+          <li>Click <em>Start</em> to join the room and bring up the AI agent.</li>
+          <li>Click <em>Mic</em> to talk; the AI will answer with voice + live captions.</li>
+          <li>Or type a message — the AI will reply by voice + caption too.</li>
+        </ul>
+      </div>
+    </section>
+  </main>
+
+  <footer class="control-bar">
+    <button id="btn-start" class="btn btn-primary" type="button" disabled>Start</button>
+    <button id="btn-mic" class="btn btn-ghost" type="button" disabled title="Push-to-talk">
+      <span id="mic-icon">🎙️</span> <span id="mic-label">Mic off</span>
+    </button>
+    <button id="btn-stop" class="btn btn-danger" type="button" disabled>Hang up</button>
+    <div class="text-input-group">
+      <input id="text-input" type="text" placeholder="Type a message and press Enter..." disabled />
+      <button id="btn-send" class="btn btn-ghost" type="button" disabled>Send</button>
+    </div>
+    <div class="agent-status" id="agent-status" aria-live="polite">
+      <span class="agent-state agent-state-idle">idle</span>
+    </div>
+  </footer>
+
+  <!-- TRTC Web SDK v5 -->
+  <script src="https://web.sdk.qcloud.com/trtc/webrtc/v5/dist/trtc.js"></script>
+  <script src="/static/app.js"></script>
+</body>
+</html>

package/skills/trtc-ai-service/capabilities/conversation-core/web-demo/styles.css

@@ -0,0 +1,136 @@
+:root {
+  --bg: #0e1116;
+  --panel: #161b22;
+  --border: #30363d;
+  --text: #e6edf3;
+  --muted: #8b949e;
+  --green: #3fb950;
+  --red: #f85149;
+  --yellow: #d29922;
+  --accent: #2f81f7;
+}
+* { box-sizing: border-box; }
+html, body { height: 100%; margin: 0; }
+body {
+  font-family: -apple-system, BlinkMacSystemFont, "PingFang SC", "Helvetica Neue", Arial, sans-serif;
+  background: var(--bg);
+  color: var(--text);
+  display: flex;
+  flex-direction: column;
+}
+
+/* ---------------- Status bar ---------------- */
+.status-bar {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  padding: 12px 24px;
+  background: var(--panel);
+  border-bottom: 1px solid var(--border);
+}
+.brand { display: flex; align-items: center; gap: 8px; font-size: 15px; }
+.brand .dot { width: 8px; height: 8px; border-radius: 50%; background: var(--accent); }
+.indicators { display: flex; align-items: center; gap: 12px; }
+.indicator {
+  display: flex; align-items: center; gap: 6px;
+  background: rgba(255,255,255,0.03);
+  padding: 6px 10px; border-radius: 999px;
+  border: 1px solid var(--border);
+  font-size: 13px;
+}
+.led {
+  width: 10px; height: 10px; border-radius: 50%;
+  display: inline-block;
+  box-shadow: 0 0 6px currentColor;
+}
+.led-unknown { background: var(--muted); color: var(--muted); }
+.led-ok      { background: var(--green); color: var(--green); }
+.led-fail    { background: var(--red);   color: var(--red); }
+.led-pending { background: var(--yellow); color: var(--yellow); animation: blink 1s infinite; }
+@keyframes blink { 50% { opacity: .35; } }
+.indicator .latency { color: var(--muted); font-variant-numeric: tabular-nums; min-width: 38px; text-align: right; }
+
+/* ---------------- Conversation ---------------- */
+main { flex: 1; overflow: auto; padding: 24px; }
+.conversation { max-width: 880px; margin: 0 auto; }
+.placeholder {
+  border: 1px dashed var(--border);
+  border-radius: 12px;
+  padding: 32px;
+  color: var(--muted);
+}
+.placeholder h1 { color: var(--text); margin-top: 0; }
+.hint-list { line-height: 1.8; padding-left: 20px; }
+.hint-list em { color: var(--accent); font-style: normal; }
+
+.bubble {
+  max-width: 70%;
+  padding: 10px 14px;
+  border-radius: 14px;
+  margin: 8px 0;
+  white-space: pre-wrap;
+  word-break: break-word;
+  line-height: 1.5;
+  font-size: 14px;
+}
+.bubble.user {
+  background: var(--accent);
+  margin-left: auto;
+  border-bottom-right-radius: 4px;
+}
+.bubble.agent {
+  background: var(--panel);
+  border: 1px solid var(--border);
+  border-bottom-left-radius: 4px;
+}
+
+/* ---------------- Control bar ---------------- */
+.control-bar {
+  display: flex;
+  flex-wrap: wrap;
+  align-items: center;
+  gap: 12px;
+  padding: 12px 24px;
+  border-top: 1px solid var(--border);
+  background: var(--panel);
+}
+.btn {
+  border: 1px solid var(--border);
+  background: transparent;
+  color: var(--text);
+  padding: 8px 16px;
+  border-radius: 8px;
+  font-size: 14px;
+  cursor: pointer;
+  transition: opacity .15s, background .15s;
+  display: inline-flex; align-items: center; gap: 6px;
+}
+.btn:disabled { opacity: .4; cursor: not-allowed; }
+.btn-primary { background: var(--accent); border-color: var(--accent); }
+.btn-danger  { background: var(--red); border-color: var(--red); }
+.btn-ghost:hover:not(:disabled) { background: rgba(255,255,255,.05); }
+.btn-active { background: var(--red); border-color: var(--red); color: #fff; }
+
+.text-input-group { flex: 1; display: flex; gap: 8px; min-width: 240px; }
+.text-input-group input {
+  flex: 1;
+  padding: 8px 12px;
+  border-radius: 8px;
+  border: 1px solid var(--border);
+  background: var(--bg);
+  color: var(--text);
+  font-size: 14px;
+}
+.text-input-group input:focus { border-color: var(--accent); outline: none; }
+
+.agent-status { font-size: 12px; color: var(--muted); }
+.agent-state {
+  padding: 4px 10px;
+  border-radius: 999px;
+  border: 1px solid var(--border);
+  text-transform: lowercase;
+}
+.agent-state-idle      { color: var(--muted); }
+.agent-state-listening { color: var(--green); border-color: var(--green); }
+.agent-state-thinking  { color: var(--yellow); border-color: var(--yellow); animation: blink 1.2s infinite; }
+.agent-state-speaking  { color: var(--accent); border-color: var(--accent); }

package/skills/trtc-ai-service/capabilities/digital-human/README.md

@@ -0,0 +1,31 @@
+# digital-human · Digital Human Capability (Placeholder)
+
+> Phase 2 only declares the interface contract. Rendering / lip-sync / expression driving
+> and other rendering layer implementations are deferred to future iterations (Phase 3+).
+
+## Current Capabilities
+
+- Register placeholder REST endpoints via manifest: `/api/v1/digital-human/*`
+- Does not modify skeleton runtime behavior; only serves as an integration anchor for the future rendering layer
+
+## REST Placeholders
+
+| Method | Path | Behavior |
+|:---|:---|:---|
+| GET  | `/api/v1/digital-human/status` | Returns current status / roadmap |
+| POST | `/api/v1/digital-human/render` | Always returns `501 Not Implemented` |
+
+## Roadmap
+
+1. Integrate third-party rendering SDKs (Avatar / Lipsync / Expression)
+2. Push rendering driver data via WebRTC datachannel
+3. Align frame output with conversation-core TTS
+
+## Configuration
+
+| Env Variable | Default | Description |
+|:---|:---|:---|
+| `DH_ENABLED` | `false` | Keep false before real enablement to avoid accidental use |
+| `DH_AVATAR_ID` | _(empty)_ | Avatar ID |
+| `DH_LIPSYNC_PROVIDER` | `tencent-cloud-vmp` | Lip-sync provider |
+| `DH_EXPRESSION_PROVIDER` | `internal-rule` | Expression driver provider |