@agentunion/kite 1.0.6 → 1.2.0

package/extensions/services/web/routes/routes_voicechat.py

@@ -0,0 +1,554 @@
+"""Voice Chat WebSocket endpoint — browser-based ASR → LLM → TTS pipeline."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import re
+import struct
+from pathlib import Path
+from typing import Any
+
+from fastapi import APIRouter, Request, WebSocket, WebSocketDisconnect
+
+from vendor import config as cfg
+from vendor.conversation.asr import create_asr_provider
+from vendor.conversation.llm import create_llm_provider
+from vendor.conversation.tts import create_tts_provider
+from vendor.storage import identity
+from vendor.tools.registry import get_registry
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+# ---------------------------------------------------------------------------
+# Lightweight energy-based VAD (no C extension needed)
+# ---------------------------------------------------------------------------
+
+class EnergyVAD:
+    """Simple RMS-energy voice activity detector for S16LE 16 kHz mono PCM."""
+
+    FRAME_BYTES = 640  # 20 ms at 16 kHz S16LE mono
+
+    def __init__(
+        self,
+        energy_threshold: float = 300.0,
+        silence_ms: int = 800,
+        min_speech_ms: int = 250,
+    ) -> None:
+        self._base_threshold = energy_threshold
+        self._threshold = energy_threshold
+        self._silence_frames = silence_ms // 20
+        self._speech_frames = max(1, min_speech_ms // 20)
+        self._in_speech = False
+        self._speech_count = 0
+        self._silence_count = 0
+        self._buf = bytearray()
+        self._last_rms = 0.0
+        self._noise_floor = 0.0
+        self._noise_alpha = 0.05
+        self._noise_margin = 3.0
+        self._noise_samples = 0
+        self._noise_warmup = 15
+        logger.info(
+            "EnergyVAD: base_threshold=%.0f, silence_frames=%d (%dms), speech_frames=%d (%dms), adaptive=on (margin=%.1fx)",
+            self._base_threshold, self._silence_frames, silence_ms,
+            self._speech_frames, min_speech_ms, self._noise_margin,
+        )
+
+    def feed(self, chunk: bytes, external_rms: float | None = None) -> str | None:
+        if external_rms is not None:
+            self._buf.extend(chunk)
+            last_event: str | None = None
+            while len(self._buf) >= self.FRAME_BYTES:
+                self._buf = self._buf[self.FRAME_BYTES:]
+                self._last_rms = external_rms
+                ev = self._process_rms(external_rms)
+                if ev is not None:
+                    last_event = ev
+            return last_event
+
+        self._buf.extend(chunk)
+        last_event: str | None = None
+        while len(self._buf) >= self.FRAME_BYTES:
+            frame = self._buf[: self.FRAME_BYTES]
+            self._buf = self._buf[self.FRAME_BYTES :]
+            ev = self._process(bytes(frame))
+            if ev is not None:
+                last_event = ev
+        return last_event
+
+    def reset(self) -> None:
+        self._in_speech = False
+        self._speech_count = 0
+        self._silence_count = 0
+        self._buf = bytearray()
+        self._noise_samples = 0
+        self._noise_floor = 0.0
+        self._threshold = self._base_threshold
+
+    @property
+    def last_rms(self) -> float:
+        return self._last_rms
+
+    def _rms(self, frame: bytes) -> float:
+        n = len(frame) // 2
+        if n == 0:
+            return 0.0
+        samples = struct.unpack(f"<{n}h", frame)
+        return (sum(s * s for s in samples) / n) ** 0.5
+
+    def _process(self, frame: bytes) -> str | None:
+        rms = self._rms(frame)
+        return self._process_rms(rms)
+
+    def _process_rms(self, rms: float) -> str | None:
+        self._last_rms = rms
+
+        if not self._in_speech and rms < self._threshold:
+            self._noise_samples += 1
+            if self._noise_samples == 1:
+                self._noise_floor = rms
+            else:
+                self._noise_floor += self._noise_alpha * (rms - self._noise_floor)
+            if self._noise_samples >= self._noise_warmup:
+                self._threshold = max(self._base_threshold,
+                                      self._noise_floor * self._noise_margin)
+
+        is_speech = rms > self._threshold
+        if is_speech:
+            self._silence_count = 0
+            self._speech_count += 1
+            if not self._in_speech and self._speech_count >= self._speech_frames:
+                self._in_speech = True
+                logger.debug("VAD: speech_start (rms=%.0f, threshold=%.0f, noise=%.0f)",
+                             rms, self._threshold, self._noise_floor)
+                return "speech_start"
+        else:
+            self._speech_count = 0
+            if self._in_speech:
+                self._silence_count += 1
+                if self._silence_count >= self._silence_frames:
+                    self._in_speech = False
+                    self._silence_count = 0
+                    logger.debug(
+                        "VAD: speech_end (rms=%.0f, threshold=%.0f, noise=%.0f)",
+                        rms, self._threshold, self._noise_floor,
+                    )
+                    return "speech_end"
+        return None
+
+
+DEFAULT_SYSTEM_PROMPT = (
+    "你是一个AI语音助手。用户正在通过语音与你对话。\n"
+    "- 用自然、简洁的中文回复\n"
+    "- 每次回复控制在1-3句话，适合语音播放\n"
+    "- 语气友好自然，像朋友聊天"
+)
+
+_SENTENCE_RE = re.compile(r'(?<=[。！？!?\n])\s*')
+
+
+def _split_sentences(text: str) -> list[str]:
+    parts = _SENTENCE_RE.split(text)
+    sentences: list[str] = []
+    for p in parts:
+        p = p.strip()
+        if not p:
+            continue
+        if sentences and len(p) < 5:
+            sentences[-1] += p
+        else:
+            sentences.append(p)
+    return sentences if sentences else [text]
+
+
+async def _send_json(ws: WebSocket, data: dict) -> bool:
+    try:
+        await ws.send_json(data)
+        return True
+    except Exception:
+        return False
+
+
+async def _send_bytes(ws: WebSocket, data: bytes) -> bool:
+    try:
+        await ws.send_bytes(data)
+        return True
+    except Exception:
+        return False
+
+
+async def _send_tts_segments(ws: WebSocket, tts, text: str, language: str) -> bool:
+    if not await _send_json(ws, {"type": "speaking"}):
+        return False
+
+    sentences = _split_sentences(text)
+    chunk_size = 16 * 1024
+
+    for sentence in sentences:
+        try:
+            audio_data = await tts.synthesize(sentence, language)
+            if audio_data:
+                for i in range(0, len(audio_data), chunk_size):
+                    if not await _send_bytes(ws, audio_data[i:i + chunk_size]):
+                        return False
+                if not await _send_json(ws, {"type": "segment_end"}):
+                    return False
+        except Exception as e:
+            logger.exception("VoiceChat: TTS segment error for: %s", sentence)
+            await _send_json(ws, {"type": "error", "message": f"TTS错误: {e}"})
+
+    await _send_json(ws, {"type": "audio_done"})
+    return True
+
+
+@router.post("/api/voicechat/dump-log")
+async def dump_log(request: Request):
+    """Save voicechat status log to data/voicechat_log.txt for diagnostics."""
+    body = await request.json()
+    log_text = body.get("log", "")
+    if not log_text:
+        return {"ok": False}
+    log_path = cfg.data_dir() / "voicechat_log.txt"
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    log_path.write_text(log_text, encoding="utf-8")
+    logger.info("VoiceChat: status log saved to %s (%d bytes)", log_path, len(log_text))
+    return {"ok": True}
+
+
+@router.websocket("/ws/voice-chat")
+async def voice_chat(ws: WebSocket):
+    """Full-duplex voice chat: browser mic → VAD+ASR → LLM → TTS → browser speaker."""
+    await ws.accept()
+    logger.info("VoiceChat: WebSocket connected")
+
+    asr = None
+    llm = None
+    tts = None
+    vad = None
+    messages: list[dict[str, Any]] = []
+    running = False
+    ctx: dict[str, Any] = {"pending_prompt": None, "tools": None, "tool_ctx": None}
+
+    try:
+        while True:
+            raw = await ws.receive_text()
+            msg = json.loads(raw)
+            if msg.get("type") == "start":
+                system_prompt = msg.get("system_prompt", "").strip() or DEFAULT_SYSTEM_PROMPT
+                break
+
+        asr = create_asr_provider()
+        llm = create_llm_provider()
+        tts = create_tts_provider()
+        vad = EnergyVAD(
+            energy_threshold=float(cfg.get("vad.energy_threshold", 300)),
+            silence_ms=int(cfg.get("vad.silence_threshold_ms", 800)),
+            min_speech_ms=int(cfg.get("vad.min_speech_ms", 250)),
+        )
+
+        phone_number = msg.get("phone_number", "")
+        user_phone = identity.get_user_phone()
+        is_owner = identity.is_owner(user_phone, phone_number) if (user_phone and phone_number) else False
+
+        registry = get_registry()
+        active_provider = cfg.get("llm.active_provider", "openai")
+
+        global_tools_cfg = identity.load_tools_config(cfg.data_dir() / "tools.yaml")
+        user_tools_cfg = identity.load_tools_config(
+            identity.user_dir(user_phone) / "tools.yaml"
+        ) if user_phone else None
+        contact_tools_cfg = identity.load_tools_config(
+            identity.contact_dir(user_phone, phone_number) / "tools.yaml"
+        ) if (user_phone and phone_number) else None
+
+        enabled_tools = registry.resolve_enabled(
+            global_tools_cfg, user_tools_cfg, contact_tools_cfg, None, is_owner,
+        )
+        provider_tools = registry.get_tools_for_provider(active_provider, enabled_tools) if enabled_tools else None
+
+        tool_ctx = {
+            "task_info": {
+                "task_id": f"voicechat-{id(ws)}",
+                "phone_number": phone_number,
+                "user_phone": user_phone,
+            },
+            "webhook": None,
+            "engine": None,
+            "data_dir": cfg.data_dir(),
+            "root_dir": cfg.root_dir(),
+            "is_owner": is_owner,
+        }
+
+        ctx["tools"] = provider_tools
+        ctx["tool_ctx"] = tool_ctx
+        ctx["enabled_tools"] = enabled_tools
+        ctx["registry"] = registry
+        ctx["active_provider"] = active_provider
+
+        logger.info(
+            "VoiceChat: tools resolved — phone=%s, is_owner=%s, enabled=%s",
+            phone_number or "(none)", is_owner, enabled_tools,
+        )
+
+        language = cfg.get("asr.language", "zh")
+
+        tools_summary = registry.build_tools_summary(enabled_tools)
+        full_prompt = system_prompt
+        if tools_summary:
+            full_prompt += "\n\n" + tools_summary
+
+        messages = [{"role": "system", "content": full_prompt}]
+        running = True
+
+        await _send_json(ws, {"type": "ready"})
+        logger.info("VoiceChat: session started")
+
+        ai_first = msg.get("ai_first", False)
+        if ai_first and system_prompt:
+            logger.info("VoiceChat: ai_first mode — generating opening")
+            await _send_json(ws, {"type": "thinking"})
+            try:
+                opening_messages = messages + [
+                    {"role": "user", "content": "请根据系统提示词中的任务目标，主动开始对话。直接说第一句话即可，不要解释。"}
+                ]
+                result = await llm.generate(opening_messages, tools=provider_tools)
+                ai_text = result.get("content", "").strip()
+            except Exception as e:
+                logger.exception("VoiceChat: AI-first LLM error")
+                ai_text = ""
+
+            if ai_text:
+                messages.append({"role": "assistant", "content": ai_text})
+                await _send_json(ws, {"type": "ai_text", "text": ai_text})
+                await _send_json(ws, {"type": "debug_messages", "messages": opening_messages + [{"role": "assistant", "content": ai_text}]})
+                if not await _send_tts_segments(ws, tts, ai_text, language):
+                    running = False
+
+        while running:
+            await _conversation_turn(ws, asr, llm, tts, vad, messages, language, ctx)
+
+    except WebSocketDisconnect:
+        logger.info("VoiceChat: client disconnected")
+    except Exception as e:
+        logger.exception("VoiceChat: unexpected error")
+        await _send_json(ws, {"type": "error", "message": str(e)})
+    finally:
+        running = False
+        if asr:
+            try:
+                await asr.stop_stream()
+            except Exception:
+                pass
+        logger.info("VoiceChat: session ended")
+
+
+async def _conversation_turn(ws, asr, llm, tts, vad, messages, language, ctx):
+    """Execute one listen → ASR → LLM → TTS turn."""
+
+    await _send_json(ws, {"type": "listening"})
+    vad.reset()
+
+    await asr.stop_stream()
+    await asr.start_stream(language)
+
+    speech_started = False
+    audio_chunks: list[bytes] = []
+    last_interim = ""
+    vad_report_counter = 0
+
+    while True:
+        try:
+            raw = await ws.receive()
+        except RuntimeError:
+            raise WebSocketDisconnect()
+
+        if "text" in raw:
+            msg = json.loads(raw["text"])
+            if msg.get("type") == "stop":
+                raise WebSocketDisconnect()
+            if msg.get("type") == "update_system_prompt":
+                new_prompt = msg.get("system_prompt", "").strip()
+                if new_prompt:
+                    ctx["pending_prompt"] = new_prompt
+                    logger.info("VoiceChat: system prompt updated (pending)")
+            if msg.get("type") == "update_vad":
+                threshold = float(msg.get("energy_threshold", vad._base_threshold))
+                silence_ms = int(msg.get("silence_threshold_ms", vad._silence_frames * 20))
+                min_speech_ms = int(msg.get("min_speech_ms", vad._speech_frames * 20))
+                vad._base_threshold = threshold
+                vad._threshold = max(threshold, vad._noise_floor * vad._noise_margin)
+                vad._silence_frames = silence_ms // 20
+                vad._speech_frames = max(1, min_speech_ms // 20)
+                logger.info(
+                    "VoiceChat: VAD params updated live — base_threshold=%.0f, effective=%.0f, silence=%dms, min_speech=%dms",
+                    threshold, vad._threshold, silence_ms, min_speech_ms,
+                )
+            continue
+
+        if "bytes" not in raw or raw["bytes"] is None:
+            continue
+
+        raw_bytes = raw["bytes"]
+        if len(raw_bytes) > 4:
+            pre_agc_rms = struct.unpack('<f', raw_bytes[:4])[0]
+            chunk = raw_bytes[4:]
+        else:
+            pre_agc_rms = None
+            chunk = raw_bytes
+
+        event = vad.feed(chunk, external_rms=pre_agc_rms)
+
+        vad_report_counter += 1
+        if vad_report_counter % 4 == 0:
+            await _send_json(ws, {
+                "type": "vad_info",
+                "rms": round(vad.last_rms, 1),
+                "threshold": round(vad._threshold, 1),
+                "noise_floor": round(vad._noise_floor, 1),
+                "in_speech": vad._in_speech,
+                "silence_count": vad._silence_count,
+                "silence_target": vad._silence_frames,
+            })
+
+        if event == "speech_start" and not speech_started:
+            speech_started = True
+            audio_chunks.clear()
+            last_interim = ""
+            await _send_json(ws, {"type": "speech_start"})
+
+        if speech_started:
+            audio_chunks.append(chunk)
+            await asr.feed_audio(chunk)
+
+            interim = await asr.get_interim_result()
+            if interim and interim != last_interim:
+                last_interim = interim
+                logger.debug("VoiceChat: interim → %s", interim)
+                await _send_json(ws, {"type": "interim", "text": interim})
+
+        if event == "speech_end" and speech_started:
+            await _send_json(ws, {"type": "speech_end"})
+            break
+
+    user_text = await asr.get_result()
+    user_text = user_text.strip()
+
+    if not user_text:
+        logger.info("VoiceChat: empty ASR result, resuming listening")
+        return
+
+    logger.info("VoiceChat: user said: %s", user_text)
+    await _send_json(ws, {"type": "user_text", "text": user_text})
+    messages.append({"role": "user", "content": user_text})
+
+    if ctx.get("pending_prompt"):
+        messages[0]["content"] = ctx["pending_prompt"]
+        ctx["pending_prompt"] = None
+        logger.info("VoiceChat: applied updated system prompt")
+
+    await _send_json(ws, {"type": "thinking"})
+
+    provider_tools = ctx.get("tools")
+    registry = ctx.get("registry")
+    tool_ctx_base = ctx.get("tool_ctx", {})
+
+    try:
+        result = await llm.generate(messages, tools=provider_tools)
+    except Exception as e:
+        logger.exception("VoiceChat: LLM error")
+        await _send_json(ws, {"type": "error", "message": f"LLM错误: {e}"})
+        return
+
+    assistant_msg: dict[str, Any] = {"role": "assistant"}
+    if result.get("content"):
+        assistant_msg["content"] = result["content"]
+    if result.get("tool_calls"):
+        assistant_msg["tool_calls"] = result.get("raw_tool_calls") or result["tool_calls"]
+    messages.append(assistant_msg)
+
+    while result.get("tool_calls") and registry:
+        tool_calls = result["tool_calls"]
+
+        for tc in tool_calls:
+            tc_name = tc.get("name", "")
+            tc_args = tc.get("arguments", {})
+            tc_id = tc.get("id", "")
+
+            await _send_json(ws, {
+                "type": "tool_call",
+                "id": tc_id,
+                "name": tc_name,
+                "arguments": tc_args,
+            })
+            logger.info("VoiceChat: tool call: %s(%s)", tc_name, tc_args)
+
+            handler = registry.get_handler(tc_name)
+            if handler:
+                perm_files = registry.get_permission_files(
+                    tc_name,
+                    tool_ctx_base.get("task_info", {}).get("user_phone", ""),
+                    tool_ctx_base.get("task_info", {}).get("phone_number", ""),
+                )
+                handler_ctx = {
+                    **tool_ctx_base,
+                    "permission_files": perm_files,
+                }
+                try:
+                    tool_result = await handler(tc_args, handler_ctx)
+                except Exception as exc:
+                    logger.exception("VoiceChat: tool execution failed: %s", tc_name)
+                    tool_result = f"Error: {exc}"
+            else:
+                tool_result = f"Unknown tool: {tc_name}"
+
+            await _send_json(ws, {
+                "type": "tool_result",
+                "id": tc_id,
+                "name": tc_name,
+                "result": tool_result[:500],
+            })
+            logger.info("VoiceChat: tool result: %s → %s", tc_name, tool_result[:100])
+
+            messages.append({
+                "role": "tool",
+                "tool_call_id": tc_id,
+                "name": tc_name,
+                "content": tool_result,
+            })
+
+        try:
+            result = await llm.generate(messages, tools=provider_tools)
+        except Exception as e:
+            logger.exception("VoiceChat: LLM error after tool call")
+            await _send_json(ws, {"type": "error", "message": f"LLM错误: {e}"})
+            return
+
+        assistant_msg = {"role": "assistant"}
+        if result.get("content"):
+            assistant_msg["content"] = result["content"]
+        if result.get("tool_calls"):
+            assistant_msg["tool_calls"] = result.get("raw_tool_calls") or result["tool_calls"]
+        messages.append(assistant_msg)
+
+    ai_text = (result.get("content") or "").strip()
+
+    if not ai_text:
+        ai_text = "抱歉，我没有理解你的意思，能再说一遍吗？"
+
+    logger.info("VoiceChat: AI reply: %s", ai_text)
+    await _send_json(ws, {"type": "ai_text", "text": ai_text})
+
+    await _send_json(ws, {"type": "debug_messages", "messages": [m for m in messages]})
+
+    await _send_tts_segments(ws, tts, ai_text, language)
+
+    try:
+        await asr.stop_stream()
+        await asr.start_stream(language)
+    except Exception:
+        logger.debug("VoiceChat: ASR re-init failed", exc_info=True)