@agentunion/kite 1.0.6 → 1.2.0

package/extensions/services/web/vendor/conversation/engine.py

@@ -0,0 +1,548 @@
+"""ConversationEngine -- main orchestrator for AI phone calls."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from datetime import datetime, timezone
+from typing import Any
+
+from .. import config as cfg
+from .asr import ASRProvider, create_asr_provider
+from .llm import (
+    LLMProvider,
+    create_llm_provider,
+    generate_summary,
+    should_end_call,
+)
+from .tts import TTSProvider, create_tts_provider
+from .vad import VADDetector, create_vad_detector
+from ..storage import store
+from ..storage import identity
+from .registry import get_registry
+
+logger = logging.getLogger(__name__)
+
+
+class ConversationEngine:
+    """Orchestrates a single phone call: ASR -> LLM -> TTS loop.
+
+    Parameters
+    ----------
+    audio_pipeline:
+        An object providing ``read_audio() -> bytes`` and
+        ``write_audio(data: bytes) -> None`` async methods for the
+        underlying Bluetooth / telephony layer.
+    task_info:
+        Dict describing the call task -- should include keys like
+        ``task_id``, ``phone_number``, ``purpose``, ``system_prompt``,
+        ``play_text`` (optional pre-scripted opener), etc.
+    webhook_client:
+        An object (or ``None``) with an ``async send(event, payload)``
+        method for notifying the task caller about events.
+    """
+
+    def __init__(
+        self,
+        audio_pipeline: Any,
+        task_info: dict[str, Any],
+        webhook_client: Any | None = None,
+    ) -> None:
+        # Sub-components
+        self.asr: ASRProvider = create_asr_provider()
+        self.llm: LLMProvider = create_llm_provider()
+        self.tts: TTSProvider = create_tts_provider()
+        self.vad: VADDetector = create_vad_detector()
+
+        # External references
+        self.audio = audio_pipeline
+        self.task_info = task_info
+        self.webhook = webhook_client
+
+        # Conversation state
+        self.messages: list[dict[str, Any]] = []
+        self.transcript: list[dict[str, Any]] = []
+        self.running: bool = False
+
+        # Timing
+        self._start_time: float = 0.0
+        self._no_response_count: int = 0
+
+        # Config shortcuts
+        self._max_duration = cfg.get("call.max_duration_seconds", 300)
+        self._no_response_timeout = cfg.get("call.no_response_timeout", 15)
+        self._no_response_max_retries = cfg.get("call.no_response_max_retries", 3)
+        self._active_provider = cfg.get("llm.active_provider", "openai")
+
+        # Tool registry integration
+        self._registry = get_registry()
+        user_phone = task_info.get("user_phone", "")
+        contact_phone = task_info.get("phone_number", "")
+        self._is_owner = identity.is_owner(user_phone, contact_phone) if (user_phone and contact_phone) else False
+
+        # Resolve enabled tools through 4-level hierarchy
+        global_tools_cfg = identity.load_tools_config(cfg.data_dir() / "tools.yaml")
+        user_tools_cfg = identity.load_tools_config(
+            identity.user_dir(user_phone) / "tools.yaml"
+        ) if user_phone else None
+        contact_tools_cfg = identity.load_tools_config(
+            identity.contact_dir(user_phone, contact_phone) / "tools.yaml"
+        ) if (user_phone and contact_phone) else None
+        call_tools_cfg = None
+        if task_info.get("tools_enabled") or task_info.get("tools_disabled"):
+            call_tools_cfg = {
+                "enabled": task_info.get("tools_enabled", []),
+                "disabled": task_info.get("tools_disabled", []),
+            }
+
+        self._enabled_tools = self._registry.resolve_enabled(
+            global_tools_cfg, user_tools_cfg, contact_tools_cfg, call_tools_cfg,
+            self._is_owner,
+        )
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    async def start(self) -> None:
+        """Run the main conversation loop until completion or timeout."""
+        self.running = True
+        self._start_time = time.time()
+        task_id = self.task_info.get("task_id", "unknown")
+
+        logger.info("ConversationEngine: starting call %s", task_id)
+
+        try:
+            # Initialise ASR stream
+            language = cfg.get("asr.whisper.language", "zh")
+            await self.asr.start_stream(language)
+
+            # Build the system message
+            await self._init_system_prompt()
+
+            # Play opening
+            if self.task_info.get("play_text"):
+                await self._speak(self.task_info["play_text"])
+            else:
+                opening = await self._generate_opening()
+                await self._speak(opening)
+
+            # Main loop
+            while self.running:
+                # Check max duration
+                if self._check_timeout():
+                    await self._speak("抱歉，通话时间到了，感谢您的配合，再见。")
+                    break
+
+                # Listen for speech
+                text = await self._listen()
+
+                if not text:
+                    if not await self._handle_no_response():
+                        break
+                    continue
+
+                # Reset no-response counter on valid input
+                self._no_response_count = 0
+
+                # Record the human turn
+                self._record_turn("human", text)
+
+                # Generate AI response (may include tool calls)
+                response = await self._generate_response(text)
+
+                # Handle tool-call loops
+                while response.get("tool_calls"):
+                    tool_results = await self._execute_tools(response["tool_calls"])
+                    response = await self._continue_with_tool_results(
+                        response["tool_calls"], tool_results
+                    )
+
+                # Speak the text response
+                if response.get("content"):
+                    self._record_turn("ai", response["content"])
+                    await self._speak(response["content"])
+
+                # Check if the conversation should end
+                if await self._should_end_call():
+                    farewell = await self._generate_farewell()
+                    self._record_turn("ai", farewell)
+                    await self._speak(farewell)
+                    break
+
+        except asyncio.CancelledError:
+            logger.info("ConversationEngine: call %s cancelled", task_id)
+        except Exception:
+            logger.exception("ConversationEngine: unhandled error in call %s", task_id)
+        finally:
+            self.running = False
+            await self.asr.stop_stream()
+            logger.info("ConversationEngine: call %s ended", task_id)
+
+    async def stop(self) -> None:
+        """Signal the engine to stop at the next opportunity."""
+        self.running = False
+
+    async def inject_message(self, message: str) -> None:
+        """Inject a message from the task caller into the conversation context."""
+        self.messages.append({
+            "role": "system",
+            "content": f"[来自任务调用者的消息]: {message}",
+        })
+        logger.info("ConversationEngine: injected caller message: %s", message[:80])
+
+    async def generate_call_summary(self) -> str:
+        """Generate and persist a summary of the entire call."""
+        summary = await generate_summary(self.llm, self.messages)
+
+        # Save to identity directory
+        call_path = self.task_info.get("call_dir")
+        if call_path and summary:
+            from pathlib import Path
+            await identity.save_call_summary(Path(call_path), summary)
+
+        return summary
+
+    # ------------------------------------------------------------------
+    # System prompt
+    # ------------------------------------------------------------------
+
+    async def _init_system_prompt(self) -> None:
+        """Build the initial system message from task info, config, and identity context."""
+        purpose = self.task_info.get("purpose", "")
+        custom_prompt = self.task_info.get("system_prompt", "")
+        phone_number = self.task_info.get("phone_number", "")
+        contact_name = self.task_info.get("contact_name", "")
+        direction = self.task_info.get("direction", "outgoing")
+
+        opening_hint = cfg.get("call.opening_message", "")
+
+        parts: list[str] = [
+            "你是一个AI电话助手。你正在进行一通电话。",
+            f"通话方向: {'拨出' if direction == 'outgoing' else '接入'}",
+        ]
+        if phone_number:
+            parts.append(f"对方号码: {phone_number}")
+        if contact_name:
+            parts.append(f"对方姓名: {contact_name}")
+        if purpose:
+            parts.append(f"通话目的: {purpose}")
+        if custom_prompt:
+            parts.append(f"\n额外指令:\n{custom_prompt}")
+        if opening_hint:
+            parts.append(f"开场白参考: {opening_hint}")
+
+        parts.append(
+            "\n注意事项:\n"
+            "- 用自然、礼貌的中文对话\n"
+            "- 保持简洁，每次回复不超过两三句话\n"
+            "- 如需确认信息可以使用工具\n"
+            "- 完成任务后礼貌结束通话"
+        )
+
+        # Load identity context (user profile, contact profile, recent summaries)
+        identity_context = await self._load_identity_context()
+        if identity_context:
+            parts.append(identity_context)
+
+        # Load data README (panoramic overview)
+        data_readme = cfg.data_dir() / "README.md"
+        if data_readme.exists():
+            try:
+                readme_content = data_readme.read_text(encoding="utf-8").strip()
+                if readme_content:
+                    parts.append(f"\n# 系统全景\n{readme_content}")
+            except Exception:
+                pass
+
+        # Append tools summary
+        tools_summary = self._registry.build_tools_summary(self._enabled_tools)
+        if tools_summary:
+            parts.append(f"\n{tools_summary}")
+
+        system_content = "\n".join(parts)
+        self.messages.append({"role": "system", "content": system_content})
+
+    async def _load_identity_context(self) -> str:
+        """Load identity context files for the system prompt."""
+        user_phone = self.task_info.get("user_phone", "")
+        contact_phone = self.task_info.get("phone_number", "")
+
+        if not user_phone or not contact_phone:
+            return ""
+
+        parts: list[str] = []
+
+        try:
+            user_ctx = await identity.load_user_context(user_phone)
+            contact_ctx = await identity.load_contact_context(user_phone, contact_phone)
+            summaries = await identity.load_recent_summaries(user_phone, contact_phone)
+        except Exception as exc:
+            logger.warning("Failed to load identity context: %s", exc)
+            return ""
+
+        if user_ctx:
+            parts.append(f"\n# 用户信息\n{user_ctx}")
+        if contact_ctx:
+            parts.append(f"\n# 联系人信息\n{contact_ctx}")
+        if summaries:
+            parts.append(f"\n# 近期通话记录\n{summaries}")
+
+        return "\n".join(parts)
+
+    # ------------------------------------------------------------------
+    # Opening / farewell
+    # ------------------------------------------------------------------
+
+    async def _generate_opening(self) -> str:
+        """Ask the LLM to produce an appropriate opening line."""
+        self.messages.append({
+            "role": "user",
+            "content": "[系统] 请生成一句合适的开场白。",
+        })
+        result = await self.llm.generate(self.messages)
+        opening = result.get("content", "你好！")
+        # Replace the placeholder user message with the actual assistant response
+        self.messages.pop()
+        self.messages.append({"role": "assistant", "content": opening})
+        return opening
+
+    async def _generate_farewell(self) -> str:
+        """Ask the LLM to produce a farewell message."""
+        self.messages.append({
+            "role": "user",
+            "content": "[系统] 通话即将结束，请生成一句结束语。",
+        })
+        result = await self.llm.generate(self.messages)
+        farewell = result.get("content", "感谢您的时间，再见！")
+        self.messages.pop()
+        self.messages.append({"role": "assistant", "content": farewell})
+        return farewell
+
+    # ------------------------------------------------------------------
+    # Listening (ASR + VAD)
+    # ------------------------------------------------------------------
+
+    async def _listen(self) -> str | None:
+        """Listen for speech using VAD to detect utterance boundaries.
+
+        Returns the transcribed text or ``None`` on timeout / no speech.
+        """
+        self.vad.reset()
+        await self.asr.start_stream(cfg.get("asr.whisper.language", "zh"))
+        speech_detected = False
+        timeout = self._no_response_timeout
+        deadline = time.time() + timeout
+
+        while self.running and time.time() < deadline:
+            try:
+                chunk = await asyncio.wait_for(
+                    self.audio.read_audio(),
+                    timeout=0.5,
+                )
+            except asyncio.TimeoutError:
+                continue
+            except Exception:
+                logger.debug("_listen: audio read error")
+                break
+
+            if not chunk:
+                continue
+
+            # Feed to VAD
+            event = self.vad.feed(chunk)
+
+            if event == "speech_start":
+                speech_detected = True
+                # Extend deadline while user is speaking
+                deadline = time.time() + timeout
+
+            if speech_detected:
+                await self.asr.feed_audio(chunk)
+
+            if event == "speech_end" and speech_detected:
+                break
+
+            # If speech is ongoing, keep extending the deadline
+            if self.vad.is_speaking:
+                deadline = time.time() + timeout
+
+        if not speech_detected:
+            await self.asr.stop_stream()
+            return None
+
+        text = await self.asr.get_result()
+        await self.asr.stop_stream()
+        text = text.strip() if text else None
+        if text:
+            logger.info("User said: %s", text)
+        return text
+
+    # ------------------------------------------------------------------
+    # Speaking (TTS)
+    # ------------------------------------------------------------------
+
+    async def _speak(self, text: str) -> None:
+        """Synthesize *text* via TTS and play through the audio pipeline."""
+        if not text:
+            return
+        logger.info("AI says: %s", text)
+        try:
+            audio_data = await self.tts.synthesize(text)
+            if audio_data and self.audio:
+                await self.audio.write_audio(audio_data)
+        except Exception:
+            logger.exception("_speak: TTS / playback error")
+
+    # ------------------------------------------------------------------
+    # LLM response generation
+    # ------------------------------------------------------------------
+
+    async def _generate_response(self, user_text: str) -> dict[str, Any]:
+        """Append the user message and ask the LLM for a response."""
+        self.messages.append({"role": "user", "content": user_text})
+        tools = self._registry.get_tools_for_provider(
+            self._active_provider, self._enabled_tools
+        )
+        result = await self.llm.generate(self.messages, tools=tools)
+
+        # Append assistant message to history
+        assistant_msg: dict[str, Any] = {"role": "assistant"}
+        if result.get("content"):
+            assistant_msg["content"] = result["content"]
+        if result.get("tool_calls"):
+            # Use raw_tool_calls (original API format) for message history if available
+            assistant_msg["tool_calls"] = result.get("raw_tool_calls") or result["tool_calls"]
+        self.messages.append(assistant_msg)
+
+        return result
+
+    async def _continue_with_tool_results(
+        self,
+        tool_calls: list[dict[str, Any]],
+        tool_results: list[dict[str, Any]],
+    ) -> dict[str, Any]:
+        """Feed tool results back to the LLM and get the next response."""
+        for tc, result in zip(tool_calls, tool_results):
+            self.messages.append({
+                "role": "tool",
+                "tool_call_id": tc.get("id", ""),
+                "name": tc.get("name", ""),
+                "content": result.get("content", ""),
+            })
+
+        tools = self._registry.get_tools_for_provider(
+            self._active_provider, self._enabled_tools
+        )
+        response = await self.llm.generate(self.messages, tools=tools)
+
+        # Append assistant response to history
+        assistant_msg: dict[str, Any] = {"role": "assistant"}
+        if response.get("content"):
+            assistant_msg["content"] = response["content"]
+        if response.get("tool_calls"):
+            assistant_msg["tool_calls"] = response.get("raw_tool_calls") or response["tool_calls"]
+        self.messages.append(assistant_msg)
+
+        return response
+
+    # ------------------------------------------------------------------
+    # Tool execution
+    # ------------------------------------------------------------------
+
+    async def _execute_tools(
+        self, tool_calls: list[dict[str, Any]]
+    ) -> list[dict[str, Any]]:
+        """Execute a batch of tool calls and return their results."""
+        results: list[dict[str, Any]] = []
+        for tc in tool_calls:
+            name = tc.get("name", "")
+            args = tc.get("arguments", {})
+            logger.info("Executing tool: %s(%s)", name, args)
+
+            try:
+                result = await self._execute_single_tool(name, args)
+                results.append({"content": result})
+            except Exception as exc:
+                logger.exception("Tool execution failed: %s", name)
+                results.append({"content": f"Error: {exc}"})
+
+        return results
+
+    async def _execute_single_tool(self, name: str, args: dict[str, Any]) -> str:
+        """Dispatch a single tool call to the registry handler."""
+        handler = self._registry.get_handler(name)
+        if handler is None:
+            return f"Unknown tool: {name}"
+
+        context = self._registry.build_context(name, self, self._is_owner)
+        return await handler(args, context)
+
+    # ------------------------------------------------------------------
+    # No-response handling
+    # ------------------------------------------------------------------
+
+    async def _handle_no_response(self) -> bool:
+        """Handle the case where the user did not respond.
+
+        Returns ``True`` to continue the loop, ``False`` to end the call.
+        """
+        self._no_response_count += 1
+        if self._no_response_count >= self._no_response_max_retries:
+            await self._speak("您好？因为长时间没有回应，我先挂断了，再见。")
+            return False
+
+        prompts = [
+            "您好？请问您还在吗？",
+            "不好意思，我没有听到您的声音，请问您还在线吗？",
+            "抱歉打扰了，如果您还在的话请说话。",
+        ]
+        idx = min(self._no_response_count - 1, len(prompts) - 1)
+        await self._speak(prompts[idx])
+        return True
+
+    # ------------------------------------------------------------------
+    # End-of-call detection
+    # ------------------------------------------------------------------
+
+    async def _should_end_call(self) -> bool:
+        """Determine whether the call should end naturally."""
+        return await should_end_call(self.llm, self.messages)
+
+    # ------------------------------------------------------------------
+    # Timeout
+    # ------------------------------------------------------------------
+
+    def _check_timeout(self) -> bool:
+        """Return ``True`` if the call has exceeded the maximum duration."""
+        if self._max_duration <= 0:
+            return False
+        elapsed = time.time() - self._start_time
+        return elapsed >= self._max_duration
+
+    # ------------------------------------------------------------------
+    # Transcript recording
+    # ------------------------------------------------------------------
+
+    def _record_turn(self, role: str, text: str) -> None:
+        """Record a conversation turn in the transcript and persist it."""
+        now = datetime.now(timezone.utc)
+        entry = {
+            "role": role,
+            "text": text,
+            "timestamp": now.isoformat(),
+            "elapsed": round(time.time() - self._start_time, 1),
+        }
+        self.transcript.append(entry)
+
+        # Persist to identity directory (fire-and-forget)
+        call_path = self.task_info.get("call_dir")
+        if call_path:
+            from pathlib import Path
+            msg_entry = {
+                "role": "human" if role == "human" else "assistant",
+                "timestamp": now.isoformat(),
+                "text": text,
+            }
+            asyncio.ensure_future(identity.save_call_message(Path(call_path), msg_entry))