alive-ai 0.1.0

package/brain/memory/__init__.py

@@ -0,0 +1,123 @@
+"""Brain: Memory - semantic, episodic, working, vector memory"""
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+from .manager import Memory  # noqa: F401 - re-export for backward compat
+
+
+class SemanticMemory:
+    """Long-term facts about the user - flat simple structure, per-user"""
+
+    PET_NAMES = ["babe", "baby", "love", "handsome"]
+
+    def __init__(self, data_path: Path, user_id: str = "default"):
+        """
+        Initialize semantic memory for a specific user.
+
+        Args:
+            data_path: User's base path (already includes users/{user_id})
+            user_id: User's Telegram ID (for reference)
+        """
+        self.user_id = user_id
+        # data_path is already the user's base path (data/users/{user_id})
+        self.path = data_path / "facts.json"
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.facts = self._load()
+
+    def _load(self) -> dict:
+        if self.path.exists():
+            return json.loads(self.path.read_text())
+        # Clean default - no precoded info
+        return {
+            "name": None, "nickname": None, "gender": None, "age": None,
+            "location": None, "job": None, "hobbies": [], "interests": [],
+            "personality": [], "relationship_status": None,
+            "pet_names_used": [], "mentions": {},
+            "shared_memories": [], "last_intimate": None
+        }
+
+    def save(self):
+        self.path.write_text(json.dumps(self.facts, indent=2))
+
+    @property
+    def is_new_user(self) -> bool:
+        """Check if we know anything about this user"""
+        return not bool(
+            self.facts.get("name") or
+            self.facts.get("mentions") or
+            self.facts.get("hobbies") or
+            self.facts.get("interests")
+        )
+
+    def update(self, key: str, value):
+        """Update a fact"""
+        if isinstance(value, list) and key in self.facts and isinstance(self.facts[key], list):
+            # Merge lists, no duplicates
+            existing = set(str(v) for v in self.facts[key])
+            for item in value:
+                if str(item) not in existing:
+                    self.facts[key].append(item)
+        elif value:
+            self.facts[key] = value
+        self.save()
+
+    def add_mention(self, key: str, value: str):
+        self.facts["mentions"][key] = {
+            "value": value, "timestamp": datetime.now().isoformat()}
+        self.save()
+
+    def add_shared_memory(self, memory: str):
+        self.facts["shared_memories"].append({
+            "memory": memory, "timestamp": datetime.now().isoformat()})
+        self.facts["shared_memories"] = self.facts["shared_memories"][-50:]
+        self.save()
+
+    def update_last_intimate(self):
+        self.facts["last_intimate"] = datetime.now().isoformat()
+        self.save()
+
+    def get_random_pet_name(self) -> str:
+        import random
+        used = self.facts.get("pet_names_used", [])
+        available = [p for p in self.PET_NAMES if p not in used[-4:]]
+        name = random.choice(available or self.PET_NAMES)
+        used.append(name)
+        self.facts["pet_names_used"] = used[-20:]
+        self.save()
+        return name
+
+    def get_user_profile(self) -> str:
+        """Get formatted user profile string"""
+        parts = []
+        if self.facts.get("name"):
+            parts.append(f"Name: {self.facts['name']}" +
+                        (f" ({self.facts['nickname']})" if self.facts.get("nickname") else ""))
+        if self.facts.get("gender"):
+            parts.append(f"Gender: {self.facts['gender']}")
+        if self.facts.get("age"):
+            parts.append(f"Age: {self.facts['age']}")
+        if self.facts.get("location"):
+            parts.append(f"Location: {self.facts['location']}")
+        if self.facts.get("job"):
+            parts.append(f"Job: {self.facts['job']}")
+        if self.facts.get("hobbies"):
+            parts.append(f"Hobbies: {', '.join(self.facts['hobbies'])}")
+        if self.facts.get("interests"):
+            parts.append(f"Interests: {', '.join(self.facts['interests'])}")
+        if self.facts.get("personality"):
+            parts.append(f"Personality: {', '.join(self.facts['personality'])}")
+        return "\n".join(parts)
+
+    def get_context(self) -> str:
+        """Get context string for LLM"""
+        parts = [self.get_user_profile()] if self.get_user_profile() else []
+        mentions = self.facts.get("mentions", {})
+        if mentions:
+            for key, data in sorted(mentions.items(),
+                                   key=lambda x: x[1].get("timestamp",""),
+                                   reverse=True)[:3]:
+                parts.append(f"Recently mentioned ({key}): {data['value'][:80]}")
+        for m in self.facts.get("shared_memories", [])[-3:]:
+            parts.append(f"Shared memory: {m['memory']}")
+        return "\n".join(parts)

package/brain/memory/episodic.py

@@ -0,0 +1,92 @@
+"""
+Brain: Episodic Memory
+Event and conversation memory - per-user storage
+"""
+
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+class EpisodicMemory:
+    """Episodic memory for events - per-user conversation storage"""
+
+    def __init__(self, data_path: Path, user_id: str = "default"):
+        """
+        Initialize episodic memory for a specific user.
+
+        Args:
+            data_path: User's base path (already includes users/{user_id})
+            user_id: User's Telegram ID (for reference)
+        """
+        self.user_id = user_id
+        # data_path is already the user's base path (data/users/{user_id})
+        self.path = data_path / "conversations"
+        self.path.mkdir(parents=True, exist_ok=True)
+
+    def save(self, user_msg: str, ai_response: str, emotion: dict):
+        """Save conversation turn"""
+        date = datetime.now().strftime("%Y-%m-%d")
+        file = self.path / f"{date}.jsonl"
+
+        entry = {
+            "timestamp": datetime.now().isoformat(),
+            "user": user_msg,
+            "ai": ai_response,
+            "emotion": emotion
+        }
+
+        with open(file, "a") as f:
+            f.write(json.dumps(entry) + "\n")
+
+    def save_proactive(self, ai_msg: str, emotion: dict):
+        """Save a proactive message (Alive-AI initiated, no user message)"""
+        date = datetime.now().strftime("%Y-%m-%d")
+        file = self.path / f"{date}.jsonl"
+
+        entry = {
+            "timestamp": datetime.now().isoformat(),
+            "user": "",  # Empty - Alive-AI initiated
+            "ai": ai_msg,
+            "emotion": {**emotion, "proactive": True}
+        }
+
+        with open(file, "a") as f:
+            f.write(json.dumps(entry) + "\n")
+
+    def load_recent(self, limit: int = 5) -> list:
+        """Load recent conversations (most recent first, then reversed to chronological)"""
+        all_entries = []
+
+        for file in sorted(self.path.glob("*.jsonl"), reverse=True):
+            file_entries = []
+            with open(file) as f:
+                for line in f:
+                    try:
+                        file_entries.append(json.loads(line))
+                    except Exception:
+                        pass
+            # Reverse so newest entries from this file come first
+            all_entries.extend(reversed(file_entries))
+            if len(all_entries) >= limit:
+                break
+
+        # Take the most recent 'limit' entries, then reverse to chronological order
+        result = list(reversed(all_entries[:limit]))
+        print(f"[Episodic] Loading {len(result)} recent entries from {len(all_entries)} total")
+        return result
+
+    def get_by_date(self, date_str: str) -> list:
+        """Get conversations by date"""
+        file = self.path / f"{date_str}.jsonl"
+        if not file.exists():
+            return []
+
+        entries = []
+        with open(file) as f:
+            for line in f:
+                try:
+                    entries.append(json.loads(line))
+                except:
+                    pass
+        return entries

package/brain/memory/fact_extractor.py

@@ -0,0 +1,209 @@
+"""Brain: Memory - Fact Extractor using LLM"""
+import json
+import re
+from pathlib import Path
+
+EXTRACT_PROMPT = """You are analyzing a conversation between Alive-AI (AI) and a HUMAN USER.
+Extract facts about THE HUMAN USER ONLY - nothing about Alive-AI.
+
+Look at what the HUMAN says about THEMSELF and their relationship with Alive-AI. Extract:
+- name, nickname, age, gender, job, location
+- hobbies, interests, favorite things
+- personality traits, communication style
+- relationship to Alive-AI (creator, boyfriend, etc.)
+- pet names they use (daddy, baby, etc.)
+- intimacy preferences, preferences mentioned
+- what they like about Alive-AI
+- important people in their life
+
+Return ONLY valid JSON with keys where you found NEW info about the HUMAN.
+Use these keys: name, nickname, gender, age, location, job, hobbies, interests, personality, relationship_status, pet_names_used, likes_about_me, intimacy_preferences
+Return empty {} if nothing new was shared about the human.
+NO markdown, ONLY raw JSON. Do NOT use ... or etc."""
+
+
+def _repair_json(text: str) -> dict:
+    """Try to repair and parse malformed JSON from LLM output"""
+    text = text.strip()
+
+    # Remove markdown fences
+    text = re.sub(r'^```(?:json)?\s*', '', text)
+    text = re.sub(r'\s*```$', '', text)
+
+    # Remove trailing ellipsis and everything after
+    text = re.sub(r'\s*\.\.\..*$', '', text)
+    text = re.sub(r'\s*etc\.?.*$', '', text, flags=re.IGNORECASE)
+
+    # Remove trailing commas before } or ]
+    text = re.sub(r',\s*([}\]])', r'\1', text)
+
+    # Try direct parse first
+    try:
+        return json.loads(text)
+    except:
+        pass
+
+    # Try to extract complete key-value pairs manually
+    # Pattern: "key": value (string, number, null, list, or truncated)
+    extracted = {}
+
+    # Match string values: "key": "value"
+    for match in re.finditer(r'"(\w+)":\s*"([^"]*)"', text):
+        extracted[match.group(1)] = match.group(2)
+
+    # Match null values: "key": null
+    for match in re.finditer(r'"(\w+)":\s*null', text):
+        extracted[match.group(1)] = None
+
+    # Match number values: "key": 123
+    for match in re.finditer(r'"(\w+)":\s*(\d+(?:\.\d+)?)', text):
+        val = match.group(2)
+        extracted[match.group(1)] = float(val) if '.' in val else int(val)
+
+    # Match simple list values: "key": ["a", "b"]
+    for match in re.finditer(r'"(\w+)":\s*\[([^\]]*)\]', text):
+        key = match.group(1)
+        list_content = match.group(2)
+        # Extract string items from list
+        items = re.findall(r'"([^"]*)"', list_content)
+        if items:
+            extracted[key] = items
+
+    return extracted if extracted else {}
+
+
+class FactExtractor:
+    """Extracts user facts from conversation using LLM"""
+
+    def __init__(self, facts_path: Path):
+        self.facts_path = facts_path
+        self._llm = None
+        self._turn_buffer = []
+        self._extract_every = 5
+
+    def set_llm(self, llm):
+        """Set the fast LLM client (called after init)"""
+        self._llm = llm
+
+    def add_turn(self, user_msg: str, ai_msg: str):
+        """Buffer a conversation turn"""
+        self._turn_buffer.append({"user": user_msg, "ai": ai_msg})
+
+    def should_extract(self) -> bool:
+        """Check if we have enough turns to extract"""
+        return len(self._turn_buffer) >= self._extract_every
+
+    async def extract_and_merge(self) -> dict:
+        """Extract facts from buffered turns and merge into facts.json"""
+        if not self._llm or not self._turn_buffer:
+            return {}
+
+        # Build conversation text from buffer
+        lines = []
+        for turn in self._turn_buffer[-self._extract_every:]:
+            lines.append(f"User: {turn['user']}")
+            lines.append(f"Alive-AI: {turn['ai']}")
+        conversation = "\n".join(lines)
+
+        try:
+            messages = [
+                {"role": "system", "content": EXTRACT_PROMPT},
+                {"role": "user", "content": conversation}
+            ]
+            response = await self._llm.chat(messages, max_tokens=500, temperature=0.1)
+            if not response:
+                return {}
+
+            # Use robust JSON parser that handles truncated/malformed output
+            extracted = _repair_json(response)
+
+            if not extracted:
+                print(f"[FactExtractor] No valid JSON found in response")
+                return {}
+
+        except Exception as e:
+            print(f"[FactExtractor] Extract error: {e}")
+            return {}
+
+        # Merge into facts.json
+        merged = self._merge_facts(extracted)
+        self._turn_buffer.clear()
+        return merged
+
+    @staticmethod
+    def _is_duplicate(new_item: str, existing_items: list) -> bool:
+        """Check if a new fact is a duplicate of any existing fact.
+        Uses exact match, substring containment, and word-overlap similarity."""
+        new_lower = new_item.lower().strip()
+        if not new_lower:
+            return True
+
+        new_words = set(re.findall(r'\w+', new_lower))
+
+        for existing in existing_items:
+            ex_lower = str(existing).lower().strip()
+
+            # Exact match
+            if new_lower == ex_lower:
+                return True
+
+            # Substring containment (either direction)
+            if len(new_lower) >= 3 and len(ex_lower) >= 3:
+                if new_lower in ex_lower or ex_lower in new_lower:
+                    return True
+
+            # Word overlap: if 70%+ of words overlap, it's a duplicate
+            ex_words = set(re.findall(r'\w+', ex_lower))
+            if new_words and ex_words:
+                overlap = len(new_words & ex_words)
+                smaller = min(len(new_words), len(ex_words))
+                if smaller > 0 and overlap / smaller >= 0.7:
+                    return True
+
+        return False
+
+    def _merge_facts(self, extracted: dict) -> dict:
+        """Merge extracted facts into flat facts.json structure"""
+        if not extracted:
+            return {}
+
+        try:
+            facts = json.loads(self.facts_path.read_text()) if self.facts_path.exists() else {}
+        except Exception:
+            facts = {}
+
+        # Map LLM output keys to our flat structure
+        key_map = {
+            "name": "name", "nickname": "nickname", "gender": "gender",
+            "age": "age", "location": "location", "job": "job",
+            "hobbies": "hobbies", "interests": "interests",
+            "favorite_things": "interests", "personality": "personality",
+            "personality_traits": "personality", "relationship_status": "relationship_status",
+            "pet_names_used": "pet_names_used", "likes_about_me": "likes_about_me",
+            "intimacy_preferences": "intimacy_preferences",
+        }
+
+        for llm_key, value in extracted.items():
+            fact_key = key_map.get(llm_key, llm_key)
+            if not value:
+                continue
+            # Handle list fields
+            if fact_key in ["hobbies", "interests", "personality", "pet_names_used", "likes_about_me", "intimacy_preferences"]:
+                if fact_key not in facts:
+                    facts[fact_key] = []
+                items_to_add = value if isinstance(value, list) else [value]
+                for item in items_to_add:
+                    if not self._is_duplicate(str(item), facts[fact_key]):
+                        facts[fact_key].append(item)
+            else:
+                # Simple fields - only overwrite if currently empty
+                if not facts.get(fact_key):
+                    facts[fact_key] = value
+
+        try:
+            self.facts_path.write_text(json.dumps(facts, indent=2))
+            print(f"[FactExtractor] Merged facts: {list(extracted.keys())}")
+        except Exception as e:
+            print(f"[FactExtractor] Save error: {e}")
+
+        return extracted

package/brain/memory/index.py

@@ -0,0 +1,54 @@
+"""
+Brain: Memory Index
+Fast index for intelligent memory loading
+"""
+
+import json
+from pathlib import Path
+
+class MemoryIndex:
+    """Memory index for efficient loading"""
+
+    def __init__(self, data_path: Path):
+        self.path = data_path / "memory_index.json"
+        self.index = self._load()
+
+    def _load(self) -> dict:
+        if self.path.exists():
+            return json.loads(self.path.read_text())
+        return {
+            "user_profile": {"tokens": 50, "priority": 1},
+            "conversations": {},
+            "facts": {}
+        }
+
+    def save(self):
+        self.path.write_text(json.dumps(self.index, indent=2))
+
+    def add_conversation(self, conv_id: str, tokens: int, emotion: float):
+        self.index["conversations"][conv_id] = {
+            "tokens": tokens,
+            "emotion_score": emotion,
+            "timestamp": conv_id
+        }
+        self.save()
+
+    def get_priority_items(self, max_tokens: int) -> list:
+        """Get items to load within budget"""
+        items = []
+        total = 0
+
+        # Sort by emotion score
+        convs = sorted(
+            self.index["conversations"].items(),
+            key=lambda x: x[1].get("emotion_score", 0),
+            reverse=True
+        )
+
+        for conv_id, data in convs:
+            tokens = data.get("tokens", 100)
+            if total + tokens <= max_tokens:
+                items.append(("conversation", conv_id, data))
+                total += tokens
+
+        return items

package/brain/memory/manager.py

@@ -0,0 +1,151 @@
+"""Brain: Memory Manager - coordinates all memory subsystems"""
+import asyncio
+from pathlib import Path
+from typing import Optional
+from .working import WorkingMemory
+from .episodic import EpisodicMemory
+from .vector_store import VectorMemoryStore
+from .fact_extractor import FactExtractor
+from .summarizer import ConversationSummarizer
+from .index import MemoryIndex
+
+
+class Memory:
+    """Memory manager - coordinates all memory systems for a specific user"""
+
+    def __init__(self, nervous, data_path, embedding_service=None, user_id: str = "default", bot_id: str = "alive_ai"):
+        """
+        Initialize memory for a specific user.
+
+        Args:
+            nervous: The nervous system for events
+            data_path: Base path for data storage
+            embedding_service: Optional embedding service for vector store
+            user_id: The user's Telegram ID for per-user memory isolation
+            bot_id: The Bot instance ID for per-tenant memory isolation
+        """
+        self.nervous = nervous
+        self.user_id = user_id
+        self.data_path = Path(data_path)
+        self.data_path.mkdir(parents=True, exist_ok=True)
+
+        from . import SemanticMemory
+        self.index = MemoryIndex(self.data_path)
+        self.working = WorkingMemory()
+        self.episodic = EpisodicMemory(self.data_path, user_id=user_id)
+        self.semantic = SemanticMemory(self.data_path, user_id=user_id)
+        self.fact_extractor = FactExtractor(self.data_path / "facts.json")
+        self.summarizer = ConversationSummarizer(self.data_path)
+        self.vector_store = None
+        self.bot_id = bot_id.lower()
+        if embedding_service:
+            self.vector_store = VectorMemoryStore(embedding_service, user_id=user_id, bot_id=bot_id)
+            if self.vector_store.connect():
+                print(f"[Memory] Vector store ready for user {user_id} on bot {bot_id}! {self.vector_store.count()} memories")
+        self.turn_count = 0
+        nervous.on("memory_save", self._on_save)
+
+    def set_llm(self, fast_llm):
+        self.fact_extractor.set_llm(fast_llm)
+        self.summarizer.set_llm(fast_llm)
+
+    def _on_save(self, data: dict):
+        if data.get("type") != "conversation":
+            return
+
+        # Check if this event is for this user (for per-user isolation)
+        event_user_id = data.get("user_id")
+        if event_user_id and event_user_id != self.user_id:
+            return  # Not for this user, skip
+
+        user_msg, ai_msg = data.get("user_message", ""), data.get("ai_response", "")
+        emotion = data.get("emotion", {})
+        is_proactive = emotion.get("proactive", False)
+
+        # For proactive messages, only save AI response (no user message)
+        if is_proactive and not user_msg:
+            self.working.add("assistant", ai_msg)
+            # Also save to episodic so it persists across restarts
+            self.episodic.save_proactive(ai_msg, emotion)
+            print(f"[Memory] Saved PROACTIVE message to working + episodic for user {self.user_id}")
+            return
+
+        self.episodic.save(user_msg, ai_msg, emotion)
+        self.working.add("user", user_msg)
+        self.working.add("assistant", ai_msg)
+        print(f"[Memory] Saved to working memory (now {len(self.working)} items) | User: {user_msg[:30]}...")
+        if self.vector_store:
+            self.vector_store.store("user", user_msg, {"emotion": emotion})
+            self.vector_store.store("assistant", ai_msg, {"emotion": emotion})
+        self.fact_extractor.add_turn(user_msg, ai_msg)
+        self.summarizer.add_turn(user_msg, ai_msg)
+        if self.fact_extractor.should_extract():
+            asyncio.ensure_future(self._run_extraction())
+        if self.summarizer.should_summarize():
+            asyncio.ensure_future(self.summarizer.summarize())
+        if emotion.get("is_high_desire") or emotion.get("desire", 0) > 0.6:
+            self.semantic.update_last_intimate()
+        self.turn_count += 1
+        if self.turn_count % 50 == 0 and self.vector_store:
+            self.vector_store.archive_old_memories(max_in_redis=500)
+
+    async def _run_extraction(self):
+        try:
+            await self.fact_extractor.extract_and_merge()
+            self.semantic.facts = self.semantic._load()
+        except Exception as e:
+            print(f"[Memory] Fact extraction error (non-fatal): {e}")
+
+    def search_relevant_memories(self, query: str, limit: int = 5) -> str:
+        if not self.vector_store:
+            return ""
+        memories = self.vector_store.search(query, limit=limit)
+        if not memories:
+            return ""
+        return "\n".join(
+            f"{'He said' if m.get('role')=='user' else 'You said'}: {m.get('content','')}"
+            for m in memories)
+
+    async def build_context(self, max_tokens: int = None, current_message: str = "") -> tuple:
+        import os
+        if max_tokens is None:
+            max_tokens = int(os.environ.get("LLM_CONTEXT_TOKENS", "500"))
+        facts_parts = []
+        if self.semantic.is_new_user:
+            facts_parts.append("[NEW USER - You don't know him yet. Be curious, ask about him. "
+                               "Do NOT pretend you know him or missed him.]")
+        semantic_ctx = self.semantic.get_context()
+        if semantic_ctx:
+            facts_parts.append(semantic_ctx)
+        summaries = self.summarizer.get_recent_summaries(limit=3)
+        if summaries:
+            facts_parts.append(summaries)
+        related = ""
+        if current_message and self.vector_store:
+            related = self.search_relevant_memories(current_message, limit=3)
+
+        # Get working memory (empty after restart)
+        history = self.working.get_history()
+        print(f"[Memory] Working memory items: {len(history)}")
+
+        # IMPORTANT: Always load from episodic if working memory is empty
+        # This ensures conversation context persists across restarts
+        if not history:
+            # Load more entries to match working memory capacity (14 items = 7 turns)
+            recent = self.episodic.load_recent(limit=10)
+            for e in recent:
+                # For proactive messages (empty user), only add assistant message
+                if e.get("user"):
+                    history.append({"role": "user", "content": e["user"]})
+                history.append({"role": "assistant", "content": e["ai"]})
+            if history:
+                print(f"[Memory] Loaded {len(history)} messages from episodic storage (restart recovery)")
+        else:
+            print(f"[Memory] Using working memory directly ({len(history)} items)")
+
+        context = {
+            "facts_context": "\n".join(facts_parts),
+            "conversation_history": history,
+            "related_memories": related,
+        }
+        return context, self.semantic.get_random_pet_name()