@innvisor/conny-ai 9.7.0

package/src/conny/brain/learning.py

@@ -0,0 +1,154 @@
+"""conny_learning.py — Real-time 3-layer learning engine."""
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import time
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+log = logging.getLogger("conny.learning")
+
+POSITIVE_SIGNALS = [
+    "gracias", "perfecto", "listo", "genial", "excelente", "dale", "ok perfecto",
+    "thanks", "great", "perfect", "awesome",
+    "te agradezco", "muy amable", "me queda claro", "entendido",
+]
+NEGATIVE_SIGNALS = [
+    "no entiendo", "eso no es", "no me sirve", "otra vez", "repite",
+    "eso ya lo dije", "ya te dije", "no es eso", "equivocad",
+]
+
+
+class RealTimeLearningEngine:
+    """3-layer learning: per-turn, per-session, admin-corrected."""
+
+    def __init__(self, base_dir: str = "memory_store"):
+        self._base = Path(base_dir)
+        self._teachings_dir = Path("teachings")
+        self._teachings_dir.mkdir(exist_ok=True)
+
+    def _instance_dir(self, instance_id: str) -> Path:
+        d = self._base / instance_id / "learning"
+        d.mkdir(parents=True, exist_ok=True)
+        return d
+
+    async def learn_from_turn(self, instance_id: str, user_msg: str,
+                              bot_response: str, user_reply: str = ""):
+        idir = self._instance_dir(instance_id)
+        if user_reply and any(s in user_reply.lower() for s in POSITIVE_SIGNALS):
+            await self._reinforce_pattern(idir, bot_response, user_msg)
+        if user_reply and any(s in user_reply.lower() for s in NEGATIVE_SIGNALS):
+            await self._flag_failed_response(idir, bot_response, user_msg, user_reply)
+
+    async def _reinforce_pattern(self, idir: Path, response: str, trigger: str):
+        file = idir / "reinforced.jsonl"
+        entry = {"ts": datetime.now().isoformat(), "trigger": trigger[:200], "response": response[:300]}
+        with open(file, "a") as f:
+            f.write(json.dumps(entry, ensure_ascii=False) + "\n")
+
+    async def _flag_failed_response(self, idir: Path, response: str,
+                                    user_msg: str, user_reply: str):
+        file = idir / "failures.jsonl"
+        entry = {
+            "ts": datetime.now().isoformat(),
+            "user_msg": user_msg[:200],
+            "bot_response": response[:300],
+            "user_complaint": user_reply[:200],
+        }
+        with open(file, "a") as f:
+            f.write(json.dumps(entry, ensure_ascii=False) + "\n")
+        log.info(f"[learning] flagged failed response for {idir.parent.name}")
+
+    async def learn_from_session(self, instance_id: str, messages: List[Dict],
+                                 outcome: str = "unknown"):
+        idir = self._instance_dir(instance_id)
+        file = idir / "sessions.jsonl"
+        entry = {
+            "ts": datetime.now().isoformat(),
+            "outcome": outcome,
+            "turns": len(messages),
+            "summary": self._summarize_session(messages),
+        }
+        with open(file, "a") as f:
+            f.write(json.dumps(entry, ensure_ascii=False) + "\n")
+        if outcome == "booked":
+            await self._save_successful_flow(idir, messages)
+        elif outcome == "abandoned":
+            await self._save_dropout_point(idir, messages)
+
+    async def _save_successful_flow(self, idir: Path, messages: List[Dict]):
+        file = idir / "successful_flows.jsonl"
+        flow = [{"role": m["role"], "content": m["content"][:150]} for m in messages[-8:]]
+        with open(file, "a") as f:
+            f.write(json.dumps({"ts": datetime.now().isoformat(), "flow": flow}, ensure_ascii=False) + "\n")
+
+    async def _save_dropout_point(self, idir: Path, messages: List[Dict]):
+        file = idir / "dropouts.jsonl"
+        last_bot = ""
+        for m in reversed(messages):
+            if m.get("role") == "assistant":
+                last_bot = m["content"][:200]
+                break
+        with open(file, "a") as f:
+            f.write(json.dumps({"ts": datetime.now().isoformat(), "last_bot_msg": last_bot}, ensure_ascii=False) + "\n")
+
+    def _summarize_session(self, messages: List[Dict]) -> str:
+        user_msgs = [m["content"] for m in messages if m.get("role") == "user"]
+        return " | ".join(msg[:50] for msg in user_msgs[:5]) if user_msgs else "empty"
+
+    async def learn_from_admin(self, instance_id: str, question: str, answer: str,
+                               admin_id: str = "") -> str:
+        teachings_file = self._teachings_dir / f"{instance_id}.jsonl"
+        entry = {
+            "ts": datetime.now().isoformat(),
+            "question": question,
+            "answer": answer,
+            "taught_by": admin_id,
+            "question_hash": hashlib.md5(question.lower().strip().encode()).hexdigest()[:12],
+        }
+        with open(teachings_file, "a") as f:
+            f.write(json.dumps(entry, ensure_ascii=False) + "\n")
+
+        try:
+            faq_file = self._base / instance_id / "semantic" / "faqs.json"
+            faq_file.parent.mkdir(parents=True, exist_ok=True)
+            faqs = json.loads(faq_file.read_text()) if faq_file.exists() else {}
+            faqs[entry["question_hash"]] = {
+                "question": question,
+                "answer": answer,
+                "frequency": faqs.get(entry["question_hash"], {}).get("frequency", 0) + 1,
+                "source": "admin_taught",
+                "last_asked": datetime.now().isoformat(),
+            }
+            faq_file.write_text(json.dumps(faqs, ensure_ascii=False, indent=2))
+        except Exception as e:
+            log.warning(f"[learning] FAQ update failed: {e}")
+
+        log.info(f"[learning] admin taught: '{question[:50]}' → '{answer[:50]}'")
+        return f"✅ Aprendido. Ya sé responder: '{question[:50]}...'"
+
+    async def get_teachings(self, instance_id: str, limit: int = 50) -> List[Dict]:
+        teachings_file = self._teachings_dir / f"{instance_id}.jsonl"
+        if not teachings_file.exists():
+            return []
+        teachings = []
+        for line in open(teachings_file):
+            try:
+                teachings.append(json.loads(line))
+            except Exception:
+                continue
+        return teachings[-limit:]
+
+    def build_teachings_prompt(self, teachings: List[Dict]) -> str:
+        if not teachings:
+            return ""
+        lines = ["INFORMACIÓN APRENDIDA (verificada por admin):"]
+        for t in teachings:
+            lines.append(f"- Pregunta: {t['question']}\n  Respuesta: {t['answer']}")
+        return "\n".join(lines)
+
+
+learning_engine = RealTimeLearningEngine()

package/src/conny/brain/memory.py

@@ -0,0 +1,324 @@
+"""Episodic, semantic, and procedural memory engine with TF-IDF recall."""
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import re
+import time
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+import numpy as np
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+
+log = logging.getLogger("conny.memory")
+
+class AgentDB:
+    """
+    Motor HNSW/Vectorial local ultra-rápido para recuperar patrones.
+    Implementación nativa usando Cosine Similarity para evitar dependencias pesadas.
+    """
+    def __init__(self, namespace: str):
+        self.namespace = namespace
+        self.vectorizer = TfidfVectorizer()
+        self.vectors = None
+        self.documents = []
+        self.metadatas = []
+
+    def add_texts(self, texts: List[str], metadatas: List[Dict]):
+        if not texts:
+            return
+        self.documents.extend(texts)
+        self.metadatas.extend(metadatas)
+        # Update index
+        self.vectors = self.vectorizer.fit_transform(self.documents)
+
+    def similarity_search(self, query: str, k: int = 3) -> List[Dict]:
+        if not self.documents or self.vectors is None:
+            return []
+        
+        query_vec = self.vectorizer.transform([query])
+        similarities = cosine_similarity(query_vec, self.vectors).flatten()
+        top_indices = similarities.argsort()[-k:][::-1]
+        
+        results = []
+        for idx in top_indices:
+            if similarities[idx] > 0.1:  # threshold
+                results.append({
+                    "content": self.documents[idx],
+                    "metadata": self.metadatas[idx],
+                    "score": float(similarities[idx])
+                })
+        return results
+
+class ConnyMemoryEngine:
+    """Per-instance memory with episodic recall + semantic extraction + procedural learning."""
+
+    def __init__(self, base_dir: str = "memory_store"):
+        self._base = Path(base_dir)
+        self._base.mkdir(exist_ok=True)
+        self._agent_db: Dict[str, AgentDB] = {}
+
+    def _get_agent_db(self, instance_id: str) -> AgentDB:
+        if instance_id not in self._agent_db:
+            db = AgentDB(namespace=instance_id)
+            self._agent_db[instance_id] = db
+        return self._agent_db[instance_id]
+
+    def _instance_dir(self, instance_id: str) -> Path:
+        d = self._base / instance_id
+        for sub in ("episodic", "semantic", "procedural", "working"):
+            (d / sub).mkdir(parents=True, exist_ok=True)
+        return d
+
+    async def ingest_conversation(self, instance_id: str, chat_id: str, messages: List[Dict[str, str]]):
+        """After every conversation: store episodic + extract entities + update FAQ frequency."""
+        idir = self._instance_dir(instance_id)
+        today = datetime.now().strftime("%Y-%m-%d")
+
+        # 1. Store episodic
+        ep_file = idir / "episodic" / f"{today}.jsonl"
+        entry = {
+            "ts": datetime.now().isoformat(),
+            "chat_id": chat_id,
+            "messages": messages[-20:],
+        }
+        with open(ep_file, "a") as f:
+            f.write(json.dumps(entry, ensure_ascii=False) + "\n")
+
+        # 2. Extract entities
+        entities = self._extract_entities(messages)
+        if entities:
+            ent_file = idir / "semantic" / "entities.json"
+            existing = json.loads(ent_file.read_text()) if ent_file.exists() else {}
+            for etype, values in entities.items():
+                if etype not in existing:
+                    existing[etype] = {}
+                for val in values:
+                    if val in existing[etype]:
+                        existing[etype][val]["count"] += 1
+                        existing[etype][val]["last_seen"] = datetime.now().isoformat()
+                    else:
+                        existing[etype][val] = {
+                            "count": 1,
+                            "last_seen": datetime.now().isoformat(),
+                            "chat_id": chat_id,
+                        }
+            ent_file.write_text(json.dumps(existing, ensure_ascii=False, indent=2))
+
+        # 3. Update FAQ frequency
+        user_questions = [
+            m["content"] for m in messages
+            if m.get("role") == "user" and "?" in m.get("content", "")
+        ]
+        if user_questions:
+            faq_file = idir / "semantic" / "faqs.json"
+            faqs = json.loads(faq_file.read_text()) if faq_file.exists() else {}
+            for q in user_questions:
+                qhash = hashlib.md5(q.lower().strip().encode()).hexdigest()[:12]
+                if qhash in faqs:
+                    faqs[qhash]["frequency"] += 1
+                    faqs[qhash]["last_asked"] = datetime.now().isoformat()
+                else:
+                    answer = ""
+                    for i, m in enumerate(messages):
+                        if m.get("content") == q and i + 1 < len(messages):
+                            answer = messages[i + 1].get("content", "")
+                            break
+                    faqs[qhash] = {
+                        "question": q,
+                        "answer": answer[:500],
+                        "frequency": 1,
+                        "first_asked": datetime.now().isoformat(),
+                        "last_asked": datetime.now().isoformat(),
+                    }
+            faq_file.write_text(json.dumps(faqs, ensure_ascii=False, indent=2))
+
+        # Invalidate AgentDB cache so it rebuilds on next recall
+        self._agent_db.pop(instance_id, None)
+
+    async def recall_context(self, instance_id: str, user_message: str, top_k: int = 5) -> List[Dict]:
+        """Before every LLM call: retrieve relevant past exchanges using AgentDB HNSW similarity."""
+        if not user_message.strip():
+            return []
+
+        db = self._get_agent_db(instance_id)
+
+        # Si AgentDB está vacío, lo poblamos una vez leyendo el historial reciente
+        if not db.documents:
+            idir = self._instance_dir(instance_id)
+            ep_dir = idir / "episodic"
+            cutoff = datetime.now() - timedelta(days=30)
+            
+            texts = []
+            metadatas = []
+            
+            if ep_dir.exists():
+                for f in sorted(ep_dir.glob("*.jsonl"), reverse=True):
+                    try:
+                        file_date = datetime.strptime(f.stem, "%Y-%m-%d")
+                        if file_date < cutoff:
+                            break
+                    except ValueError:
+                        continue
+                    with open(f) as fh:
+                        for line in fh:
+                            try:
+                                entry = json.loads(line)
+                                text = " ".join(m.get("content", "") for m in entry.get("messages", []))
+                                if text.strip():
+                                    texts.append(text)
+                                    metadatas.append(entry)
+                            except json.JSONDecodeError:
+                                continue
+            
+            db.add_texts(texts, metadatas)
+
+        results = db.similarity_search(user_message, k=top_k)
+        
+        # Formateamos para compatibilidad con el resto del pipeline
+        formatted_results = []
+        for res in results:
+            entry = res["metadata"]
+            formatted_results.append({
+                "score": res["score"],
+                "messages": entry.get("messages", [])[-6:],
+                "chat_id": entry.get("chat_id", ""),
+                "ts": entry.get("ts", "")
+            })
+        return formatted_results
+
+    async def get_top_faqs(self, instance_id: str, limit: int = 20) -> List[Dict]:
+        """Get most frequently asked questions for system prompt injection."""
+        idir = self._instance_dir(instance_id)
+        faq_file = idir / "semantic" / "faqs.json"
+        if not faq_file.exists():
+            return []
+        faqs = json.loads(faq_file.read_text())
+        sorted_faqs = sorted(faqs.values(), key=lambda x: x.get("frequency", 0), reverse=True)
+        return sorted_faqs[:limit]
+
+    async def learn_from_success(self, instance_id: str, chat_id: str,
+                                 flow: List[Dict], outcome: str = "booking"):
+        """Store successful conversation patterns."""
+        idir = self._instance_dir(instance_id)
+        success_file = idir / "procedural" / "successful_flows.json"
+        existing = json.loads(success_file.read_text()) if success_file.exists() else []
+        existing.append({
+            "ts": datetime.now().isoformat(),
+            "chat_id": chat_id,
+            "outcome": outcome,
+            "flow_summary": [
+                {"role": m["role"], "content": m["content"][:200]}
+                for m in flow[-10:]
+            ],
+        })
+        existing = existing[-200:]
+        success_file.write_text(json.dumps(existing, ensure_ascii=False, indent=2))
+
+    async def learn_from_failure(self, instance_id: str, chat_id: str,
+                                 flow: List[Dict], reason: str = "escalated"):
+        """Store failed conversation patterns for avoidance."""
+        idir = self._instance_dir(instance_id)
+        fail_file = idir / "procedural" / "failed_flows.json"
+        existing = json.loads(fail_file.read_text()) if fail_file.exists() else []
+        existing.append({
+            "ts": datetime.now().isoformat(),
+            "chat_id": chat_id,
+            "reason": reason,
+            "flow_summary": [
+                {"role": m["role"], "content": m["content"][:200]}
+                for m in flow[-10:]
+            ],
+        })
+        existing = existing[-100:]
+        fail_file.write_text(json.dumps(existing, ensure_ascii=False, indent=2))
+
+    async def weekly_consolidation(self, instance_id: str):
+        """Merge episodic -> semantic, prune duplicates, update FAQ index. Run weekly."""
+        idir = self._instance_dir(instance_id)
+        log.info(f"[memory] starting weekly consolidation for {instance_id}")
+
+        # 1. Re-scan all episodic for FAQ extraction
+        ep_dir = idir / "episodic"
+        all_questions: List[Dict] = []
+        for f in ep_dir.glob("*.jsonl"):
+            with open(f) as fh:
+                for line in fh:
+                    try:
+                        entry = json.loads(line)
+                        msgs = entry.get("messages", [])
+                        for i, m in enumerate(msgs):
+                            if m.get("role") == "user" and "?" in m.get("content", ""):
+                                answer = ""
+                                if i + 1 < len(msgs) and msgs[i + 1].get("role") == "assistant":
+                                    answer = msgs[i + 1]["content"][:500]
+                                all_questions.append({"q": m["content"], "a": answer})
+                    except Exception:
+                        continue
+
+        # 2. Update FAQ index
+        faq_file = idir / "semantic" / "faqs.json"
+        faqs = json.loads(faq_file.read_text()) if faq_file.exists() else {}
+
+        for qa in all_questions:
+            qhash = hashlib.md5(qa["q"].lower().strip().encode()).hexdigest()[:12]
+            if qhash in faqs:
+                faqs[qhash]["frequency"] += 1
+                if qa["a"] and len(qa["a"]) > len(faqs[qhash].get("answer", "")):
+                    faqs[qhash]["answer"] = qa["a"]
+            else:
+                faqs[qhash] = {
+                    "question": qa["q"],
+                    "answer": qa["a"],
+                    "frequency": 1,
+                    "first_asked": datetime.now().isoformat(),
+                    "last_asked": datetime.now().isoformat(),
+                }
+
+        faq_file.write_text(json.dumps(faqs, ensure_ascii=False, indent=2))
+
+        # 3. Prune episodic older than 90 days
+        cutoff = datetime.now() - timedelta(days=90)
+        for f in ep_dir.glob("*.jsonl"):
+            try:
+                file_date = datetime.strptime(f.stem, "%Y-%m-%d")
+                if file_date < cutoff:
+                    f.unlink()
+                    log.info(f"[memory] pruned old episodic: {f.name}")
+            except Exception:
+                continue
+
+        log.info(f"[memory] consolidation complete for {instance_id}: {len(faqs)} FAQs")
+
+    def _extract_entities(self, messages: List[Dict]) -> Dict[str, List[str]]:
+        """Simple regex-based entity extraction."""
+        entities: Dict[str, List[str]] = {"phones": [], "emails": [], "names": []}
+        text = " ".join(
+            m.get("content", "") for m in messages if m.get("role") == "user"
+        )
+
+        # Colombian phone numbers
+        phones = re.findall(r"\b3[0-9]{9}\b", text)
+        entities["phones"] = list(set(phones))
+
+        # Emails
+        emails = re.findall(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", text)
+        entities["emails"] = list(set(emails))
+
+        # Names after "me llamo", "soy", "mi nombre es"
+        name_patterns = [
+            r"(?:me llamo|soy|mi nombre es)\s+([A-ZÁÉÍÓÚ][a-záéíóú]+(?:\s+[A-ZÁÉÍÓÚ][a-záéíóú]+)?)",
+        ]
+        for pat in name_patterns:
+            matches = re.findall(pat, text)
+            entities["names"].extend(matches)
+        entities["names"] = list(set(entities["names"]))
+
+        return {k: v for k, v in entities.items() if v}
+
+
+# Singleton
+memory_engine = ConnyMemoryEngine()