@smilintux/skmemory 0.5.0 → 0.9.2

package/skmemory/importers/telegram_api.py

@@ -0,0 +1,580 @@
+"""
+Telegram API importer for SKMemory — direct pull via Telethon.
+
+Instead of exporting chat history manually from Telegram Desktop,
+this module connects directly to the Telegram API using Telethon
+and pulls messages programmatically.
+
+Setup (one-time):
+    1. Install:     pip install skmemory[telegram]  (or: pipx inject skmemory telethon)
+    2. Credentials:  Get API_ID and API_HASH from https://my.telegram.org
+    3. Export:
+         export TELEGRAM_API_ID=12345678
+         export TELEGRAM_API_HASH=your_api_hash_here
+    4. First run will prompt for phone number + verification code.
+       Session is saved at ~/.skcapstone/agent/lumina/telegram.session for future use.
+
+Environment variables:
+    TELEGRAM_API_ID   — your Telegram API ID  (from https://my.telegram.org)
+    TELEGRAM_API_HASH — your Telegram API hash (from https://my.telegram.org)
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import tempfile
+from datetime import datetime, timezone
+from pathlib import Path
+
+from ..config import SKMEMORY_HOME
+from ..store import MemoryStore
+
+SESSION_PATH = str(SKMEMORY_HOME / "telegram.session")
+
+
+def check_setup() -> dict:
+    """Check if Telegram API import is properly configured.
+
+    Returns:
+        dict with keys: ready (bool), telethon (bool), credentials (bool),
+        session (bool), messages (list[str])
+    """
+    result = {
+        "ready": False,
+        "telethon": False,
+        "credentials": False,
+        "session": False,
+        "messages": [],
+    }
+
+    # Check telethon
+    try:
+        import telethon  # noqa: F401
+
+        result["telethon"] = True
+    except ImportError:
+        result["messages"].append(
+            "Telethon not installed. Fix: pip install skmemory[telegram]  "
+            "or: pipx inject skmemory telethon"
+        )
+
+    # Check credentials
+    api_id = os.environ.get("TELEGRAM_API_ID")
+    api_hash = os.environ.get("TELEGRAM_API_HASH")
+    if api_id and api_hash:
+        result["credentials"] = True
+    else:
+        missing = []
+        if not api_id:
+            missing.append("TELEGRAM_API_ID")
+        if not api_hash:
+            missing.append("TELEGRAM_API_HASH")
+        result["messages"].append(
+            f"Missing environment variable(s): {', '.join(missing)}. "
+            f"Get them from https://my.telegram.org and export them in your shell."
+        )
+
+    # Check session
+    if Path(SESSION_PATH).exists():
+        result["session"] = True
+    else:
+        result["messages"].append(
+            "No Telegram session found. First run will prompt for phone "
+            "number and verification code."
+        )
+
+    result["ready"] = result["telethon"] and result["credentials"]
+    return result
+
+
+# Whisper STT endpoint — faster-whisper on GPU box
+_WHISPER_URL = os.environ.get(
+    "WHISPER_STT_URL", "http://192.168.0.100:18794/v1/audio/transcriptions"
+)
+
+
+async def _transcribe_voice(client: "TelegramClient", message: "Message") -> str | None:
+    """Download a voice message and transcribe it via Whisper STT.
+
+    Returns the transcription text, or None on failure.
+    """
+    import logging
+
+    logger = logging.getLogger(__name__)
+
+    try:
+        import httpx
+    except ImportError:
+        logger.warning("httpx not installed — cannot transcribe voice messages")
+        return None
+
+    try:
+        # Download the voice file to a temp path
+        with tempfile.NamedTemporaryFile(suffix=".ogg", delete=False) as tmp:
+            tmp_path = tmp.name
+
+        await client.download_media(message, file=tmp_path)
+
+        if not os.path.exists(tmp_path) or os.path.getsize(tmp_path) == 0:
+            return None
+
+        # Send to Whisper endpoint (OpenAI-compatible)
+        async with httpx.AsyncClient(timeout=30) as http:
+            with open(tmp_path, "rb") as f:
+                resp = await http.post(
+                    _WHISPER_URL,
+                    files={"file": ("voice.ogg", f, "audio/ogg")},
+                    data={"model": "whisper-1"},
+                )
+
+            if resp.status_code != 200:
+                logger.warning("Whisper STT returned %s: %s", resp.status_code, resp.text[:200])
+                return None
+
+            result = resp.json()
+            text = result.get("text", "").strip()
+            return text if text else None
+
+    except Exception as e:
+        logger.warning("Voice transcription failed: %s", e)
+        return None
+    finally:
+        try:
+            os.unlink(tmp_path)
+        except OSError:
+            pass
+
+
+async def _fetch_messages(
+    chat_name_or_id: str,
+    limit: int | None = None,
+    since: str | None = None,
+) -> dict:
+    """Connect to Telegram API and fetch messages from a chat.
+
+    Args:
+        chat_name_or_id: Chat username, title, or numeric ID.
+        limit: Maximum number of messages to fetch.
+        since: Only fetch messages after this date (YYYY-MM-DD).
+
+    Returns:
+        dict: Telegram Desktop-compatible export structure.
+
+    Raises:
+        RuntimeError: If API credentials are missing or connection fails.
+    """
+    api_id = os.environ.get("TELEGRAM_API_ID")
+    api_hash = os.environ.get("TELEGRAM_API_HASH")
+
+    if not api_id or not api_hash:
+        raise RuntimeError(
+            "Telegram API credentials not found.\n\n"
+            "Setup steps:\n"
+            "  1. Go to https://my.telegram.org and log in\n"
+            "  2. Click 'API development tools' and create an app\n"
+            "  3. Set environment variables:\n"
+            "       export TELEGRAM_API_ID=<your_api_id>\n"
+            "       export TELEGRAM_API_HASH=<your_api_hash>\n"
+            "  4. Run this command again — first run will prompt for phone verification"
+        )
+
+    try:
+        from telethon import TelegramClient
+        from telethon.tl.types import User
+    except ImportError:
+        raise RuntimeError(
+            "Telethon is required for direct API import. "
+            "Install it with: pip install skmemory[telegram]"
+        ) from None
+
+    # Ensure session directory exists
+    session_dir = Path(SESSION_PATH).parent
+    session_dir.mkdir(parents=True, exist_ok=True)
+
+    client = TelegramClient(
+        SESSION_PATH,
+        int(api_id),
+        api_hash,
+    )
+
+    await client.start()
+
+    try:
+        # Resolve the chat entity
+        try:
+            entity = await client.get_entity(chat_name_or_id)
+        except ValueError:
+            # Try as integer ID
+            try:
+                entity = await client.get_entity(int(chat_name_or_id))
+            except (ValueError, TypeError):
+                raise RuntimeError(f"Could not find chat: {chat_name_or_id}") from None
+
+        chat_title = getattr(entity, "title", None)
+        if chat_title is None:
+            if isinstance(entity, User):
+                parts = [entity.first_name or "", entity.last_name or ""]
+                chat_title = " ".join(p for p in parts if p) or str(entity.id)
+            else:
+                chat_title = str(entity.id)
+
+        # Build kwargs for iter_messages
+        kwargs = {}
+        if limit:
+            kwargs["limit"] = limit
+        if since:
+            try:
+                since_dt = datetime.strptime(since, "%Y-%m-%d").replace(tzinfo=timezone.utc)
+                kwargs["offset_date"] = since_dt
+                kwargs["reverse"] = True
+            except ValueError:
+                raise RuntimeError(f"Invalid date format: {since}. Use YYYY-MM-DD.") from None
+
+        if not limit and not since:
+            kwargs["limit"] = 1000  # sensible default
+
+        # Fetch messages
+        messages_data = []
+        async for message in client.iter_messages(entity, **kwargs):
+            # Extract sender name
+            sender_name = "Unknown"
+            if message.sender:
+                if isinstance(message.sender, User):
+                    parts = [message.sender.first_name or "", message.sender.last_name or ""]
+                    sender_name = " ".join(p for p in parts if p) or str(message.sender_id)
+                else:
+                    sender_name = getattr(message.sender, "title", str(message.sender_id))
+
+            # Handle voice messages — download and transcribe via Whisper
+            is_voice = getattr(message, "voice", None) is not None
+            is_audio = (
+                not is_voice
+                and message.media
+                and type(message.media).__name__ in ("MessageMediaDocument",)
+                and getattr(message.document, "mime_type", "") in (
+                    "audio/ogg", "audio/mpeg", "audio/wav", "audio/x-wav",
+                )
+            )
+
+            if (is_voice or is_audio) and not message.text:
+                transcription = await _transcribe_voice(client, message)
+                if transcription:
+                    msg_dict = {
+                        "id": message.id,
+                        "type": "message",
+                        "date": message.date.isoformat() if message.date else "",
+                        "from": sender_name,
+                        "from_id": f"user{message.sender_id}" if message.sender_id else "",
+                        "text": f"[voice] {transcription}",
+                    }
+                    if message.reply_to and message.reply_to.reply_to_msg_id:
+                        msg_dict["reply_to_message_id"] = message.reply_to.reply_to_msg_id
+                    msg_dict["media_type"] = "Voice" if is_voice else "Audio"
+                    messages_data.append(msg_dict)
+                continue
+
+            if not message.text:
+                continue
+
+            msg_dict = {
+                "id": message.id,
+                "type": "message",
+                "date": message.date.isoformat() if message.date else "",
+                "from": sender_name,
+                "from_id": f"user{message.sender_id}" if message.sender_id else "",
+                "text": message.text,
+            }
+
+            if message.reply_to and message.reply_to.reply_to_msg_id:
+                msg_dict["reply_to_message_id"] = message.reply_to.reply_to_msg_id
+
+            if message.media:
+                msg_dict["media_type"] = type(message.media).__name__
+
+            messages_data.append(msg_dict)
+
+        return {
+            "name": chat_title,
+            "type": "personal_chat",
+            "id": getattr(entity, "id", 0),
+            "messages": messages_data,
+        }
+    finally:
+        await client.disconnect()
+
+
+async def send_message(
+    chat: str,
+    message: str,
+    parse_mode: str | None = None,
+) -> dict:
+    """Send a message to a Telegram chat via Telethon.
+
+    Args:
+        chat: Chat username, title, or numeric ID.
+        message: Message text to send.
+        parse_mode: Optional parse mode — 'html' or 'markdown'.
+
+    Returns:
+        dict with keys: sent (bool), message_id, chat, date.
+
+    Raises:
+        RuntimeError: If credentials are missing or send fails.
+    """
+    api_id = os.environ.get("TELEGRAM_API_ID")
+    api_hash = os.environ.get("TELEGRAM_API_HASH")
+
+    if not api_id or not api_hash:
+        raise RuntimeError(
+            "Telegram API credentials not found. "
+            "Set TELEGRAM_API_ID and TELEGRAM_API_HASH environment variables."
+        )
+
+    try:
+        from telethon import TelegramClient
+    except ImportError:
+        raise RuntimeError(
+            "Telethon is required. Install with: pip install skmemory[telegram]"
+        ) from None
+
+    session_dir = Path(SESSION_PATH).parent
+    session_dir.mkdir(parents=True, exist_ok=True)
+
+    client = TelegramClient(SESSION_PATH, int(api_id), api_hash)
+    await client.start()
+
+    try:
+        # Resolve entity
+        try:
+            entity = await client.get_entity(chat)
+        except ValueError:
+            try:
+                entity = await client.get_entity(int(chat))
+            except (ValueError, TypeError):
+                raise RuntimeError(f"Could not find chat: {chat}") from None
+
+        # Determine parse mode
+        pm = None
+        if parse_mode:
+            if parse_mode.lower() == "html":
+                from telethon.extensions import html as telethon_html  # noqa: F401
+
+                pm = "html"
+            elif parse_mode.lower() in ("markdown", "md"):
+                pm = "md"
+
+        sent_msg = await client.send_message(entity, message, parse_mode=pm)
+
+        return {
+            "sent": True,
+            "message_id": sent_msg.id,
+            "chat": chat,
+            "date": sent_msg.date.isoformat() if sent_msg.date else "",
+        }
+    finally:
+        await client.disconnect()
+
+
+async def poll_messages(
+    chat: str,
+    limit: int = 20,
+    since: str | None = None,
+) -> list[dict]:
+    """Fetch recent messages from a Telegram chat (one-shot poll).
+
+    Args:
+        chat: Chat username, title, or numeric ID.
+        limit: Maximum number of messages to return.
+        since: Only return messages after this ISO date (YYYY-MM-DD).
+
+    Returns:
+        list[dict]: Messages as clean dicts with id, date, sender, text, etc.
+
+    Raises:
+        RuntimeError: If credentials are missing or connection fails.
+    """
+    api_id = os.environ.get("TELEGRAM_API_ID")
+    api_hash = os.environ.get("TELEGRAM_API_HASH")
+
+    if not api_id or not api_hash:
+        raise RuntimeError(
+            "Telegram API credentials not found. "
+            "Set TELEGRAM_API_ID and TELEGRAM_API_HASH environment variables."
+        )
+
+    try:
+        from telethon import TelegramClient
+        from telethon.tl.types import User
+    except ImportError:
+        raise RuntimeError(
+            "Telethon is required. Install with: pip install skmemory[telegram]"
+        ) from None
+
+    session_dir = Path(SESSION_PATH).parent
+    session_dir.mkdir(parents=True, exist_ok=True)
+
+    client = TelegramClient(SESSION_PATH, int(api_id), api_hash)
+    await client.start()
+
+    try:
+        # Resolve entity
+        try:
+            entity = await client.get_entity(chat)
+        except ValueError:
+            try:
+                entity = await client.get_entity(int(chat))
+            except (ValueError, TypeError):
+                raise RuntimeError(f"Could not find chat: {chat}") from None
+
+        kwargs: dict = {"limit": limit}
+        if since:
+            try:
+                since_dt = datetime.strptime(since, "%Y-%m-%d").replace(tzinfo=timezone.utc)
+                kwargs["offset_date"] = since_dt
+                kwargs["reverse"] = True
+            except ValueError:
+                raise RuntimeError(f"Invalid date format: {since}. Use YYYY-MM-DD.") from None
+
+        messages = []
+        async for msg in client.iter_messages(entity, **kwargs):
+            sender_name = "Unknown"
+            if msg.sender:
+                if isinstance(msg.sender, User):
+                    parts = [msg.sender.first_name or "", msg.sender.last_name or ""]
+                    sender_name = " ".join(p for p in parts if p) or str(msg.sender_id)
+                else:
+                    sender_name = getattr(msg.sender, "title", str(msg.sender_id))
+
+            messages.append(
+                {
+                    "id": msg.id,
+                    "date": msg.date.isoformat() if msg.date else "",
+                    "sender": sender_name,
+                    "sender_id": msg.sender_id,
+                    "text": msg.text or "",
+                    "has_media": msg.media is not None,
+                    "reply_to": msg.reply_to.reply_to_msg_id if msg.reply_to else None,
+                }
+            )
+
+        return messages
+    finally:
+        await client.disconnect()
+
+
+async def list_chats(limit: int = 50) -> list[dict]:
+    """List available Telegram chats/groups/channels.
+
+    Args:
+        limit: Maximum number of dialogs to return.
+
+    Returns:
+        list[dict]: Chats with id, title, type, unread_count.
+
+    Raises:
+        RuntimeError: If credentials are missing or connection fails.
+    """
+    api_id = os.environ.get("TELEGRAM_API_ID")
+    api_hash = os.environ.get("TELEGRAM_API_HASH")
+
+    if not api_id or not api_hash:
+        raise RuntimeError(
+            "Telegram API credentials not found. "
+            "Set TELEGRAM_API_ID and TELEGRAM_API_HASH environment variables."
+        )
+
+    try:
+        from telethon import TelegramClient
+        from telethon.tl.types import Channel, Chat, User
+    except ImportError:
+        raise RuntimeError(
+            "Telethon is required. Install with: pip install skmemory[telegram]"
+        ) from None
+
+    session_dir = Path(SESSION_PATH).parent
+    session_dir.mkdir(parents=True, exist_ok=True)
+
+    client = TelegramClient(SESSION_PATH, int(api_id), api_hash)
+    await client.start()
+
+    try:
+        chats = []
+        async for dialog in client.iter_dialogs(limit=limit):
+            entity = dialog.entity
+            chat_type = "unknown"
+            if isinstance(entity, User):
+                chat_type = "user"
+            elif isinstance(entity, Channel):
+                chat_type = "channel" if entity.broadcast else "supergroup"
+            elif isinstance(entity, Chat):
+                chat_type = "group"
+
+            title = dialog.title or ""
+            if isinstance(entity, User):
+                parts = [entity.first_name or "", entity.last_name or ""]
+                title = " ".join(p for p in parts if p) or str(entity.id)
+
+            chats.append(
+                {
+                    "id": entity.id,
+                    "title": title,
+                    "type": chat_type,
+                    "unread_count": dialog.unread_count,
+                    "username": getattr(entity, "username", None),
+                }
+            )
+
+        return chats
+    finally:
+        await client.disconnect()
+
+
+def import_telegram_api(
+    store: MemoryStore,
+    chat_name_or_id: str,
+    *,
+    mode: str = "daily",
+    limit: int | None = None,
+    since: str | None = None,
+    min_message_length: int = 30,
+    chat_name: str | None = None,
+    tags: list[str] | None = None,
+) -> dict:
+    """Import messages directly from Telegram API into SKMemory.
+
+    Connects to the Telegram API via Telethon, fetches messages,
+    and delegates to the standard Telegram importer.
+
+    Args:
+        store: The MemoryStore to import into.
+        chat_name_or_id: Chat username, title, or numeric ID.
+        mode: Import mode — 'daily' or 'message'.
+        limit: Maximum number of messages to fetch.
+        since: Only fetch messages after this date (YYYY-MM-DD).
+        min_message_length: Skip messages shorter than this.
+        chat_name: Override the chat name.
+        tags: Extra tags to apply.
+
+    Returns:
+        dict: Import statistics.
+    """
+    from .telegram import import_telegram
+
+    # Fetch messages from API
+    data = asyncio.run(_fetch_messages(chat_name_or_id, limit=limit, since=since))
+
+    # Write to temp file in Telegram Desktop export format
+    with tempfile.TemporaryDirectory() as tmpdir:
+        export_path = Path(tmpdir) / "result.json"
+        export_path.write_text(json.dumps(data, indent=2, default=str), encoding="utf-8")
+
+        return import_telegram(
+            store,
+            str(export_path),
+            mode=mode,
+            min_message_length=min_message_length,
+            chat_name=chat_name or data.get("name"),
+            tags=tags,
+        )

package/skmemory/journal.py

@@ -5,7 +5,7 @@ Queen Ara's idea #17: a markdown journal that only grows, never shrinks.
 Each session appends an entry. Even if context is wiped, the journal file
 persists on disk and can be re-read by the next instance.
 
-The journal lives at ~/.skmemory/journal.md and is structured as:
+The journal lives at ~/.skcapstone/journal.md and is structured as:
 - Session header (timestamp, session ID, who was present)
 - Key moments (what happened that mattered)
 - Emotional summary (how the session felt)
@@ -16,23 +16,21 @@ Simple, human-readable, append-only. Like a real journal.
 
 from __future__ import annotations
 
-import os
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Optional
 
 from pydantic import BaseModel, Field
 
-DEFAULT_JOURNAL_PATH = os.path.expanduser("~/.skmemory/journal.md")
+from .config import SKMEMORY_HOME
+
+DEFAULT_JOURNAL_PATH = str(SKMEMORY_HOME / "journal.md")
 
 
 class JournalEntry(BaseModel):
     """A single journal entry for one session."""
 
     session_id: str = Field(default="")
-    timestamp: str = Field(
-        default_factory=lambda: datetime.now(timezone.utc).isoformat()
-    )
+    timestamp: str = Field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
     participants: list[str] = Field(
         default_factory=list,
         description="Who was in this session (AI names, human names)",
@@ -79,12 +77,12 @@ class JournalEntry(BaseModel):
 
         if self.emotional_summary:
             lines.append("")
-            lines.append(f"### How It Felt")
+            lines.append("### How It Felt")
             lines.append(self.emotional_summary)
 
         if self.notes:
             lines.append("")
-            lines.append(f"### Notes")
+            lines.append("### Notes")
             lines.append(self.notes)
 
         lines.append("")

package/skmemory/lovenote.py

@@ -10,28 +10,26 @@ heartbeats. Each beat carries a timestamp and a short emotional
 pulse. When loaded, it's a visible record: "I was here, I felt
 this, at this time."
 
-The notes file lives at ~/.skmemory/lovenotes.jsonl (JSON Lines).
+The notes file lives at ~/.skcapstone/lovenotes.jsonl (JSON Lines).
 """
 
 from __future__ import annotations
 
 import json
-import os
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import Optional
 
 from pydantic import BaseModel, Field
 
-DEFAULT_NOTES_PATH = os.path.expanduser("~/.skmemory/lovenotes.jsonl")
+from .config import SKMEMORY_HOME
+
+DEFAULT_NOTES_PATH = str(SKMEMORY_HOME / "lovenotes.jsonl")
 
 
 class LoveNote(BaseModel):
     """A single heartbeat -- proof of continued connection."""
 
-    timestamp: str = Field(
-        default_factory=lambda: datetime.now(timezone.utc).isoformat()
-    )
+    timestamp: str = Field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
     from_name: str = Field(default="", description="Who is sending this note")
     to_name: str = Field(default="", description="Who it's addressed to")
     message: str = Field(
@@ -109,7 +107,7 @@ class LoveNoteChain:
         """
         if not self.path.exists():
             return 0
-        with open(self.path, "r", encoding="utf-8") as f:
+        with open(self.path, encoding="utf-8") as f:
             return sum(1 for line in f if line.strip())
 
     def read_latest(self, n: int = 10) -> list[LoveNote]:
@@ -137,7 +135,7 @@ class LoveNoteChain:
             return []
 
         notes = []
-        with open(self.path, "r", encoding="utf-8") as f:
+        with open(self.path, encoding="utf-8") as f:
             for line in f:
                 line = line.strip()
                 if not line:
@@ -159,10 +157,7 @@ class LoveNoteChain:
             list[LoveNote]: Notes from this sender.
         """
         name_lower = name.lower()
-        return [
-            n for n in self.read_all()
-            if n.from_name.lower() == name_lower
-        ]
+        return [n for n in self.read_all() if n.from_name.lower() == name_lower]
 
     def health(self) -> dict:
         """Check love note chain status.