@trac3er/oh-my-god 2.0.4 → 2.0.7

package/build/lib/runtime/memory_parsers/chatgpt_parser.py

@@ -0,0 +1,257 @@
+"""ChatGPT conversations.json parser for OMG shared memory import.
+
+Parses the ChatGPT data export format (``conversations.json``) and converts
+conversations into OMG memory items compatible with :class:`MemoryStore`.
+
+Functions:
+    extract_linear_conversation — traverse mapping tree to ordered message list
+    parse_conversations_file   — read + parse the export JSON file
+    conversations_to_memory_items — convert parsed conversations to memory items
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import uuid
+import warnings
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+# Maximum traversal depth to guard against circular parent references.
+_MAX_DEPTH = 10_000
+
+# Maximum content length per memory item (chars).
+_MAX_CONTENT_LENGTH = 5_000
+
+
+# ---------------------------------------------------------------------------
+# extract_linear_conversation
+# ---------------------------------------------------------------------------
+
+
+def extract_linear_conversation(
+    mapping: dict[str, Any],
+    current_node: str,
+) -> list[dict[str, Any]]:
+    """Walk *current_node* back to root via ``parent`` pointers.
+
+    Returns a chronologically ordered list (root -> leaf) of message dicts::
+
+        {"role": str, "content": str, "timestamp": float | None}
+
+    * Skips nodes whose ``message`` is ``None`` or has empty text.
+    * Skips ``system`` role messages.
+    * Guards against circular references with a depth limit of 10 000.
+    """
+    if current_node not in mapping:
+        return []
+
+    # Collect nodes from current_node back to root.
+    chain: list[dict[str, Any]] = []
+    visited: set[str] = set()
+    node_id: str | None = current_node
+
+    depth = 0
+    while node_id is not None and depth < _MAX_DEPTH:
+        if node_id in visited:
+            break
+        visited.add(node_id)
+
+        node = mapping.get(node_id)
+        if node is None:
+            break
+
+        chain.append(node)
+        node_id = node.get("parent")
+        depth += 1
+
+    # Reverse so root comes first (chronological order).
+    chain.reverse()
+
+    messages: list[dict[str, Any]] = []
+    for node in chain:
+        msg = node.get("message")
+        if msg is None:
+            continue
+
+        author = msg.get("author") or {}
+        role = author.get("role", "")
+
+        # Skip system messages.
+        if role == "system":
+            continue
+
+        content_obj = msg.get("content") or {}
+        parts = content_obj.get("parts") or []
+        text = "".join(str(p) for p in parts).strip()
+
+        # Skip empty content.
+        if not text:
+            continue
+
+        timestamp = msg.get("create_time")
+
+        messages.append(
+            {
+                "role": role,
+                "content": text,
+                "timestamp": timestamp,
+            }
+        )
+
+    return messages
+
+
+# ---------------------------------------------------------------------------
+# parse_conversations_file
+# ---------------------------------------------------------------------------
+
+
+def parse_conversations_file(file_path: str) -> list[dict[str, Any]]:
+    """Read and parse a ChatGPT ``conversations.json`` export file.
+
+    For each conversation object, calls :func:`extract_linear_conversation`
+    to obtain an ordered message list.
+
+    Returns a list of conversation dicts::
+
+        {
+            "id": str,
+            "title": str,
+            "messages": list[dict],
+            "create_time": float | None,
+            "source": "chatgpt",
+        }
+
+    Gracefully handles:
+    * File not found -> empty list
+    * Invalid JSON -> empty list
+    * Malformed individual conversations -> logged warning, skipped
+    """
+    path = Path(file_path)
+    if not path.exists():
+        logger.warning("Conversations file not found: %s", file_path)
+        return []
+
+    try:
+        raw_text = path.read_text(encoding="utf-8")
+    except OSError as exc:
+        logger.warning("Failed to read conversations file: %s", exc)
+        return []
+
+    # Warn about potentially large files.
+    file_size = path.stat().st_size
+    if file_size > 50 * 1024 * 1024:  # 50 MB
+        warnings.warn(
+            f"Large conversations file ({file_size / 1024 / 1024:.1f} MB). "
+            "Consider using streaming JSON parser (ijson) for better memory usage.",
+            ResourceWarning,
+            stacklevel=2,
+        )
+
+    try:
+        data = json.loads(raw_text)
+    except json.JSONDecodeError as exc:
+        logger.warning("Invalid JSON in conversations file: %s", exc)
+        return []
+
+    if not isinstance(data, list):
+        logger.warning("Expected JSON array, got %s", type(data).__name__)
+        return []
+
+    results: list[dict[str, Any]] = []
+
+    for conv in data:
+        try:
+            conv_id = conv["id"]
+            title = conv.get("title", "Untitled")
+            mapping = conv["mapping"]
+            current_node = conv["current_node"]
+            create_time = conv.get("create_time")
+
+            messages = extract_linear_conversation(mapping, current_node)
+
+            results.append(
+                {
+                    "id": conv_id,
+                    "title": title,
+                    "messages": messages,
+                    "create_time": create_time,
+                    "source": "chatgpt",
+                }
+            )
+        except (KeyError, TypeError) as exc:
+            conv_id_str = conv.get("id", "<unknown>") if isinstance(conv, dict) else "<invalid>"
+            logger.warning(
+                "Skipping malformed conversation %s: %s", conv_id_str, exc
+            )
+
+    return results
+
+
+# ---------------------------------------------------------------------------
+# conversations_to_memory_items
+# ---------------------------------------------------------------------------
+
+
+def conversations_to_memory_items(
+    conversations: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+    """Convert parsed conversations to OMG memory item format.
+
+    Each conversation becomes ONE memory item with::
+
+        {
+            "key": "chatgpt-{id[:8]}",
+            "content": "# {title}\\n\\n**role**: content\\n\\n...",
+            "source_cli": "chatgpt",
+            "tags": ["chatgpt", "imported"],
+        }
+
+    * Conversations with no messages are skipped.
+    * Content is truncated to 5 000 chars max.
+    """
+    items: list[dict[str, Any]] = []
+
+    for conv in conversations:
+        messages = conv.get("messages", [])
+        if not messages:
+            continue
+
+        conv_id = conv.get("id", "unknown")
+        title = conv.get("title", "Untitled")
+
+        # Build content string.
+        parts = [f"# {title}", ""]
+        for msg in messages:
+            role = msg.get("role", "unknown")
+            content = msg.get("content", "")
+            parts.append(f"**{role}**: {content}")
+
+        full_content = "\n\n".join(parts)
+
+        # Truncate to max length.
+        if len(full_content) > _MAX_CONTENT_LENGTH:
+            full_content = full_content[:_MAX_CONTENT_LENGTH]
+
+        items.append(
+            {
+                "id": str(uuid.uuid4()),
+                "key": f"chatgpt-{conv_id[:8]}",
+                "content": full_content,
+                "source_cli": "chatgpt",
+                "tags": ["chatgpt", "imported"],
+            }
+        )
+
+    return items
+
+
+__all__ = [
+    "extract_linear_conversation",
+    "parse_conversations_file",
+    "conversations_to_memory_items",
+]

package/build/lib/runtime/memory_parsers/claude_import.py

@@ -0,0 +1,107 @@
+"""Claude.ai paste-based memory import."""
+
+import re
+from typing import TYPE_CHECKING, TypedDict
+
+if TYPE_CHECKING:
+    from runtime.memory_store import MemoryStore
+
+
+class MemoryItem(TypedDict):
+    """Memory item structure."""
+
+    key: str
+    content: str
+    source_cli: str
+    tags: list[str]
+
+
+EXTRACTION_PROMPT = (
+    "List every memory you have stored about me. "
+    "Format each memory as a bullet point starting with '- '. "
+    "Include all preferences, facts, and context you remember."
+)
+
+
+def parse_claude_paste(text: str) -> list[MemoryItem]:
+    """Parse Claude.ai paste-based memory list into structured items.
+
+    Handles multiple formats:
+    - Bullet points: "- item", "* item", "• item"
+    - Numbered lists: "1. item", "2. item"
+    - Plain paragraphs: one per line
+
+    Args:
+        text: Freeform text pasted from Claude.ai memory list
+
+    Returns:
+        List of memory item dicts with keys: key, content, source_cli, tags
+    """
+    if not text or not text.strip():
+        return []
+
+    lines = text.split("\n")
+    items: list[MemoryItem] = []
+    item_index = 1
+
+    for line in lines:
+        stripped = line.strip()
+
+        if not stripped:
+            continue
+
+        content = stripped
+
+        if content.startswith("- "):
+            content = content[2:].strip()
+        elif content.startswith("* "):
+            content = content[2:].strip()
+        elif content.startswith("• "):
+            content = content[2:].strip()
+        elif re.match(r"^\d+\.\s", content):
+            content = re.sub(r"^\d+\.\s+", "", content).strip()
+
+        if not content:
+            continue
+
+        item: MemoryItem = {
+            "key": f"claude-memory-{item_index}",
+            "content": content,
+            "source_cli": "claude-web",
+            "tags": ["claude-web", "imported"],
+        }
+        items.append(item)
+        item_index += 1
+
+    return items
+
+
+def import_from_paste(text: str, store: "MemoryStore") -> int:
+    """Import memories from Claude.ai paste into a MemoryStore.
+
+    Args:
+        text: Pasted memory list from Claude.ai
+        store: MemoryStore instance to add items to
+
+    Returns:
+        Count of items successfully added to store
+    """
+    items = parse_claude_paste(text)
+    count = 0
+
+    for item in items:
+        content = item.get("content", "")
+        if content:  # Skip empty content
+            key = item.get("key", "")
+            source_cli = item.get("source_cli", "claude-web")
+            tags = item.get("tags", [])
+            if isinstance(key, str) and isinstance(source_cli, str) and isinstance(tags, list):
+                store.add(
+                    key=key,
+                    content=content,
+                    source_cli=source_cli,
+                    tags=tags,
+                )
+                count += 1
+
+    return count

package/build/lib/runtime/memory_parsers/export.py

@@ -0,0 +1,97 @@
+"""Export memory items to markdown format."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+def export_to_markdown(items: list[dict[str, Any]]) -> str:
+    """Convert list of memory items to markdown string.
+
+    Args:
+        items: List of memory item dicts with keys: id, key, content, source_cli, tags, created_at, updated_at
+
+    Returns:
+        Markdown string with formatted memory items.
+    """
+    # Generate current timestamp in ISO format
+    now = datetime.now(timezone.utc).isoformat()
+
+    # Start with header
+    lines = [
+        "# OMG Shared Memory Export",
+        "",
+        f"Generated: {now}",
+        f"Total items: {len(items)}",
+        "",
+        "---",
+        "",
+    ]
+
+    # Add each item
+    for idx, item in enumerate(items, start=1):
+        key = item.get("key", "unknown")
+        content = item.get("content", "")
+        source_cli = item.get("source_cli", "unknown")
+        tags = item.get("tags", [])
+        created_at = item.get("created_at", "")
+        updated_at = item.get("updated_at", "")
+
+        # Format tags as comma-separated string
+        tags_str = ", ".join(tags) if tags else ""
+
+        lines.append(f"## Memory {idx}: {key}")
+        lines.append("")
+        lines.append(f"**Source**: {source_cli}")
+        lines.append(f"**Tags**: {tags_str}")
+        lines.append(f"**Created**: {created_at}")
+        lines.append(f"**Updated**: {updated_at}")
+        lines.append("")
+        lines.append(content)
+        lines.append("")
+        lines.append("---")
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+def export_to_file(items: list[dict[str, Any]], output_path: str) -> None:
+    """Write export_to_markdown(items) to the given file path.
+
+    Creates parent directories if missing. Overwrites if file exists.
+
+    Args:
+        items: List of memory item dicts
+        output_path: Path where to write the markdown file
+    """
+    path = Path(output_path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    markdown = export_to_markdown(items)
+    _ = path.write_text(markdown)
+
+
+def export_from_store(store: Any, output_path: str | None = None) -> str:  # pyright: ignore[reportExplicitAny]
+    """Export all items from a MemoryStore to markdown.
+
+    Gets all items from store.export_all(). If output_path is given, writes to file.
+    Always returns the markdown string.
+
+    Args:
+        store: MemoryStore instance
+        output_path: Optional path to write markdown file to
+
+    Returns:
+        Markdown string representation of all items
+    """
+    items = store.export_all()
+    markdown = export_to_markdown(items)
+
+    if output_path is not None:
+        export_to_file(items, output_path)
+
+    return markdown
+
+
+__all__ = ["export_to_markdown", "export_to_file", "export_from_store"]

package/build/lib/runtime/memory_parsers/gemini_import.py

@@ -0,0 +1,91 @@
+"""Gemini Web paste-based memory import — parse and import memories from Gemini Web interface."""
+
+from __future__ import annotations
+
+import re
+from typing import Any  # pyright: ignore[reportExplicitAny]
+
+EXTRACTION_PROMPT = (
+    "What do you remember about me? List all preferences, facts, and context you have stored. "
+    "Format each as a bullet point starting with '- '."
+)
+
+
+def parse_gemini_paste(text: str) -> list[dict[str, Any]]:  # pyright: ignore[reportExplicitAny]
+    """Parse memories from Gemini Web paste text.
+
+    Supports bullet points (- item) and numbered lists (1. item).
+
+    Args:
+        text: Raw text pasted from Gemini Web interface.
+
+    Returns:
+        List of memory items with keys, content, source_cli, and tags.
+    """
+    if not text or not text.strip():
+        return []
+
+    items: list[str] = []
+    lines = text.split("\n")
+
+    for line in lines:
+        line = line.strip()
+        if not line:
+            continue
+
+        # Try bullet point format: "- item"
+        if line.startswith("-"):
+            content = line[1:].strip()
+            if content:
+                items.append(content)
+            continue
+
+        # Try numbered format: "1. item", "2. item", etc.
+        match = re.match(r"^\d+\.\s+(.+)$", line)
+        if match:
+            content = match.group(1).strip()
+            if content:
+                items.append(content)
+            continue
+
+    # Convert to memory item dicts
+    result: list[dict[str, Any]] = []
+    for i, content in enumerate(items):
+        result.append(
+            {
+                "key": f"gemini-memory-{i+1}",
+                "content": content,
+                "source_cli": "gemini-web",
+                "tags": ["gemini-web", "imported"],
+            }
+        )
+
+    return result
+
+
+def import_from_paste(text: str, store: Any) -> int:  # pyright: ignore[reportAny]
+    """Import memories from Gemini Web paste into the memory store.
+
+    Args:
+        text: Raw text pasted from Gemini Web interface.
+        store: MemoryStore instance to add items to.
+
+    Returns:
+        Number of items successfully imported.
+    """
+    items = parse_gemini_paste(text)
+    count = 0
+
+    for item in items:
+        store.add(  # pyright: ignore[reportAny]
+            key=item["key"],
+            content=item["content"],
+            source_cli=item["source_cli"],
+            tags=item["tags"],
+        )
+        count += 1
+
+    return count
+
+
+__all__ = ["EXTRACTION_PROMPT", "parse_gemini_paste", "import_from_paste"]

package/build/lib/runtime/memory_parsers/kimi_import.py

@@ -0,0 +1,91 @@
+"""Kimi Web paste-based memory import — parse and import memories from Kimi Web interface."""
+
+from __future__ import annotations
+
+import re
+from typing import Any  # pyright: ignore[reportExplicitAny]
+
+EXTRACTION_PROMPT = (
+    "What information do you remember about me? Please list all stored preferences, facts, and context "
+    "as bullet points starting with '- '."
+)
+
+
+def parse_kimi_paste(text: str) -> list[dict[str, Any]]:  # pyright: ignore[reportExplicitAny]
+    """Parse memories from Kimi Web paste text.
+
+    Supports bullet points (- item) and numbered lists (1. item).
+
+    Args:
+        text: Raw text pasted from Kimi Web interface.
+
+    Returns:
+        List of memory items with keys, content, source_cli, and tags.
+    """
+    if not text or not text.strip():
+        return []
+
+    items: list[str] = []
+    lines = text.split("\n")
+
+    for line in lines:
+        line = line.strip()
+        if not line:
+            continue
+
+        # Try bullet point format: "- item"
+        if line.startswith("-"):
+            content = line[1:].strip()
+            if content:
+                items.append(content)
+            continue
+
+        # Try numbered format: "1. item", "2. item", etc.
+        match = re.match(r"^\d+\.\s+(.+)$", line)
+        if match:
+            content = match.group(1).strip()
+            if content:
+                items.append(content)
+            continue
+
+    # Convert to memory item dicts
+    result: list[dict[str, Any]] = []  # pyright: ignore[reportExplicitAny]
+    for i, content in enumerate(items):
+        result.append(
+            {
+                "key": f"kimi-memory-{i+1}",
+                "content": content,
+                "source_cli": "kimi-web",
+                "tags": ["kimi-web", "imported"],
+            }
+        )
+
+    return result
+
+
+def import_from_paste(text: str, store: Any) -> int:  # pyright: ignore[reportAny]
+    """Import memories from Kimi Web paste into the memory store.
+
+    Args:
+        text: Raw text pasted from Kimi Web interface.
+        store: MemoryStore instance to add items to.
+
+    Returns:
+        Number of items successfully imported.
+    """
+    items = parse_kimi_paste(text)
+    count = 0
+
+    for item in items:
+        store.add(  # pyright: ignore[reportAny]
+            key=item["key"],
+            content=item["content"],
+            source_cli=item["source_cli"],
+            tags=item["tags"],
+        )
+        count += 1
+
+    return count
+
+
+__all__ = ["EXTRACTION_PROMPT", "parse_kimi_paste", "import_from_paste"]