@elizaos/python 2.0.0-alpha.10

package/elizaos/media/mime.py

@@ -0,0 +1,315 @@
+"""
+MIME type detection and media utilities for Eliza.
+
+Provides robust MIME type detection from file buffers, headers, and extensions.
+"""
+
+from __future__ import annotations
+
+import re
+from enum import Enum
+from urllib.parse import urlparse
+
+# Try to import python-magic for MIME sniffing, fallback to filetype
+try:
+    import magic
+
+    HAS_MAGIC = True
+except ImportError:
+    HAS_MAGIC = False
+
+try:
+    import filetype
+
+    HAS_FILETYPE = True
+except ImportError:
+    HAS_FILETYPE = False
+
+
+class MediaKind(str, Enum):
+    """Media kind categories."""
+
+    IMAGE = "image"
+    AUDIO = "audio"
+    VIDEO = "video"
+    DOCUMENT = "document"
+    UNKNOWN = "unknown"
+
+
+# Map common MIME types to preferred file extensions
+EXT_BY_MIME: dict[str, str] = {
+    "image/heic": ".heic",
+    "image/heif": ".heif",
+    "image/jpeg": ".jpg",
+    "image/png": ".png",
+    "image/webp": ".webp",
+    "image/gif": ".gif",
+    "audio/ogg": ".ogg",
+    "audio/mpeg": ".mp3",
+    "audio/x-m4a": ".m4a",
+    "audio/mp4": ".m4a",
+    "video/mp4": ".mp4",
+    "video/quicktime": ".mov",
+    "application/pdf": ".pdf",
+    "application/json": ".json",
+    "application/zip": ".zip",
+    "application/gzip": ".gz",
+    "application/x-tar": ".tar",
+    "application/x-7z-compressed": ".7z",
+    "application/vnd.rar": ".rar",
+    "application/msword": ".doc",
+    "application/vnd.ms-excel": ".xls",
+    "application/vnd.ms-powerpoint": ".ppt",
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
+    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
+    "application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",
+    "text/csv": ".csv",
+    "text/plain": ".txt",
+    "text/markdown": ".md",
+}
+
+# Reverse map: extension to MIME
+MIME_BY_EXT: dict[str, str] = {v: k for k, v in EXT_BY_MIME.items()}
+MIME_BY_EXT[".jpeg"] = "image/jpeg"
+
+# Audio file extensions
+AUDIO_FILE_EXTENSIONS: set[str] = {".aac", ".flac", ".m4a", ".mp3", ".oga", ".ogg", ".opus", ".wav"}
+
+# Voice-compatible audio extensions (Opus/Ogg)
+VOICE_AUDIO_EXTENSIONS: set[str] = {".oga", ".ogg", ".opus"}
+
+
+def _normalize_header_mime(mime: str | None) -> str | None:
+    """Normalize a MIME type from HTTP headers."""
+    if not mime:
+        return None
+    cleaned = mime.split(";")[0].strip().lower()
+    return cleaned or None
+
+
+def _sniff_mime(buffer: bytes | None) -> str | None:
+    """Detect MIME type from a buffer using magic bytes."""
+    if not buffer:
+        return None
+
+    # Try python-magic first
+    if HAS_MAGIC:
+        try:
+            mime = magic.from_buffer(buffer, mime=True)
+            return mime if mime else None
+        except Exception:
+            pass
+
+    # Fallback to filetype
+    if HAS_FILETYPE:
+        try:
+            kind = filetype.guess(buffer)
+            return kind.mime if kind else None
+        except Exception:
+            pass
+
+    return None
+
+
+def get_file_extension(file_path: str | None) -> str | None:
+    """
+    Get the file extension from a path or URL.
+
+    Args:
+        file_path: File path or URL
+
+    Returns:
+        File extension including the dot (e.g., ".jpg"), or None
+    """
+    if not file_path:
+        return None
+
+    # Try parsing as URL
+    if re.match(r"^https?://", file_path, re.IGNORECASE):
+        try:
+            parsed = urlparse(file_path)
+            path_parts = parsed.path.split(".")
+            if len(path_parts) >= 2:
+                ext = path_parts[-1].lower()
+                return f".{ext}"
+        except Exception:
+            pass
+
+    # Plain path parsing
+    parts = file_path.split(".")
+    if len(parts) < 2:
+        return None
+    return f".{parts[-1].lower()}"
+
+
+def _is_generic_mime(mime: str | None) -> bool:
+    """Check if a MIME type is generic/container type."""
+    if not mime:
+        return True
+    m = mime.lower()
+    return m == "application/octet-stream" or m == "application/zip"
+
+
+def detect_mime(
+    buffer: bytes | None = None,
+    header_mime: str | None = None,
+    file_path: str | None = None,
+) -> str | None:
+    """
+    Detect MIME type from buffer, headers, and/or file path.
+    Prioritizes sniffed types over extension-based detection.
+
+    Args:
+        buffer: File contents for magic byte detection
+        header_mime: MIME type from HTTP headers
+        file_path: File path for extension-based detection
+
+    Returns:
+        Detected MIME type or None
+    """
+    ext = get_file_extension(file_path)
+    ext_mime = MIME_BY_EXT.get(ext) if ext else None
+    normalized_header = _normalize_header_mime(header_mime)
+    sniffed = _sniff_mime(buffer)
+
+    # Prefer sniffed types, but don't let generic container types override
+    # a more specific extension mapping (e.g., XLSX vs ZIP)
+    if sniffed and (not _is_generic_mime(sniffed) or not ext_mime):
+        return sniffed
+    if ext_mime:
+        return ext_mime
+    if normalized_header and not _is_generic_mime(normalized_header):
+        return normalized_header
+    if sniffed:
+        return sniffed
+    if normalized_header:
+        return normalized_header
+
+    return None
+
+
+def extension_for_mime(mime: str | None) -> str | None:
+    """
+    Get the file extension for a MIME type.
+
+    Args:
+        mime: MIME type string
+
+    Returns:
+        File extension including the dot (e.g., ".jpg"), or None
+    """
+    if not mime:
+        return None
+    return EXT_BY_MIME.get(mime.lower())
+
+
+def is_audio_filename(filename: str | None) -> bool:
+    """
+    Check if a file appears to be an audio file by extension.
+
+    Args:
+        filename: File name or path
+
+    Returns:
+        True if the file has an audio extension
+    """
+    ext = get_file_extension(filename)
+    return ext in AUDIO_FILE_EXTENSIONS if ext else False
+
+
+def is_gif_media(
+    content_type: str | None = None,
+    filename: str | None = None,
+) -> bool:
+    """
+    Check if media is a GIF.
+
+    Args:
+        content_type: MIME type / content type header
+        filename: File name or path
+
+    Returns:
+        True if the media is a GIF
+    """
+    if content_type and content_type.lower() == "image/gif":
+        return True
+    return get_file_extension(filename) == ".gif"
+
+
+def is_voice_compatible_audio(
+    content_type: str | None = None,
+    filename: str | None = None,
+) -> bool:
+    """
+    Check if audio is voice-compatible (Opus/Ogg format).
+
+    Args:
+        content_type: MIME type / content type header
+        filename: File name or path
+
+    Returns:
+        True if the audio is in Opus/Ogg format
+    """
+    mime = content_type.lower() if content_type else None
+    if mime and ("ogg" in mime or "opus" in mime):
+        return True
+    ext = get_file_extension(filename)
+    return ext in VOICE_AUDIO_EXTENSIONS if ext else False
+
+
+def media_kind_from_mime(mime: str | None) -> MediaKind:
+    """
+    Get media kind from MIME type.
+
+    Args:
+        mime: MIME type string
+
+    Returns:
+        MediaKind enum value
+    """
+    if not mime:
+        return MediaKind.UNKNOWN
+
+    m = mime.lower()
+    if m.startswith("image/"):
+        return MediaKind.IMAGE
+    if m.startswith("audio/"):
+        return MediaKind.AUDIO
+    if m.startswith("video/"):
+        return MediaKind.VIDEO
+    if (
+        m.startswith("application/pdf")
+        or m.startswith("application/msword")
+        or m.startswith("application/vnd.ms-")
+        or m.startswith("application/vnd.openxmlformats")
+        or m.startswith("text/")
+    ):
+        return MediaKind.DOCUMENT
+
+    return MediaKind.UNKNOWN
+
+
+def image_mime_from_format(format_name: str | None) -> str | None:
+    """
+    Get image MIME type from format name.
+
+    Args:
+        format_name: Image format name (e.g., "jpg", "png")
+
+    Returns:
+        MIME type string or None
+    """
+    if not format_name:
+        return None
+
+    fmt = format_name.lower()
+    mapping = {
+        "jpg": "image/jpeg",
+        "jpeg": "image/jpeg",
+        "heic": "image/heic",
+        "heif": "image/heif",
+        "png": "image/png",
+        "webp": "image/webp",
+        "gif": "image/gif",
+    }
+    return mapping.get(fmt)

package/elizaos/media/search.py

@@ -0,0 +1,161 @@
+"""
+Hybrid search utilities for combining vector and keyword search results.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+
+@dataclass
+class HybridVectorResult:
+    """Result from vector similarity search."""
+
+    id: str
+    path: str
+    start_line: int
+    end_line: int
+    source: str
+    snippet: str
+    vector_score: float
+
+
+@dataclass
+class HybridKeywordResult:
+    """Result from keyword (BM25) search."""
+
+    id: str
+    path: str
+    start_line: int
+    end_line: int
+    source: str
+    snippet: str
+    text_score: float
+
+
+@dataclass
+class HybridMergedResult:
+    """Merged result from hybrid search."""
+
+    path: str
+    start_line: int
+    end_line: int
+    score: float
+    snippet: str
+    source: str
+
+
+def build_fts_query(raw: str) -> str | None:
+    """
+    Build an FTS (Full-Text Search) query from a raw search string.
+    Extracts alphanumeric tokens and joins them with AND for strict matching.
+
+    Args:
+        raw: The raw search query string
+
+    Returns:
+        The FTS query string, or None if no valid tokens found
+    """
+    tokens = re.findall(r"[A-Za-z0-9_]+", raw)
+    tokens = [t.strip() for t in tokens if t.strip()]
+
+    if not tokens:
+        return None
+
+    quoted = [f'"{t.replace(chr(34), "")}"' for t in tokens]
+    return " AND ".join(quoted)
+
+
+def bm25_rank_to_score(rank: float) -> float:
+    """
+    Convert BM25 rank to a normalized score between 0 and 1.
+    Lower rank = higher score.
+
+    Args:
+        rank: The BM25 rank value
+
+    Returns:
+        A normalized score where 1 is best and 0 is worst
+    """
+    import math
+
+    normalized = max(0.0, rank) if math.isfinite(rank) else 999.0
+    return 1.0 / (1.0 + normalized)
+
+
+def merge_hybrid_results(
+    vector: list[HybridVectorResult],
+    keyword: list[HybridKeywordResult],
+    vector_weight: float = 0.7,
+    text_weight: float = 0.3,
+) -> list[HybridMergedResult]:
+    """
+    Merge vector similarity and keyword search results using weighted scoring.
+
+    This implements a hybrid search approach where results from both vector
+    similarity search and keyword (BM25) search are combined. Results that
+    appear in both searches get boosted scores.
+
+    Args:
+        vector: Results from vector similarity search
+        keyword: Results from keyword (BM25) search
+        vector_weight: Weight for vector similarity scores (default: 0.7)
+        text_weight: Weight for keyword/text scores (default: 0.3)
+
+    Returns:
+        Merged and sorted results with combined scores
+    """
+    by_id: dict[str, dict] = {}
+
+    # Add vector search results
+    for r in vector:
+        by_id[r.id] = {
+            "id": r.id,
+            "path": r.path,
+            "start_line": r.start_line,
+            "end_line": r.end_line,
+            "source": r.source,
+            "snippet": r.snippet,
+            "vector_score": r.vector_score,
+            "text_score": 0.0,
+        }
+
+    # Merge keyword search results
+    for r in keyword:
+        if r.id in by_id:
+            existing = by_id[r.id]
+            existing["text_score"] = r.text_score
+            # Prefer keyword snippet if available (may have highlights)
+            if r.snippet:
+                existing["snippet"] = r.snippet
+        else:
+            by_id[r.id] = {
+                "id": r.id,
+                "path": r.path,
+                "start_line": r.start_line,
+                "end_line": r.end_line,
+                "source": r.source,
+                "snippet": r.snippet,
+                "vector_score": 0.0,
+                "text_score": r.text_score,
+            }
+
+    # Calculate weighted scores and create results
+    merged = []
+    for entry in by_id.values():
+        score = vector_weight * entry["vector_score"] + text_weight * entry["text_score"]
+        merged.append(
+            HybridMergedResult(
+                path=entry["path"],
+                start_line=entry["start_line"],
+                end_line=entry["end_line"],
+                score=score,
+                snippet=entry["snippet"],
+                source=entry["source"],
+            )
+        )
+
+    # Sort by score descending
+    merged.sort(key=lambda x: x.score, reverse=True)
+    return merged

package/elizaos/media/tests/__init__.py

@@ -0,0 +1 @@
+# Tests for media utilities

package/elizaos/media/tests/test_mime.py

@@ -0,0 +1,117 @@
+"""Tests for MIME utilities."""
+
+from elizaos.media.mime import (
+    MediaKind,
+    extension_for_mime,
+    get_file_extension,
+    image_mime_from_format,
+    is_audio_filename,
+    is_gif_media,
+    is_voice_compatible_audio,
+    media_kind_from_mime,
+)
+
+
+class TestGetFileExtension:
+    def test_simple_path(self):
+        assert get_file_extension("test.jpg") == ".jpg"
+
+    def test_path_with_directory(self):
+        assert get_file_extension("path/to/file.PNG") == ".png"
+
+    def test_no_extension(self):
+        assert get_file_extension("noext") is None
+
+    def test_none_input(self):
+        assert get_file_extension(None) is None
+
+    def test_url(self):
+        assert get_file_extension("https://example.com/image.webp") == ".webp"
+
+
+class TestMediaKindFromMime:
+    def test_image(self):
+        assert media_kind_from_mime("image/jpeg") == MediaKind.IMAGE
+        assert media_kind_from_mime("image/png") == MediaKind.IMAGE
+
+    def test_audio(self):
+        assert media_kind_from_mime("audio/mp3") == MediaKind.AUDIO
+        assert media_kind_from_mime("audio/ogg") == MediaKind.AUDIO
+
+    def test_video(self):
+        assert media_kind_from_mime("video/mp4") == MediaKind.VIDEO
+
+    def test_document(self):
+        assert media_kind_from_mime("application/pdf") == MediaKind.DOCUMENT
+        assert media_kind_from_mime("text/plain") == MediaKind.DOCUMENT
+
+    def test_unknown(self):
+        assert media_kind_from_mime("application/octet-stream") == MediaKind.UNKNOWN
+        assert media_kind_from_mime(None) == MediaKind.UNKNOWN
+
+
+class TestIsAudioFilename:
+    def test_audio_files(self):
+        assert is_audio_filename("song.mp3") is True
+        assert is_audio_filename("voice.ogg") is True
+        assert is_audio_filename("music.wav") is True
+
+    def test_non_audio_files(self):
+        assert is_audio_filename("image.jpg") is False
+        assert is_audio_filename("document.pdf") is False
+
+    def test_none(self):
+        assert is_audio_filename(None) is False
+
+
+class TestIsGifMedia:
+    def test_gif_content_type(self):
+        assert is_gif_media(content_type="image/gif") is True
+
+    def test_gif_filename(self):
+        assert is_gif_media(filename="animation.gif") is True
+
+    def test_non_gif(self):
+        assert is_gif_media(content_type="image/jpeg") is False
+        assert is_gif_media(filename="photo.jpg") is False
+
+
+class TestIsVoiceCompatibleAudio:
+    def test_ogg_content_type(self):
+        assert is_voice_compatible_audio(content_type="audio/ogg") is True
+        assert is_voice_compatible_audio(content_type="audio/opus") is True
+
+    def test_ogg_filename(self):
+        assert is_voice_compatible_audio(filename="voice.ogg") is True
+        assert is_voice_compatible_audio(filename="voice.opus") is True
+
+    def test_non_voice_compatible(self):
+        assert is_voice_compatible_audio(content_type="audio/mp3") is False
+        assert is_voice_compatible_audio(filename="song.mp3") is False
+
+
+class TestExtensionForMime:
+    def test_known_mimes(self):
+        assert extension_for_mime("image/jpeg") == ".jpg"
+        assert extension_for_mime("audio/mpeg") == ".mp3"
+        assert extension_for_mime("video/mp4") == ".mp4"
+
+    def test_none(self):
+        assert extension_for_mime(None) is None
+
+    def test_unknown_mime(self):
+        assert extension_for_mime("application/x-unknown") is None
+
+
+class TestImageMimeFromFormat:
+    def test_known_formats(self):
+        assert image_mime_from_format("jpg") == "image/jpeg"
+        assert image_mime_from_format("jpeg") == "image/jpeg"
+        assert image_mime_from_format("png") == "image/png"
+        assert image_mime_from_format("gif") == "image/gif"
+
+    def test_none(self):
+        assert image_mime_from_format(None) is None
+
+    def test_unknown_format(self):
+        assert image_mime_from_format("bmp") is None