ltcai 11.1.0 → 11.4.0

package/lattice_brain/multimodal/audio.py

@@ -0,0 +1,77 @@
+"""A recording as a memory — transcribed through the injected port, or not.
+
+The whole module is three small pieces because that is all an audio memory is:
+the recording's facts, one attempt at turning it into words, and an honest
+score for how much of it a typed question can reach afterwards. Without a
+transcriber the memory is still kept and simply says so.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+from .ports import MultimodalPorts
+
+
+@dataclass
+class AudioFacts:
+    """A recording, its transcript, and how honestly we got one."""
+
+    path: str
+    #: ``ok`` | ``unavailable`` | ``failed`` | ``supplied``
+    transcription_status: str = "unavailable"
+    transcript: str = ""
+    detail: str = ""
+    segments: List[Dict[str, Any]] = field(default_factory=list)
+
+    @property
+    def searchable(self) -> bool:
+        return bool(self.transcript)
+
+
+def transcribe_audio(
+    path: str,
+    *,
+    ports: Optional[MultimodalPorts] = None,
+    transcript: Optional[str] = None,
+) -> AudioFacts:
+    """Transcribe a recording through the injected port, or say why not.
+
+    ``transcript`` lets a caller that already has text (a phone's own
+    dictation, ``VoiceCaptureService``) skip the local model entirely. An
+    absent transcriber yields ``transcription_status="unavailable"`` and an
+    empty transcript — the recording is still remembered by title and path,
+    and the result never claims it is searchable.
+    """
+    ports = ports or MultimodalPorts()
+    supplied = str(transcript or "").strip()
+    if supplied:
+        return AudioFacts(path=str(path), transcription_status="supplied", transcript=supplied)
+    if ports.transcriber is None:
+        return AudioFacts(
+            path=str(path),
+            transcription_status="unavailable",
+            detail="no local transcriber is configured",
+        )
+    try:
+        text = str(ports.transcriber(str(path)) or "").strip()
+    except Exception as exc:  # noqa: BLE001 — a broken transcriber is a state
+        return AudioFacts(path=str(path), transcription_status="failed", detail=str(exc))
+    if not text:
+        return AudioFacts(
+            path=str(path),
+            transcription_status="failed",
+            detail="the transcriber returned no text",
+        )
+    return AudioFacts(path=str(path), transcription_status="ok", transcript=text)
+
+
+def audio_quality_score(facts: AudioFacts) -> Dict[str, Any]:
+    """How much of this recording is actually retrievable later."""
+    if not facts.transcript:
+        return {"score": 0.0, "reasons": ["no_transcript"]}
+    # A transcript is text: length is the only honest extra signal here, and
+    # the text pipeline scores the wording itself downstream.
+    score = 0.5 + 0.5 * min(1.0, len(facts.transcript) / 400.0)
+    return {"score": round(score, 4), "reasons": ["transcript"]}

package/lattice_brain/multimodal/common.py

@@ -0,0 +1,118 @@
+"""Modality taxonomy, size budgets, and the pure helpers every door shares.
+
+The tables here are the module's own decisions (``.mp4`` is video unless the
+capture surface says otherwise), deliberately kept ahead of the platform's
+``mimetypes`` answer. Nothing in this file touches a model, a decoder, or the
+graph, which is what lets images, audio and video all import it without
+importing each other.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import mimetypes
+from pathlib import Path
+from typing import List, Optional
+
+# ── modality taxonomy ────────────────────────────────────────────────────────
+MODALITY_TEXT = "text"
+MODALITY_IMAGE = "image"
+MODALITY_AUDIO = "audio"
+MODALITY_VIDEO = "video"
+
+IMAGE_EXTENSIONS = frozenset(
+    {".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".tif", ".tiff", ".heic"}
+)
+# Containers that are *only* ever audio. ``.mp4``/``.webm`` are deliberately
+# absent: by extension alone they are video, and a voice memo recorded in one
+# of them arrives through an explicit audio MIME type (or through
+# ``VoiceCaptureService``, where the user already said "this is a memo").
+# ``.mid``/``.midi`` are listed for a second reason: CPython's *built-in* mime
+# table has neither, so ``mimetypes`` answers "audio/midi" only on a host that
+# ships a system mime file (macOS reads /etc/apache2/mime.types; a slim Linux
+# container has nothing). Leaving them to the fallback let the platform decide
+# what a MIDI file is — a module table exists precisely so it does not.
+AUDIO_EXTENSIONS = frozenset(
+    {".m4a", ".mp3", ".wav", ".aac", ".flac", ".ogg", ".opus", ".mid", ".midi"}
+)
+VIDEO_EXTENSIONS = frozenset({".mp4", ".webm", ".mov", ".mkv", ".avi", ".m4v"})
+#: Subtitle/caption files a video may arrive with. Same basename, so a
+#: ``standup.mp4`` next to a ``standup.srt`` is one memory, not two.
+SUBTITLE_EXTENSIONS = ("srt", "vtt")
+
+#: Why a video is recognized and still refused — surfaced to the caller. In
+#: 11.1.0 the reason was *scope* (nothing was implemented). Since 11.2.0 the
+#: implementation exists and the only remaining reason is a **runtime** one:
+#: this machine has no ``ffmpeg``, and inventing frames is not an option.
+VIDEO_UNAVAILABLE_DETAIL = (
+    "video ingestion needs ffmpeg on this machine and none was found; the file "
+    "was not stored (install ffmpeg to enable keyframe extraction)"
+)
+#: Kept under its 11.1.0 name so existing importers keep working; the reason it
+#: carries has changed from "out of scope" to "unavailable on this machine".
+VIDEO_OUT_OF_SCOPE = VIDEO_UNAVAILABLE_DETAIL
+
+#: Longest OCR/caption body kept on the node (a screenshot is not a novel).
+MAX_INDEX_TEXT_CHARS = 20_000
+#: Summary column budget, matching every other ingest door in the graph.
+SUMMARY_CHARS = 500
+#: Fixed-width chunking for OCR bodies that outgrow the summary.
+IMAGE_CHUNK_CHARS = 900
+#: Longest edge of the stored thumbnail, in pixels.
+THUMBNAIL_EDGE = 96
+#: A thumbnail is a UI affordance, not an archive — drop it past this size.
+MAX_THUMBNAIL_CHARS = 24_000
+
+
+def detect_modality(
+    path: Optional[str] = None, mime_type: Optional[str] = None
+) -> str:
+    """``text`` | ``image`` | ``audio`` | ``video`` for one candidate file.
+
+    The declared MIME type wins when it carries a usable top-level type: the
+    capture surface saw the bytes, this function only sees a name. Otherwise
+    the extension decides, and an unknown extension is ``text`` so existing
+    behaviour is untouched.
+    """
+    declared = str(mime_type or "").strip().lower().split(";")[0].split("/")[0]
+    if declared in {MODALITY_IMAGE, MODALITY_AUDIO, MODALITY_VIDEO}:
+        return declared
+    # The extension tables come before ``mimetypes`` on purpose: they are where
+    # this module's decisions live (``.mp4`` is video unless someone who saw
+    # the bytes says otherwise), and the stdlib table varies by platform.
+    suffix = Path(str(path or "")).suffix.lower()
+    if suffix in IMAGE_EXTENSIONS:
+        return MODALITY_IMAGE
+    if suffix in AUDIO_EXTENSIONS:
+        return MODALITY_AUDIO
+    if suffix in VIDEO_EXTENSIONS:
+        return MODALITY_VIDEO
+    if path:
+        guessed, _ = mimetypes.guess_type(str(path))
+        top = str(guessed or "").split("/")[0]
+        if top in {MODALITY_IMAGE, MODALITY_AUDIO, MODALITY_VIDEO}:
+            return top
+    return MODALITY_TEXT
+
+
+def _sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for block in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+def _sha256_text(text: str) -> str:
+    return hashlib.sha256(str(text).encode("utf-8", "ignore")).hexdigest()
+
+
+def _split_index_text(text: str) -> List[str]:
+    """Fixed-width split for OCR bodies — no markdown or code structure here."""
+    body = str(text or "").strip()
+    if len(body) <= SUMMARY_CHARS:
+        return []
+    return [
+        body[start : start + IMAGE_CHUNK_CHARS]
+        for start in range(0, len(body), IMAGE_CHUNK_CHARS)
+    ]

package/lattice_brain/{multimodal.py → multimodal/images.py}

@@ -1,160 +1,35 @@
-"""Images and audio as first-class memories (v11.1.0, Track 3).
-
-Before this module the Brain could only remember things that arrived as text.
-A screenshot of a whiteboard, a photo of a receipt, a voice memo — all of them
-either bounced off the ingestion pipeline or landed as an opaque ``Document``
-node whose only searchable content was its filename.
-
-What routes here
-----------------
-:func:`detect_modality` reads the MIME type first and the extension second, and
-answers with one of ``text`` / ``image`` / ``audio`` / ``video``. ``video`` is
-deliberately a *recognized but unsupported* answer in this release: keyframe
-extraction needs a decoder this project does not ship, and returning "video,
-out of scope" is worth more than pretending a ``.mov`` is a picture.
-
-What an image memory contains
------------------------------
-:func:`extract_image_facts` gathers only what it can actually observe:
-
-* **dimensions/format** from Pillow (a core dependency);
-* **ocr_text** from ``pytesseract`` when it is installed — otherwise
-  ``ocr_status="unavailable"`` and no text, never an empty string dressed up as
-  a successful read;
-* **caption** from an injected vision-language port, and *only* from there. No
-  VLM means ``caption is None``. Composing "Image IMG_2381.png (JPEG 3024x4032)"
-  and storing it in the caption field would make metadata indistinguishable
-  from a model's description forever after;
-* **embedding** from an injected vision port, which lives in its own vector
-  space (see :mod:`latticeai.core.embedding_providers`) and therefore its own
-  index — text queries reach images through OCR/caption text, not by scoring a
-  BGE vector against CLIP vectors.
-
-Brain Core owns none of those models. Every heavy dependency arrives as an
-injected callable (:class:`MultimodalPorts`), which is also why this module
-imports nothing from ``latticeai``.
+"""A picture as a first-class memory: what was observed, and what was written.
+
+Two halves, in order: :func:`extract_image_facts` observes one file (size, OCR,
+caption, vector — each with its own status), and :func:`write_image_memory`
+turns those facts into an ``Image`` node with its chunks, concepts and source
+link. Video reuses both verbatim for its keyframes, which is why nothing here
+knows what a video is.
 """
 
 from __future__ import annotations
 
 import base64
-import hashlib
 import io
-import mimetypes
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional
 
-from .quiet import quiet
-from .utils import utc_now_iso
-
-# ── modality taxonomy ────────────────────────────────────────────────────────
-MODALITY_TEXT = "text"
-MODALITY_IMAGE = "image"
-MODALITY_AUDIO = "audio"
-MODALITY_VIDEO = "video"
-
-IMAGE_EXTENSIONS = frozenset(
-    {".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".tif", ".tiff", ".heic"}
+from ..quiet import quiet
+from ..utils import utc_now_iso
+from .common import (
+    MAX_INDEX_TEXT_CHARS,
+    MAX_THUMBNAIL_CHARS,
+    MODALITY_IMAGE,
+    SUMMARY_CHARS,
+    THUMBNAIL_EDGE,
+    _sha256_file,
+    _sha256_text,
+    _split_index_text,
 )
-# Containers that are *only* ever audio. ``.mp4``/``.webm`` are deliberately
-# absent: by extension alone they are video, and a voice memo recorded in one
-# of them arrives through an explicit audio MIME type (or through
-# ``VoiceCaptureService``, where the user already said "this is a memo").
-# ``.mid``/``.midi`` are listed for a second reason: CPython's *built-in* mime
-# table has neither, so ``mimetypes`` answers "audio/midi" only on a host that
-# ships a system mime file (macOS reads /etc/apache2/mime.types; a slim Linux
-# container has nothing). Leaving them to the fallback let the platform decide
-# what a MIDI file is — a module table exists precisely so it does not.
-AUDIO_EXTENSIONS = frozenset(
-    {".m4a", ".mp3", ".wav", ".aac", ".flac", ".ogg", ".opus", ".mid", ".midi"}
-)
-VIDEO_EXTENSIONS = frozenset({".mp4", ".webm", ".mov", ".mkv", ".avi", ".m4v"})
+from .ports import MultimodalPorts
 
-#: Why ``video`` is recognized and still refused — surfaced to the caller.
-VIDEO_OUT_OF_SCOPE = (
-    "video ingestion is out of scope in this release (keyframe extraction is "
-    "not implemented); the file was not stored"
-)
 
-#: Longest OCR/caption body kept on the node (a screenshot is not a novel).
-MAX_INDEX_TEXT_CHARS = 20_000
-#: Summary column budget, matching every other ingest door in the graph.
-SUMMARY_CHARS = 500
-#: Fixed-width chunking for OCR bodies that outgrow the summary.
-IMAGE_CHUNK_CHARS = 900
-#: Longest edge of the stored thumbnail, in pixels.
-THUMBNAIL_EDGE = 96
-#: A thumbnail is a UI affordance, not an archive — drop it past this size.
-MAX_THUMBNAIL_CHARS = 24_000
-
-
-def detect_modality(
-    path: Optional[str] = None, mime_type: Optional[str] = None
-) -> str:
-    """``text`` | ``image`` | ``audio`` | ``video`` for one candidate file.
-
-    The declared MIME type wins when it carries a usable top-level type: the
-    capture surface saw the bytes, this function only sees a name. Otherwise
-    the extension decides, and an unknown extension is ``text`` so existing
-    behaviour is untouched.
-    """
-    declared = str(mime_type or "").strip().lower().split(";")[0].split("/")[0]
-    if declared in {MODALITY_IMAGE, MODALITY_AUDIO, MODALITY_VIDEO}:
-        return declared
-    # The extension tables come before ``mimetypes`` on purpose: they are where
-    # this module's decisions live (``.mp4`` is video unless someone who saw
-    # the bytes says otherwise), and the stdlib table varies by platform.
-    suffix = Path(str(path or "")).suffix.lower()
-    if suffix in IMAGE_EXTENSIONS:
-        return MODALITY_IMAGE
-    if suffix in AUDIO_EXTENSIONS:
-        return MODALITY_AUDIO
-    if suffix in VIDEO_EXTENSIONS:
-        return MODALITY_VIDEO
-    if path:
-        guessed, _ = mimetypes.guess_type(str(path))
-        top = str(guessed or "").split("/")[0]
-        if top in {MODALITY_IMAGE, MODALITY_AUDIO, MODALITY_VIDEO}:
-            return top
-    return MODALITY_TEXT
-
-
-# ── injected capability ports ────────────────────────────────────────────────
-@dataclass
-class MultimodalPorts:
-    """The optional model-backed capabilities Brain Core cannot ship itself.
-
-    Every field is a plain callable so the app layer can build it from
-    ``latticeai.core.embedding_providers`` without Brain Core ever importing
-    that package. ``None`` everywhere is the honest default: OCR still runs
-    (``pytesseract`` is a local binary, not a model download), and everything
-    else reports itself as unavailable.
-    """
-
-    #: ``(image_path) -> caption or None`` — a loaded VLM, or nothing.
-    captioner: Optional[Callable[[str], Optional[str]]] = None
-    #: ``(image_path) -> vector`` in the image space (raises when it cannot).
-    vision_embedder: Optional[Callable[[str], List[float]]] = None
-    #: ``(audio_path) -> transcript`` (raises/returns empty when it cannot).
-    transcriber: Optional[Callable[[str], str]] = None
-    #: Identity of the vision model, recorded next to every image vector.
-    vision_model_id: str = ""
-    #: ``image`` (own index + late fusion) or ``shared`` (same space as text).
-    vision_space: str = MODALITY_IMAGE
-
-    def describe(self) -> Dict[str, Any]:
-        """What this install can honestly do with a picture or a recording."""
-        return {
-            "caption": self.captioner is not None,
-            "vision_embedding": self.vision_embedder is not None,
-            "transcription": self.transcriber is not None,
-            "vision_model_id": self.vision_model_id,
-            "vision_space": self.vision_space,
-        }
-
-
-# ── image facts ──────────────────────────────────────────────────────────────
 @dataclass
 class ImageFacts:
     """Everything observed about one image, and the status of each attempt."""
@@ -369,30 +244,6 @@ def image_quality_score(facts: ImageFacts) -> Dict[str, Any]:
     return {"score": round(max(0.0, min(1.0, score)), 4), "reasons": reasons}
 
 
-# ── graph write ──────────────────────────────────────────────────────────────
-def _sha256_file(path: Path) -> str:
-    digest = hashlib.sha256()
-    with path.open("rb") as handle:
-        for block in iter(lambda: handle.read(1024 * 1024), b""):
-            digest.update(block)
-    return digest.hexdigest()
-
-
-def _sha256_text(text: str) -> str:
-    return hashlib.sha256(str(text).encode("utf-8", "ignore")).hexdigest()
-
-
-def _split_index_text(text: str) -> List[str]:
-    """Fixed-width split for OCR bodies — no markdown or code structure here."""
-    body = str(text or "").strip()
-    if len(body) <= SUMMARY_CHARS:
-        return []
-    return [
-        body[start : start + IMAGE_CHUNK_CHARS]
-        for start in range(0, len(body), IMAGE_CHUNK_CHARS)
-    ]
-
-
 def image_node_id(content_hash: str, workspace_id: Optional[str] = None) -> str:
     """Workspace-scoped, content-addressed id — re-ingesting is idempotent."""
     scoped = f"{workspace_id or 'legacy-global'}|{content_hash}"
@@ -583,7 +434,7 @@ def _attach_concepts(
     body = str(text or "").strip()
     if not body:
         return []
-    from .graph._kg_common import (  # local: keeps this module's import light
+    from ..graph._kg_common import (  # local: keeps this module's import light
         _classify_node_type,
         _extract_concepts,
     )
@@ -591,7 +442,7 @@ def _attach_concepts(
     # The id derivation is imported rather than re-derived: two copies of a
     # node-id rule diverge, and a diverged id is a duplicate concept nobody
     # can see.
-    from .graph.ingest import _scoped_slug_id
+    from ..graph.ingest import _scoped_slug_id
 
     concept_ids: List[str] = []
     for concept in _extract_concepts(body, limit=10):
@@ -645,94 +496,3 @@ def _attach_source(
             extra={"owner": owner, "workspace_id": workspace_id, "modality": MODALITY_IMAGE},
         )
     )
-
-
-# ── audio ────────────────────────────────────────────────────────────────────
-@dataclass
-class AudioFacts:
-    """A recording, its transcript, and how honestly we got one."""
-
-    path: str
-    #: ``ok`` | ``unavailable`` | ``failed`` | ``supplied``
-    transcription_status: str = "unavailable"
-    transcript: str = ""
-    detail: str = ""
-    segments: List[Dict[str, Any]] = field(default_factory=list)
-
-    @property
-    def searchable(self) -> bool:
-        return bool(self.transcript)
-
-
-def transcribe_audio(
-    path: str,
-    *,
-    ports: Optional[MultimodalPorts] = None,
-    transcript: Optional[str] = None,
-) -> AudioFacts:
-    """Transcribe a recording through the injected port, or say why not.
-
-    ``transcript`` lets a caller that already has text (a phone's own
-    dictation, ``VoiceCaptureService``) skip the local model entirely. An
-    absent transcriber yields ``transcription_status="unavailable"`` and an
-    empty transcript — the recording is still remembered by title and path,
-    and the result never claims it is searchable.
-    """
-    ports = ports or MultimodalPorts()
-    supplied = str(transcript or "").strip()
-    if supplied:
-        return AudioFacts(path=str(path), transcription_status="supplied", transcript=supplied)
-    if ports.transcriber is None:
-        return AudioFacts(
-            path=str(path),
-            transcription_status="unavailable",
-            detail="no local transcriber is configured",
-        )
-    try:
-        text = str(ports.transcriber(str(path)) or "").strip()
-    except Exception as exc:  # noqa: BLE001 — a broken transcriber is a state
-        return AudioFacts(path=str(path), transcription_status="failed", detail=str(exc))
-    if not text:
-        return AudioFacts(
-            path=str(path),
-            transcription_status="failed",
-            detail="the transcriber returned no text",
-        )
-    return AudioFacts(path=str(path), transcription_status="ok", transcript=text)
-
-
-def audio_quality_score(facts: AudioFacts) -> Dict[str, Any]:
-    """How much of this recording is actually retrievable later."""
-    if not facts.transcript:
-        return {"score": 0.0, "reasons": ["no_transcript"]}
-    # A transcript is text: length is the only honest extra signal here, and
-    # the text pipeline scores the wording itself downstream.
-    score = 0.5 + 0.5 * min(1.0, len(facts.transcript) / 400.0)
-    return {"score": round(score, 4), "reasons": ["transcript"]}
-
-
-__all__ = [
-    "AUDIO_EXTENSIONS",
-    "IMAGE_CHUNK_CHARS",
-    "IMAGE_EXTENSIONS",
-    "MAX_INDEX_TEXT_CHARS",
-    "MAX_THUMBNAIL_CHARS",
-    "MODALITY_AUDIO",
-    "MODALITY_IMAGE",
-    "MODALITY_TEXT",
-    "MODALITY_VIDEO",
-    "SUMMARY_CHARS",
-    "THUMBNAIL_EDGE",
-    "VIDEO_EXTENSIONS",
-    "VIDEO_OUT_OF_SCOPE",
-    "AudioFacts",
-    "ImageFacts",
-    "MultimodalPorts",
-    "audio_quality_score",
-    "detect_modality",
-    "extract_image_facts",
-    "image_node_id",
-    "image_quality_score",
-    "transcribe_audio",
-    "write_image_memory",
-]