ltcai 11.5.2 → 11.7.0

package/lattice_brain/ingestion/_contract.py

@@ -1,90 +0,0 @@
-"""The seam the three ingestion mixins share.
-
-``IngestionPipeline`` is assembled from the routing, folder and jobs mixins.
-Each of them reads constructor state it does not own and calls back into
-``ingest`` — the point of the split is that "how one picture is stored" and
-"how a folder is walked" stop sharing a thousand-line file, not that they stop
-sharing ``self``.
-
-Typing-only, exactly like :mod:`lattice_brain.graph._kg_contract`: at runtime
-the mixins alias it to ``object``, so the MRO and every method resolution stay
-byte-for-byte what the single-file class had. Adding a cross-mixin call without
-declaring it here is a type error.
-"""
-
-from __future__ import annotations
-
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-
-from ..ingestion_jobs import BackgroundIngestionJob, BackgroundIngestionQueue
-from ..multimodal import MultimodalPorts
-from .models import IngestionItem, IngestionResult
-
-
-class IngestionCore:
-    """What any ingestion mixin may assume about ``self``.
-
-    Never instantiated and never inherited at runtime — see the module
-    docstring. Members are declared, not implemented: the implementation lives
-    in ``pipeline.py`` or in whichever mixin owns it.
-    """
-
-    # ── State owned by IngestionPipeline.__init__ ────────────────────────────
-    _kg: Any
-    _hooks: Any
-    _enable: bool
-    _audit: Optional[Any]
-    _max_text_bytes: int
-    _pipeline_name: str
-    _bg_queue: BackgroundIngestionQueue
-    _auto_vector_index_opt_in: bool
-    _multimodal_opt_in: bool
-    _multimodal: MultimodalPorts
-    _keyframes: int
-
-    # ── pipeline.py: the gates, resolved when they are asked ─────────────────
-    @property
-    def _allow_multimodal(self) -> bool:
-        raise NotImplementedError
-
-    @property
-    def _allow_video(self) -> bool:
-        raise NotImplementedError
-
-    # ── pipeline.py: the one door every mixin routes back through ────────────
-    def available(self) -> bool:
-        raise NotImplementedError
-
-    def ingest(
-        self, item: IngestionItem, *, user_email: Optional[str] = None
-    ) -> IngestionResult:
-        raise NotImplementedError
-
-    # ── routing.py: reached from the folder walk and the modality doors ──────
-    def _resolve_file_path(self, item: IngestionItem) -> Path:
-        raise NotImplementedError
-
-    # ── folders.py: the folder-scan allow-list the multimodal gates widen ────
-    def _folder_extensions(self) -> frozenset:
-        raise NotImplementedError
-
-    # ── jobs_api.py: scheduling, reached from the folder walk ────────────────
-    def schedule_background(
-        self,
-        items: List[IngestionItem],
-        *,
-        incremental: bool = True,
-        user_email: Optional[str] = None,
-    ) -> BackgroundIngestionJob:
-        raise NotImplementedError
-
-    def run_background_job(
-        self, job_id: str, *, user_email: Optional[str] = None
-    ) -> Dict[str, Any]:
-        raise NotImplementedError
-
-    def _execute_background_job(
-        self, job: BackgroundIngestionJob, *, user_email: Optional[str] = None
-    ) -> Dict[str, Any]:
-        raise NotImplementedError

package/lattice_brain/ingestion/folder_scan.py

@@ -1,57 +0,0 @@
-"""``.latticeignore`` parsing and matching for the folder walk.
-
-A gitignore-like subset: blank lines and ``#`` comments are dropped, patterns
-are ``fnmatch`` globs, and a trailing ``/`` restricts a pattern to directories.
-Patterns match against both the root-relative posix path and the basename, so
-``*.log`` and ``docs/draft.md`` both behave the way a reader expects.
-"""
-
-from __future__ import annotations
-
-import fnmatch
-from pathlib import Path
-from typing import Iterable, List
-
-from .constants import LATTICEIGNORE_FILENAME
-
-
-def _load_latticeignore(root: Path) -> List[str]:
-    """Parse ``root/.latticeignore`` → glob patterns (gitignore-like subset)."""
-    ignore_file = root / LATTICEIGNORE_FILENAME
-    patterns: List[str] = []
-    if not ignore_file.is_file():
-        return patterns
-    try:
-        lines = ignore_file.read_text(encoding="utf-8", errors="ignore").splitlines()
-    except OSError:
-        return patterns
-    for raw in lines:
-        line = raw.strip()
-        if not line or line.startswith("#"):
-            continue
-        patterns.append(line)
-    return patterns
-
-
-def _matches_ignore(
-    rel_posix: str, name: str, *, is_dir: bool, patterns: Iterable[str]
-) -> bool:
-    """fnmatch-based .latticeignore matching.
-
-    - ``pattern/`` matches directories only (files under it never appear
-      because ignored directories are pruned during the walk).
-    - Patterns match against both the root-relative posix path and the
-      basename, so ``*.log`` and ``docs/draft.md`` both behave as expected.
-    """
-    for raw in patterns:
-        pattern = raw
-        if pattern.endswith("/"):
-            if not is_dir:
-                continue
-            pattern = pattern.rstrip("/")
-        pattern = pattern.lstrip("/")
-        if not pattern:
-            continue
-        if fnmatch.fnmatch(rel_posix, pattern) or fnmatch.fnmatch(name, pattern):
-            return True
-    return False

package/lattice_brain/ingestion/folders.py

@@ -1,258 +0,0 @@
-"""Walking a folder, and the one-page web hand-off, into the standard door.
-
-Neither method is a second ingest path: both build ordinary ``IngestionItem``
-values and hand them to ``IngestionPipeline.ingest``. The folder walk owns the
-filtering order (hard skip-list → hidden → ``.latticeignore`` → extension →
-size) and the choice between ingesting inline and scheduling in the background;
-the web hand-off owns the refusal to fetch or parse anything itself.
-"""
-
-from __future__ import annotations
-
-import os
-from pathlib import Path
-from typing import Any, Dict, Iterable, List, Optional
-
-from ._contract import IngestionCore as _Core
-from .constants import (
-    DEFAULT_FOLDER_EXTENSIONS,
-    DEFAULT_MAX_FILE_BYTES,
-    FOLDER_DEFAULT_SKIP_DIRS,
-    FOLDER_DOCUMENT_EXTENSIONS,
-    FOLDER_MULTIMODAL_EXTENSIONS,
-    FOLDER_VIDEO_EXTENSIONS,
-    LATTICEIGNORE_FILENAME,
-)
-from .folder_scan import _load_latticeignore, _matches_ignore
-from .models import IngestionItem, IngestionResult
-
-
-class IngestionFolderMixin(_Core):
-    """Folder walk + web hand-off. Mixed into ``IngestionPipeline``."""
-
-    def ingest_web_page(
-        self,
-        url: str,
-        extracted_text: str,
-        *,
-        title: Optional[str] = None,
-        metadata: Optional[Dict[str, Any]] = None,
-        owner: Optional[str] = None,
-        workspace_id: Optional[str] = None,
-        captured_at: Optional[str] = None,
-        user_email: Optional[str] = None,
-    ) -> IngestionResult:
-        """Ingest an *already-extracted* web page (see module docstring seam).
-
-        Fetching/parsing is upstream's responsibility (browser extension /
-        tools layer); this wrapper only normalizes ``(url, extracted_text)``
-        into an ``IngestionItem(source_type="web_url")`` and routes it through
-        the standard :meth:`ingest` door.
-        """
-        url = str(url or "").strip()
-        if not url:
-            return IngestionResult(
-                status="failed", source_type="web_url",
-                indexing_status="skipped", detail="url required",
-            )
-        text = str(extracted_text or "")
-        if not text.strip():
-            return IngestionResult(
-                status="failed", source_type="web_url",
-                indexing_status="skipped",
-                detail=(
-                    "extracted_text required — the graph layer does not fetch or "
-                    "parse the web; extraction happens upstream."
-                ),
-            )
-        item = IngestionItem(
-            source_type="web_url",
-            title=title or url,
-            text=text,
-            source_uri=url,
-            owner=owner,
-            workspace_id=workspace_id,
-            captured_at=captured_at,
-            metadata=dict(metadata or {}),
-        )
-        return self.ingest(item, user_email=user_email or owner)
-
-    def ingest_folder(
-        self,
-        root_path: Any,
-        *,
-        recursive: bool = True,
-        background: bool = False,
-        extensions: Optional[Iterable[str]] = None,
-        max_file_bytes: int = DEFAULT_MAX_FILE_BYTES,
-        include_hidden: bool = False,
-        max_files: int = 1000,
-        max_errors: int = 25,
-        owner: Optional[str] = None,
-        workspace_id: Optional[str] = None,
-        user_email: Optional[str] = None,
-    ) -> Dict[str, Any]:
-        """Walk ``root_path`` and ingest every eligible file through the pipeline.
-
-        Filtering, in order: hard skip-list directories (``.git`` …), hidden
-        entries (unless ``include_hidden``), root ``.latticeignore`` patterns
-        (fnmatch globs; ``dir/`` suffix prunes directories), extension
-        allow-list, then ``max_file_bytes``. Text/code files are read inline so
-        their content is chunked; ``.pdf`` routes through the file door without
-        inline extraction.
-
-        ``background=True`` schedules the built items on the existing
-        :class:`BackgroundIngestionQueue` instead of ingesting inline.
-        Returns a summary dict with counts and per-file errors (capped at
-        ``max_errors``).
-        """
-        summary: Dict[str, Any] = {
-            "root": str(root_path),
-            "recursive": bool(recursive),
-            "background": bool(background),
-            "scanned": 0,
-            "matched": 0,
-            "ingested": 0,
-            "duplicate": 0,
-            "failed": 0,
-            "skipped": {"ignored": 0, "extension": 0, "too_large": 0, "hidden": 0},
-            "truncated": False,
-            "errors": [],
-        }
-        try:
-            root = Path(root_path).expanduser()
-        except TypeError:
-            summary.update(status="failed", detail=f"invalid root path: {root_path!r}")
-            return summary
-        if not root.is_dir():
-            summary.update(status="failed", detail=f"not a directory: {root}")
-            return summary
-        if not self.available():
-            summary.update(
-                status="unavailable",
-                detail="Knowledge Graph is disabled (LATTICEAI_ENABLE_GRAPH).",
-            )
-            return summary
-        summary["root"] = str(root)
-        max_files = max(1, int(max_files))
-        max_errors = max(0, int(max_errors))
-        max_file_bytes = max(1, int(max_file_bytes))
-        allowed_exts = (
-            frozenset(str(e).lower() if str(e).startswith(".") else f".{str(e).lower()}" for e in extensions)
-            if extensions
-            else self._folder_extensions()
-        )
-        patterns = _load_latticeignore(root)
-        errors: List[Dict[str, Any]] = summary["errors"]
-        skipped = summary["skipped"]
-        items: List[IngestionItem] = []
-
-        def _record_error(path: Path, detail: str, status: str = "failed") -> None:
-            summary["failed"] += 1
-            if len(errors) < max_errors:
-                errors.append({"path": str(path), "status": status, "detail": detail})
-
-        for dirpath, dirnames, filenames in os.walk(root):
-            current = Path(dirpath)
-            rel_dir = current.relative_to(root)
-            kept_dirs: List[str] = []
-            for name in sorted(dirnames):
-                if name in FOLDER_DEFAULT_SKIP_DIRS:
-                    continue
-                if name.startswith(".") and not include_hidden:
-                    continue
-                rel = name if str(rel_dir) == "." else (rel_dir / name).as_posix()
-                if _matches_ignore(rel, name, is_dir=True, patterns=patterns):
-                    skipped["ignored"] += 1
-                    continue
-                kept_dirs.append(name)
-            dirnames[:] = kept_dirs if recursive else []
-
-            for name in sorted(filenames):
-                if name == LATTICEIGNORE_FILENAME:
-                    continue
-                summary["scanned"] += 1
-                path = current / name
-                rel = name if str(rel_dir) == "." else (rel_dir / name).as_posix()
-                if name.startswith(".") and not include_hidden:
-                    skipped["hidden"] += 1
-                    continue
-                if _matches_ignore(rel, name, is_dir=False, patterns=patterns):
-                    skipped["ignored"] += 1
-                    continue
-                ext = path.suffix.lower()
-                if ext not in allowed_exts:
-                    skipped["extension"] += 1
-                    continue
-                try:
-                    size = path.stat().st_size
-                except OSError as exc:
-                    _record_error(path, f"stat failed: {exc}")
-                    continue
-                if size > max_file_bytes:
-                    skipped["too_large"] += 1
-                    continue
-                if len(items) >= max_files:
-                    summary["truncated"] = True
-                    break
-                item_metadata: Dict[str, Any] = {"relative_path": rel}
-                if ext in (FOLDER_MULTIMODAL_EXTENSIONS | FOLDER_VIDEO_EXTENSIONS) and self._allow_multimodal:
-                    # Routed by modality inside ``ingest``; reading the bytes as
-                    # UTF-8 here would only produce mojibake.
-                    source_type = "file"
-                elif ext in FOLDER_DOCUMENT_EXTENSIONS:
-                    source_type = "pdf"
-                else:
-                    source_type = "file"
-                    try:
-                        content = path.read_text(encoding="utf-8", errors="ignore")
-                    except OSError as exc:
-                        _record_error(path, f"read failed: {exc}")
-                        continue
-                    item_metadata["extracted"] = {"content": content, "chars": len(content)}
-                items.append(
-                    IngestionItem(
-                        source_type=source_type,
-                        title=name,
-                        path=str(path),
-                        source_uri=str(path),
-                        owner=owner,
-                        workspace_id=workspace_id,
-                        metadata=item_metadata,
-                    )
-                )
-            if summary["truncated"]:
-                break
-
-        summary["matched"] = len(items)
-        if background:
-            job = self.schedule_background(
-                items, incremental=True, user_email=user_email or owner,
-            )
-            summary.update(status="scheduled", job_id=job.job_id, scheduled=len(items))
-            return summary
-
-        for item in items:
-            result = self.ingest(item, user_email=user_email or owner)
-            if result.status == "ok":
-                if result.duplicate:
-                    summary["duplicate"] += 1
-                else:
-                    summary["ingested"] += 1
-            else:
-                _record_error(Path(item.path or ""), result.detail or result.status, result.status)
-        summary["status"] = "ok" if summary["failed"] == 0 else "partial"
-        return summary
-
-    def _folder_extensions(self) -> frozenset:
-        """Folder-scan allow-list — pictures, recordings and films when enabled.
-
-        Video joins only when this machine can actually decode one, so a scan
-        never fills the error list with files it was always going to refuse.
-        """
-        if not self._allow_multimodal:
-            return DEFAULT_FOLDER_EXTENSIONS
-        allowed = DEFAULT_FOLDER_EXTENSIONS | FOLDER_MULTIMODAL_EXTENSIONS
-        if self._allow_video:
-            return allowed | FOLDER_VIDEO_EXTENSIONS
-        return allowed

package/lattice_brain/ingestion/jobs_api.py

@@ -1,107 +0,0 @@
-"""Scheduling many items, and running (or resuming) what was scheduled.
-
-The pipeline owns "ingest one item"; ``BackgroundIngestionQueue`` owns
-"schedule many and report progress". This mixin is the seam between them:
-per-item errors are recorded and never abort a job, progress is checkpointed
-after every item, and the same method powers both the first run and a resume
-because already-completed items are simply skipped.
-"""
-
-from __future__ import annotations
-
-from typing import Any, Dict, List, Optional
-
-from ..ingestion_jobs import BackgroundIngestionJob
-from ._contract import IngestionCore as _Core
-from .models import IngestionItem
-
-
-class IngestionJobsMixin(_Core):
-    """Background job scheduling. Mixed into ``IngestionPipeline``."""
-
-    # --- Large candidate #1: background / incremental scheduling (slice) ---
-    def schedule_background(
-        self,
-        items: List[IngestionItem],
-        *,
-        incremental: bool = True,
-        user_email: Optional[str] = None,
-    ) -> BackgroundIngestionJob:
-        """Schedule items for background incremental indexing.
-
-        Returns a job handle. Actual execution can be driven by caller
-        (or future worker) calling pipeline.ingest on each — or through
-        :meth:`run_background_job`. This seam enables large-corpus scale
-        without blocking user requests.
-        """
-        job = self._bg_queue.schedule(items, incremental=incremental, user_email=user_email)
-        # mark initial status on results concept (jobs track)
-        return job
-
-    def get_background_job(self, job_id: str) -> Optional[BackgroundIngestionJob]:
-        return self._bg_queue.get(job_id)
-
-    def list_background_jobs(self, limit: int = 20) -> List[Dict[str, Any]]:
-        """Recent jobs (newest first) in the frozen ``/api/ingestion`` schema."""
-        return [job.as_dict() for job in self._bg_queue.list_recent(limit=limit)]
-
-    def run_background_job(
-        self, job_id: str, *, user_email: Optional[str] = None
-    ) -> Dict[str, Any]:
-        """Execute a queued/interrupted job's remaining items.
-
-        Per-item errors are recorded (capped) and never abort the job. The
-        final status is ``completed`` (all done), ``partial`` (some done),
-        or ``failed`` (nothing done). Already-completed items are skipped, so
-        the same method safely powers both first-run and resume.
-        """
-        job = self._bg_queue.get(job_id)
-        if job is None:
-            return {"status": "not_found", "job_id": job_id}
-        if job.status == "running":
-            return job.as_dict()
-        return self._execute_background_job(job, user_email=user_email)
-
-    def resume_background_job(
-        self, job_id: str, *, user_email: Optional[str] = None
-    ) -> Dict[str, Any]:
-        """Resume an interrupted/partial/failed job from its remaining items."""
-        return self.run_background_job(job_id, user_email=user_email)
-
-    def _execute_background_job(
-        self, job: BackgroundIngestionJob, *, user_email: Optional[str] = None
-    ) -> Dict[str, Any]:
-        job.status = "running"
-        # Retried items get a fresh verdict: reset failure state for this run.
-        job.failed = 0
-        job.errors = []
-        job.touch()
-        self._bg_queue.save(job)
-        runner_email = user_email or job.user_email
-        for index in job.remaining_indices():
-            item = job.items[index]
-            try:
-                result = self.ingest(item, user_email=runner_email or item.owner)
-                status, detail = result.status, result.detail
-            except Exception as exc:  # noqa: BLE001 — per-item isolation: keep going
-                status, detail = "failed", str(exc)
-            if status == "ok":
-                job.done_indices.add(index)
-            else:
-                job.record_error(index, item, detail or status)
-            job.processed = len(job.done_indices)
-            job.touch()
-            # Checkpoint per item: a crash here must cost at most the item in
-            # flight, never the whole job's progress. One small UPDATE against
-            # an ingest (parse + chunk + embed) is noise.
-            self._bg_queue.save(job)
-        job.processed = len(job.done_indices)
-        if job.total == 0 or job.processed >= job.total:
-            job.status = "completed"
-        elif job.processed > 0:
-            job.status = "partial"
-        else:
-            job.status = "failed"
-        job.touch()
-        self._bg_queue.save(job)
-        return job.as_dict()