ltcai 11.2.0 → 11.4.0

package/lattice_brain/ingestion/constants.py

@@ -0,0 +1,127 @@
+"""What routes where, what a folder scan admits, and which gates decide.
+
+Source-type sets, folder-scan filters, size budgets and the four feature gates,
+with no logic beyond them. Every other submodule imports this one; this one
+imports nothing from the package, which is what keeps the layering acyclic.
+"""
+
+from __future__ import annotations
+
+from ..gates import FeatureGate
+from ..multimodal import AUDIO_EXTENSIONS, IMAGE_EXTENSIONS, VIDEO_EXTENSIONS
+
+# Source types that arrive as a file on disk (read via ingest_document).
+FILE_SOURCE_TYPES = frozenset({"file", "local_file", "upload", "pdf"})
+# Source types that arrive as extracted text (read via ingest_source).
+TEXT_SOURCE_TYPES = frozenset(
+    {"web_url", "browser_tab", "text", "markdown", "note", "code", "clipboard"}
+)
+# Conversational exchanges (read via ingest_message — role/content semantics,
+# conversation chaining). v4: chat and MCP messages stop bypassing the
+# pipeline, so they carry provenance and fire the hook lifecycle like every
+# other source.
+CHAT_SOURCE_TYPES = frozenset({"chat_message", "mcp_message"})
+# Typed memory records (read via ingest_event → Decision/Experience/Event
+# nodes). The Memory System writes through the same door as everything else.
+MEMORY_SOURCE_TYPES = frozenset({"decision", "experience", "workspace_event"})
+_MEMORY_NODE_TYPES = {"decision": "Decision", "experience": "Experience", "workspace_event": "Event"}
+
+DEFAULT_MAX_TEXT_BYTES = 5 * 1024 * 1024  # 5 MB of extracted text per item
+
+
+# ── Folder ingestion (ingest_folder) filters ─────────────────────────────────
+# Directories that are always pruned regardless of .latticeignore.
+FOLDER_DEFAULT_SKIP_DIRS = frozenset(
+    {
+        ".git",
+        "node_modules",
+        "__pycache__",
+        ".venv",
+        "venv",
+        "env",
+        ".pytest_cache",
+        ".mypy_cache",
+        ".ruff_cache",
+        "dist",
+        "build",
+        ".next",
+        "target",
+        ".cache",
+        ".idea",
+        ".vscode",
+    }
+)
+# Extension filter matching FILE_SOURCE_TYPES conventions: text/markdown/code
+# are read inline (extracted content → chunks); .pdf routes as source_type
+# "pdf" through ingest_document (content extraction is upstream's concern).
+FOLDER_TEXT_EXTENSIONS = frozenset(
+    {".txt", ".md", ".markdown", ".rst", ".csv", ".json", ".yaml", ".yml", ".toml", ".ini"}
+)
+FOLDER_CODE_EXTENSIONS = frozenset(
+    {
+        ".py", ".js", ".ts", ".tsx", ".jsx", ".html", ".css", ".go", ".rs",
+        ".java", ".c", ".h", ".cpp", ".hpp", ".rb", ".php", ".swift", ".kt",
+        ".sh", ".sql",
+    }
+)
+FOLDER_DOCUMENT_EXTENSIONS = frozenset({".pdf"})
+DEFAULT_FOLDER_EXTENSIONS = (
+    FOLDER_TEXT_EXTENSIONS | FOLDER_CODE_EXTENSIONS | FOLDER_DOCUMENT_EXTENSIONS
+)
+DEFAULT_MAX_FILE_BYTES = 4_000_000  # matches the local-index text/code budget
+LATTICEIGNORE_FILENAME = ".latticeignore"
+# Opt-out escape hatch for the post-ingest incremental vector sync.
+AUTO_VECTOR_INDEX_ENV = "LATTICEAI_AUTO_VECTOR_INDEX"
+#: Default *on*, unlike every other gate here: new material has always been made
+#: searchable straight away, and this exists so a settings surface can turn that
+#: off (batch reindex later) without a restart. ``FeatureGate`` parses the env
+#: var with the same words the hand-written opt-out check used, so an untouched
+#: install — including one with a nonsense value — answers exactly as before.
+AUTO_VECTOR_INDEX_GATE = FeatureGate(
+    AUTO_VECTOR_INDEX_ENV,
+    default=True,
+    name="auto_vector_index",
+    detail="New material is prepared for semantic search as soon as it lands.",
+)
+
+# ── Multi-modal ingestion (v11.1.0 Track 3) ──────────────────────────────────
+# Opt-in, default off, on purpose. Turning it on changes what a folder scan
+# *stores* (pictures and recordings, with OCR and — if a model is loaded —
+# captions and vectors), and that is the user's call, not a default. With the
+# flag off every routing decision below is skipped and behaviour is byte-for-
+# byte what it was before this release.
+ALLOW_MULTIMODAL_ENV = "LATTICEAI_ALLOW_MULTIMODAL"
+#: The multi-modal switch, resolved when it is asked rather than frozen into
+#: ``self`` at construction (v11.2.0). The environment variable is still the
+#: answer for an untouched install — same var, same words, same default off —
+#: but a settings surface can bind a resolver and move it without a restart.
+MULTIMODAL_GATE = FeatureGate(
+    ALLOW_MULTIMODAL_ENV,
+    default=False,
+    name="allow_multimodal",
+    detail="Pictures and recordings are only ingested when this is turned on.",
+)
+#: Video is a *sub-switch* of the one above: with multi-modal off nothing about
+#: video happens at all, and with it on video is included unless this is
+#: explicitly turned off. The effective default is therefore still "no video",
+#: and the seam exists so a settings screen can offer pictures without films.
+ALLOW_VIDEO_ENV = "LATTICEAI_ALLOW_VIDEO"
+VIDEO_GATE = FeatureGate(
+    ALLOW_VIDEO_ENV,
+    default=True,
+    name="allow_video",
+    detail="Videos are ingested as keyframes plus subtitles when multi-modal is on.",
+)
+#: Source types that name a modality outright (a caller who already knows).
+IMAGE_SOURCE_TYPES = frozenset({"image", "screenshot", "photo"})
+AUDIO_SOURCE_TYPES = frozenset({"audio", "voice_memo", "recording"})
+VIDEO_SOURCE_TYPES = frozenset({"video", "screen_recording", "movie"})
+#: Added to the folder-scan allow-list only while multimodal is enabled.
+FOLDER_MULTIMODAL_EXTENSIONS = IMAGE_EXTENSIONS | AUDIO_EXTENSIONS
+#: Videos join the folder allow-list only when this machine can decode one —
+#: scanning a folder into a pile of refusals is not a feature.
+FOLDER_VIDEO_EXTENSIONS = VIDEO_EXTENSIONS
+#: Graph node type for a recording. ``NodeType.AUDIO`` normalizes this on the
+#: KG v2 write side; the legacy tables keep the label verbatim, which is what
+#: every type-aware read (graph view, context sections, doc-gen) matches on.
+AUDIO_NODE_TYPE = "Audio"

package/lattice_brain/ingestion/folder_scan.py

@@ -0,0 +1,57 @@
+"""``.latticeignore`` parsing and matching for the folder walk.
+
+A gitignore-like subset: blank lines and ``#`` comments are dropped, patterns
+are ``fnmatch`` globs, and a trailing ``/`` restricts a pattern to directories.
+Patterns match against both the root-relative posix path and the basename, so
+``*.log`` and ``docs/draft.md`` both behave the way a reader expects.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+from pathlib import Path
+from typing import Iterable, List
+
+from .constants import LATTICEIGNORE_FILENAME
+
+
+def _load_latticeignore(root: Path) -> List[str]:
+    """Parse ``root/.latticeignore`` → glob patterns (gitignore-like subset)."""
+    ignore_file = root / LATTICEIGNORE_FILENAME
+    patterns: List[str] = []
+    if not ignore_file.is_file():
+        return patterns
+    try:
+        lines = ignore_file.read_text(encoding="utf-8", errors="ignore").splitlines()
+    except OSError:
+        return patterns
+    for raw in lines:
+        line = raw.strip()
+        if not line or line.startswith("#"):
+            continue
+        patterns.append(line)
+    return patterns
+
+
+def _matches_ignore(
+    rel_posix: str, name: str, *, is_dir: bool, patterns: Iterable[str]
+) -> bool:
+    """fnmatch-based .latticeignore matching.
+
+    - ``pattern/`` matches directories only (files under it never appear
+      because ignored directories are pruned during the walk).
+    - Patterns match against both the root-relative posix path and the
+      basename, so ``*.log`` and ``docs/draft.md`` both behave as expected.
+    """
+    for raw in patterns:
+        pattern = raw
+        if pattern.endswith("/"):
+            if not is_dir:
+                continue
+            pattern = pattern.rstrip("/")
+        pattern = pattern.lstrip("/")
+        if not pattern:
+            continue
+        if fnmatch.fnmatch(rel_posix, pattern) or fnmatch.fnmatch(name, pattern):
+            return True
+    return False

package/lattice_brain/ingestion/folders.py

@@ -0,0 +1,258 @@
+"""Walking a folder, and the one-page web hand-off, into the standard door.
+
+Neither method is a second ingest path: both build ordinary ``IngestionItem``
+values and hand them to ``IngestionPipeline.ingest``. The folder walk owns the
+filtering order (hard skip-list → hidden → ``.latticeignore`` → extension →
+size) and the choice between ingesting inline and scheduling in the background;
+the web hand-off owns the refusal to fetch or parse anything itself.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional
+
+from ._contract import IngestionCore as _Core
+from .constants import (
+    DEFAULT_FOLDER_EXTENSIONS,
+    DEFAULT_MAX_FILE_BYTES,
+    FOLDER_DEFAULT_SKIP_DIRS,
+    FOLDER_DOCUMENT_EXTENSIONS,
+    FOLDER_MULTIMODAL_EXTENSIONS,
+    FOLDER_VIDEO_EXTENSIONS,
+    LATTICEIGNORE_FILENAME,
+)
+from .folder_scan import _load_latticeignore, _matches_ignore
+from .models import IngestionItem, IngestionResult
+
+
+class IngestionFolderMixin(_Core):
+    """Folder walk + web hand-off. Mixed into ``IngestionPipeline``."""
+
+    def ingest_web_page(
+        self,
+        url: str,
+        extracted_text: str,
+        *,
+        title: Optional[str] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        owner: Optional[str] = None,
+        workspace_id: Optional[str] = None,
+        captured_at: Optional[str] = None,
+        user_email: Optional[str] = None,
+    ) -> IngestionResult:
+        """Ingest an *already-extracted* web page (see module docstring seam).
+
+        Fetching/parsing is upstream's responsibility (browser extension /
+        tools layer); this wrapper only normalizes ``(url, extracted_text)``
+        into an ``IngestionItem(source_type="web_url")`` and routes it through
+        the standard :meth:`ingest` door.
+        """
+        url = str(url or "").strip()
+        if not url:
+            return IngestionResult(
+                status="failed", source_type="web_url",
+                indexing_status="skipped", detail="url required",
+            )
+        text = str(extracted_text or "")
+        if not text.strip():
+            return IngestionResult(
+                status="failed", source_type="web_url",
+                indexing_status="skipped",
+                detail=(
+                    "extracted_text required — the graph layer does not fetch or "
+                    "parse the web; extraction happens upstream."
+                ),
+            )
+        item = IngestionItem(
+            source_type="web_url",
+            title=title or url,
+            text=text,
+            source_uri=url,
+            owner=owner,
+            workspace_id=workspace_id,
+            captured_at=captured_at,
+            metadata=dict(metadata or {}),
+        )
+        return self.ingest(item, user_email=user_email or owner)
+
+    def ingest_folder(
+        self,
+        root_path: Any,
+        *,
+        recursive: bool = True,
+        background: bool = False,
+        extensions: Optional[Iterable[str]] = None,
+        max_file_bytes: int = DEFAULT_MAX_FILE_BYTES,
+        include_hidden: bool = False,
+        max_files: int = 1000,
+        max_errors: int = 25,
+        owner: Optional[str] = None,
+        workspace_id: Optional[str] = None,
+        user_email: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Walk ``root_path`` and ingest every eligible file through the pipeline.
+
+        Filtering, in order: hard skip-list directories (``.git`` …), hidden
+        entries (unless ``include_hidden``), root ``.latticeignore`` patterns
+        (fnmatch globs; ``dir/`` suffix prunes directories), extension
+        allow-list, then ``max_file_bytes``. Text/code files are read inline so
+        their content is chunked; ``.pdf`` routes through the file door without
+        inline extraction.
+
+        ``background=True`` schedules the built items on the existing
+        :class:`BackgroundIngestionQueue` instead of ingesting inline.
+        Returns a summary dict with counts and per-file errors (capped at
+        ``max_errors``).
+        """
+        summary: Dict[str, Any] = {
+            "root": str(root_path),
+            "recursive": bool(recursive),
+            "background": bool(background),
+            "scanned": 0,
+            "matched": 0,
+            "ingested": 0,
+            "duplicate": 0,
+            "failed": 0,
+            "skipped": {"ignored": 0, "extension": 0, "too_large": 0, "hidden": 0},
+            "truncated": False,
+            "errors": [],
+        }
+        try:
+            root = Path(root_path).expanduser()
+        except TypeError:
+            summary.update(status="failed", detail=f"invalid root path: {root_path!r}")
+            return summary
+        if not root.is_dir():
+            summary.update(status="failed", detail=f"not a directory: {root}")
+            return summary
+        if not self.available():
+            summary.update(
+                status="unavailable",
+                detail="Knowledge Graph is disabled (LATTICEAI_ENABLE_GRAPH).",
+            )
+            return summary
+        summary["root"] = str(root)
+        max_files = max(1, int(max_files))
+        max_errors = max(0, int(max_errors))
+        max_file_bytes = max(1, int(max_file_bytes))
+        allowed_exts = (
+            frozenset(str(e).lower() if str(e).startswith(".") else f".{str(e).lower()}" for e in extensions)
+            if extensions
+            else self._folder_extensions()
+        )
+        patterns = _load_latticeignore(root)
+        errors: List[Dict[str, Any]] = summary["errors"]
+        skipped = summary["skipped"]
+        items: List[IngestionItem] = []
+
+        def _record_error(path: Path, detail: str, status: str = "failed") -> None:
+            summary["failed"] += 1
+            if len(errors) < max_errors:
+                errors.append({"path": str(path), "status": status, "detail": detail})
+
+        for dirpath, dirnames, filenames in os.walk(root):
+            current = Path(dirpath)
+            rel_dir = current.relative_to(root)
+            kept_dirs: List[str] = []
+            for name in sorted(dirnames):
+                if name in FOLDER_DEFAULT_SKIP_DIRS:
+                    continue
+                if name.startswith(".") and not include_hidden:
+                    continue
+                rel = name if str(rel_dir) == "." else (rel_dir / name).as_posix()
+                if _matches_ignore(rel, name, is_dir=True, patterns=patterns):
+                    skipped["ignored"] += 1
+                    continue
+                kept_dirs.append(name)
+            dirnames[:] = kept_dirs if recursive else []
+
+            for name in sorted(filenames):
+                if name == LATTICEIGNORE_FILENAME:
+                    continue
+                summary["scanned"] += 1
+                path = current / name
+                rel = name if str(rel_dir) == "." else (rel_dir / name).as_posix()
+                if name.startswith(".") and not include_hidden:
+                    skipped["hidden"] += 1
+                    continue
+                if _matches_ignore(rel, name, is_dir=False, patterns=patterns):
+                    skipped["ignored"] += 1
+                    continue
+                ext = path.suffix.lower()
+                if ext not in allowed_exts:
+                    skipped["extension"] += 1
+                    continue
+                try:
+                    size = path.stat().st_size
+                except OSError as exc:
+                    _record_error(path, f"stat failed: {exc}")
+                    continue
+                if size > max_file_bytes:
+                    skipped["too_large"] += 1
+                    continue
+                if len(items) >= max_files:
+                    summary["truncated"] = True
+                    break
+                item_metadata: Dict[str, Any] = {"relative_path": rel}
+                if ext in (FOLDER_MULTIMODAL_EXTENSIONS | FOLDER_VIDEO_EXTENSIONS) and self._allow_multimodal:
+                    # Routed by modality inside ``ingest``; reading the bytes as
+                    # UTF-8 here would only produce mojibake.
+                    source_type = "file"
+                elif ext in FOLDER_DOCUMENT_EXTENSIONS:
+                    source_type = "pdf"
+                else:
+                    source_type = "file"
+                    try:
+                        content = path.read_text(encoding="utf-8", errors="ignore")
+                    except OSError as exc:
+                        _record_error(path, f"read failed: {exc}")
+                        continue
+                    item_metadata["extracted"] = {"content": content, "chars": len(content)}
+                items.append(
+                    IngestionItem(
+                        source_type=source_type,
+                        title=name,
+                        path=str(path),
+                        source_uri=str(path),
+                        owner=owner,
+                        workspace_id=workspace_id,
+                        metadata=item_metadata,
+                    )
+                )
+            if summary["truncated"]:
+                break
+
+        summary["matched"] = len(items)
+        if background:
+            job = self.schedule_background(
+                items, incremental=True, user_email=user_email or owner,
+            )
+            summary.update(status="scheduled", job_id=job.job_id, scheduled=len(items))
+            return summary
+
+        for item in items:
+            result = self.ingest(item, user_email=user_email or owner)
+            if result.status == "ok":
+                if result.duplicate:
+                    summary["duplicate"] += 1
+                else:
+                    summary["ingested"] += 1
+            else:
+                _record_error(Path(item.path or ""), result.detail or result.status, result.status)
+        summary["status"] = "ok" if summary["failed"] == 0 else "partial"
+        return summary
+
+    def _folder_extensions(self) -> frozenset:
+        """Folder-scan allow-list — pictures, recordings and films when enabled.
+
+        Video joins only when this machine can actually decode one, so a scan
+        never fills the error list with files it was always going to refuse.
+        """
+        if not self._allow_multimodal:
+            return DEFAULT_FOLDER_EXTENSIONS
+        allowed = DEFAULT_FOLDER_EXTENSIONS | FOLDER_MULTIMODAL_EXTENSIONS
+        if self._allow_video:
+            return allowed | FOLDER_VIDEO_EXTENSIONS
+        return allowed

package/lattice_brain/ingestion/hashing.py

@@ -0,0 +1,26 @@
+"""The two content hashes the pipeline names things by.
+
+:func:`content_hash_text` matches the store's own hashing scheme, so a text
+payload hashed here and a text payload hashed there dedupe against each other.
+:func:`_file_digest` streams a file instead of reading it whole — it is the key
+a video's keyframe folder is named by, and videos are large.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+
+
+def content_hash_text(text: str) -> str:
+    """Canonical content hash for a text payload (matches store hashing scheme)."""
+    return hashlib.sha256((text or "").encode("utf-8", "ignore")).hexdigest()
+
+
+def _file_digest(path: Path) -> str:
+    """Streaming sha256 of a file — the key a video's frame folder is named by."""
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for block in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(block)
+    return digest.hexdigest()

package/lattice_brain/ingestion/jobs_api.py

@@ -0,0 +1,107 @@
+"""Scheduling many items, and running (or resuming) what was scheduled.
+
+The pipeline owns "ingest one item"; ``BackgroundIngestionQueue`` owns
+"schedule many and report progress". This mixin is the seam between them:
+per-item errors are recorded and never abort a job, progress is checkpointed
+after every item, and the same method powers both the first run and a resume
+because already-completed items are simply skipped.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from ..ingestion_jobs import BackgroundIngestionJob
+from ._contract import IngestionCore as _Core
+from .models import IngestionItem
+
+
+class IngestionJobsMixin(_Core):
+    """Background job scheduling. Mixed into ``IngestionPipeline``."""
+
+    # --- Large candidate #1: background / incremental scheduling (slice) ---
+    def schedule_background(
+        self,
+        items: List[IngestionItem],
+        *,
+        incremental: bool = True,
+        user_email: Optional[str] = None,
+    ) -> BackgroundIngestionJob:
+        """Schedule items for background incremental indexing.
+
+        Returns a job handle. Actual execution can be driven by caller
+        (or future worker) calling pipeline.ingest on each — or through
+        :meth:`run_background_job`. This seam enables large-corpus scale
+        without blocking user requests.
+        """
+        job = self._bg_queue.schedule(items, incremental=incremental, user_email=user_email)
+        # mark initial status on results concept (jobs track)
+        return job
+
+    def get_background_job(self, job_id: str) -> Optional[BackgroundIngestionJob]:
+        return self._bg_queue.get(job_id)
+
+    def list_background_jobs(self, limit: int = 20) -> List[Dict[str, Any]]:
+        """Recent jobs (newest first) in the frozen ``/api/ingestion`` schema."""
+        return [job.as_dict() for job in self._bg_queue.list_recent(limit=limit)]
+
+    def run_background_job(
+        self, job_id: str, *, user_email: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """Execute a queued/interrupted job's remaining items.
+
+        Per-item errors are recorded (capped) and never abort the job. The
+        final status is ``completed`` (all done), ``partial`` (some done),
+        or ``failed`` (nothing done). Already-completed items are skipped, so
+        the same method safely powers both first-run and resume.
+        """
+        job = self._bg_queue.get(job_id)
+        if job is None:
+            return {"status": "not_found", "job_id": job_id}
+        if job.status == "running":
+            return job.as_dict()
+        return self._execute_background_job(job, user_email=user_email)
+
+    def resume_background_job(
+        self, job_id: str, *, user_email: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """Resume an interrupted/partial/failed job from its remaining items."""
+        return self.run_background_job(job_id, user_email=user_email)
+
+    def _execute_background_job(
+        self, job: BackgroundIngestionJob, *, user_email: Optional[str] = None
+    ) -> Dict[str, Any]:
+        job.status = "running"
+        # Retried items get a fresh verdict: reset failure state for this run.
+        job.failed = 0
+        job.errors = []
+        job.touch()
+        self._bg_queue.save(job)
+        runner_email = user_email or job.user_email
+        for index in job.remaining_indices():
+            item = job.items[index]
+            try:
+                result = self.ingest(item, user_email=runner_email or item.owner)
+                status, detail = result.status, result.detail
+            except Exception as exc:  # noqa: BLE001 — per-item isolation: keep going
+                status, detail = "failed", str(exc)
+            if status == "ok":
+                job.done_indices.add(index)
+            else:
+                job.record_error(index, item, detail or status)
+            job.processed = len(job.done_indices)
+            job.touch()
+            # Checkpoint per item: a crash here must cost at most the item in
+            # flight, never the whole job's progress. One small UPDATE against
+            # an ingest (parse + chunk + embed) is noise.
+            self._bg_queue.save(job)
+        job.processed = len(job.done_indices)
+        if job.total == 0 or job.processed >= job.total:
+            job.status = "completed"
+        elif job.processed > 0:
+            job.status = "partial"
+        else:
+            job.status = "failed"
+        job.touch()
+        self._bg_queue.save(job)
+        return job.as_dict()

package/lattice_brain/ingestion/models.py

@@ -0,0 +1,80 @@
+"""The two records the pipeline speaks in: one item in, one result out.
+
+:class:`IngestionItem` is what every source normalizes to before the pipeline
+sees it; :class:`IngestionResult` is what every source normalizes to after. The
+result's ``as_dict`` is the frozen ``/api/ingestion`` payload shape — additive
+keys appear only when populated, so pre-v9.8 consumers see what they always did.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class IngestionItem:
+    """A single thing to ingest, normalized across every source type."""
+
+    source_type: str
+    title: Optional[str] = None
+    text: Optional[str] = None          # text/web sources
+    path: Optional[str] = None          # file sources
+    source_uri: Optional[str] = None
+    mime_type: Optional[str] = None
+    owner: Optional[str] = None
+    workspace_id: Optional[str] = None
+    permissions: Optional[Dict[str, Any]] = None
+    captured_at: Optional[str] = None
+    modified_at: Optional[str] = None
+    conversation_id: Optional[str] = None
+    agent_used: Optional[str] = None
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class IngestionResult:
+    """The outcome of one ingestion, including provenance and idempotency."""
+
+    status: str                         # ok | unavailable | blocked | failed
+    source_type: str
+    node_id: Optional[str] = None
+    source_node_id: Optional[str] = None
+    content_hash: Optional[str] = None
+    title: Optional[str] = None
+    chunk_ids: List[str] = field(default_factory=list)
+    chunk_count: int = 0
+    duplicate: bool = False
+    embedded: bool = False
+    indexing_status: str = "pending"    # indexed | skipped | failed | pending
+    provenance_id: Optional[str] = None
+    detail: Optional[str] = None
+    # v9.8.0 additive quality fields — advisory only, never gate behavior.
+    extraction_quality: Optional[Dict[str, Any]] = None
+    warnings: List[str] = field(default_factory=list)
+    quality_gate: Optional[Dict[str, Any]] = None
+
+    def as_dict(self) -> Dict[str, Any]:
+        payload: Dict[str, Any] = {
+            "status": self.status,
+            "source_type": self.source_type,
+            "node_id": self.node_id,
+            "source_node_id": self.source_node_id,
+            "content_hash": self.content_hash,
+            "title": self.title,
+            "chunk_ids": self.chunk_ids,
+            "chunk_count": self.chunk_count,
+            "duplicate": self.duplicate,
+            "embedded": self.embedded,
+            "indexing_status": self.indexing_status,
+            "provenance_id": self.provenance_id,
+            "detail": self.detail,
+        }
+        # Additive keys only when populated so pre-v9.8 payloads are unchanged.
+        if self.extraction_quality is not None:
+            payload["extraction_quality"] = self.extraction_quality
+        if self.warnings:
+            payload["warnings"] = list(self.warnings)
+        if self.quality_gate is not None:
+            payload["quality_gate"] = self.quality_gate
+        return payload