ltcai 10.6.0 → 10.6.2

package/lattice_brain/ingestion_jobs.py

@@ -9,15 +9,29 @@ progress* — a genuinely separate concern with its own frozen wire schema
 The seam is also where a real scheduler (thread pool, rq, celery) would plug
 in without touching the pipeline.
 
+Job state is **durable** (review 2026-08 P1 #3): ``done_indices`` used to live
+only in this process's heap, so a restart mid-import silently lost the resume
+point and re-ingesting meant replaying every item. :class:`IngestionJobStore`
+persists the queue to SQLite — by default the same database file the knowledge
+graph and :mod:`lattice_brain.conversations` use, so the existing
+backup/restore covers it with no manifest change. A queue built without a
+``db_path`` stays purely in-memory and says so through :meth:`describe`.
+
 ``IngestionItem`` is imported only for type checking: the pipeline module owns
 that dataclass, and a runtime import here would be circular.
 """
 
 from __future__ import annotations
 
+import dataclasses
+import json
+import logging
+import sqlite3
 import threading
+from contextlib import contextmanager
 from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Set
 
 from .utils import utc_now_iso
 
@@ -27,6 +41,9 @@ if TYPE_CHECKING:  # pragma: no cover - typing only
 
 JOB_ERRORS_CAP = 50  # per-job error records kept (failed count keeps counting)
 
+#: Statuses a job may hold on disk. Frozen wire schema of ``/api/ingestion/jobs*``.
+JOB_STATUSES = ("queued", "running", "completed", "failed", "partial")
+
 
 @dataclass
 class BackgroundIngestionJob:
@@ -79,17 +96,231 @@ class BackgroundIngestionJob:
         }
 
 
+def _item_payload(item: IngestionItem) -> Dict[str, Any]:
+    """One ingestion item as JSON-safe data (dataclass fields only)."""
+    return dataclasses.asdict(item)
+
+
+def _item_from_payload(payload: Dict[str, Any]) -> IngestionItem:
+    """Rebuild an ``IngestionItem`` from persisted data.
+
+    Unknown keys are dropped rather than raising: a job written by an older
+    build must still resume on a newer one. Imported here (not at module
+    scope) because ``lattice_brain.ingestion`` imports *this* module.
+    """
+    from .ingestion import IngestionItem as _Item
+
+    known = {f.name for f in dataclasses.fields(_Item)}
+    kwargs = {key: value for key, value in payload.items() if key in known}
+    metadata = kwargs.get("metadata")
+    if not isinstance(metadata, dict):
+        kwargs["metadata"] = {}
+    return _Item(**kwargs)
+
+
+class IngestionJobStore:
+    """SQLite persistence for :class:`BackgroundIngestionJob`.
+
+    Own connection per operation, always closed — ``with sqlite3.connect(...)``
+    commits but never closes (same note as
+    :meth:`lattice_brain.conversations.ConversationStore._connect`).
+    """
+
+    def __init__(self, db_path: Path) -> None:
+        self.db_path = Path(db_path)
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+        self._init_db()
+
+    @contextmanager
+    def _connect(self) -> Iterator[sqlite3.Connection]:
+        conn = sqlite3.connect(str(self.db_path))
+        conn.row_factory = sqlite3.Row
+        conn.execute("PRAGMA journal_mode=WAL")
+        try:
+            with conn:
+                yield conn
+        finally:
+            conn.close()
+
+    def _init_db(self) -> None:
+        with self._connect() as conn:
+            conn.executescript(
+                """
+                CREATE TABLE IF NOT EXISTS ingestion_jobs (
+                  job_id TEXT PRIMARY KEY,
+                  status TEXT NOT NULL,
+                  total INTEGER NOT NULL DEFAULT 0,
+                  processed INTEGER NOT NULL DEFAULT 0,
+                  failed INTEGER NOT NULL DEFAULT 0,
+                  incremental INTEGER NOT NULL DEFAULT 1,
+                  user_email TEXT,
+                  max_errors INTEGER NOT NULL DEFAULT 50,
+                  items_json TEXT NOT NULL DEFAULT '[]',
+                  done_indices_json TEXT NOT NULL DEFAULT '[]',
+                  errors_json TEXT NOT NULL DEFAULT '[]',
+                  created_at TEXT NOT NULL,
+                  updated_at TEXT NOT NULL
+                );
+                CREATE INDEX IF NOT EXISTS idx_ingestion_jobs_status
+                  ON ingestion_jobs(status);
+                CREATE INDEX IF NOT EXISTS idx_ingestion_jobs_created
+                  ON ingestion_jobs(created_at);
+                """
+            )
+
+    def save(self, job: BackgroundIngestionJob) -> None:
+        with self._connect() as conn:
+            conn.execute(
+                """
+                INSERT INTO ingestion_jobs(
+                  job_id, status, total, processed, failed, incremental, user_email,
+                  max_errors, items_json, done_indices_json, errors_json,
+                  created_at, updated_at)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(job_id) DO UPDATE SET
+                  status=excluded.status,
+                  total=excluded.total,
+                  processed=excluded.processed,
+                  failed=excluded.failed,
+                  incremental=excluded.incremental,
+                  user_email=excluded.user_email,
+                  max_errors=excluded.max_errors,
+                  items_json=excluded.items_json,
+                  done_indices_json=excluded.done_indices_json,
+                  errors_json=excluded.errors_json,
+                  updated_at=excluded.updated_at
+                """,
+                (
+                    job.job_id,
+                    job.status,
+                    int(job.total),
+                    int(job.processed),
+                    int(job.failed),
+                    1 if job.incremental else 0,
+                    job.user_email,
+                    int(job.max_errors),
+                    json.dumps(
+                        [_item_payload(item) for item in job.items],
+                        ensure_ascii=False,
+                        default=str,
+                    ),
+                    json.dumps(sorted(job.done_indices)),
+                    json.dumps(list(job.errors), ensure_ascii=False, default=str),
+                    job.created_at,
+                    job.updated_at,
+                ),
+            )
+
+    def load_all(self) -> List[BackgroundIngestionJob]:
+        """Every persisted job, oldest first.
+
+        A row still marked ``running`` means the process died mid-run. It is
+        reported as ``partial``/``queued`` (whichever the recorded progress
+        supports) so it is honestly *not* running and so
+        ``run_background_job`` will pick it up instead of refusing.
+        """
+        with self._connect() as conn:
+            rows = conn.execute(
+                "SELECT * FROM ingestion_jobs ORDER BY created_at ASC, job_id ASC"
+            ).fetchall()
+        jobs: List[BackgroundIngestionJob] = []
+        for row in rows:
+            try:
+                jobs.append(self._row_to_job(row))
+            except Exception as exc:  # noqa: BLE001 — one bad row must not hide the rest
+                logging.warning(
+                    "ingestion job %s could not be restored: %s", row["job_id"], exc
+                )
+        return jobs
+
+    @staticmethod
+    def _row_to_job(row: sqlite3.Row) -> BackgroundIngestionJob:
+        items = [
+            _item_from_payload(payload)
+            for payload in json.loads(row["items_json"] or "[]")
+        ]
+        done = {int(i) for i in json.loads(row["done_indices_json"] or "[]")}
+        status = str(row["status"] or "queued")
+        if status == "running":
+            status = "partial" if done else "queued"
+        return BackgroundIngestionJob(
+            job_id=str(row["job_id"]),
+            items=items,
+            status=status,
+            created_at=str(row["created_at"]),
+            updated_at=str(row["updated_at"]),
+            processed=len(done),
+            failed=int(row["failed"] or 0),
+            total=int(row["total"] or 0),
+            errors=list(json.loads(row["errors_json"] or "[]")),
+            incremental=bool(row["incremental"]),
+            user_email=row["user_email"],
+            max_errors=int(row["max_errors"] or JOB_ERRORS_CAP),
+            done_indices=done,
+        )
+
+
 class BackgroundIngestionQueue:
-    """Simple in-memory queue for background incremental ingestion.
+    """Queue for background incremental ingestion, durable when given a db.
 
     For large corpus: this is the seam where a real scheduler / worker pool
     (celery, rq, or internal thread) can be plugged later without changing callers.
     Supports incremental (skip duplicates) vs force reindex.
+
+    ``db_path`` makes job state survive a restart. The in-process dict stays
+    the authority *within* a process (callers hold job references and mutate
+    them), and every mutation is mirrored to SQLite through :meth:`save`.
+    Without ``db_path`` — or when the database cannot be opened — the queue is
+    memory-only and :meth:`describe` reports that instead of implying
+    durability it does not have.
     """
-    def __init__(self) -> None:
+
+    def __init__(self, db_path: Optional[Any] = None) -> None:
         self._jobs: Dict[str, BackgroundIngestionJob] = {}
         self._counter = 0
-        self._lock = threading.Lock()
+        self._lock = threading.RLock()
+        self._store: Optional[IngestionJobStore] = None
+        self._persistence_detail: Optional[str] = None
+        if db_path is None:
+            self._persistence_detail = "no database configured; job state is in-memory only"
+        elif not isinstance(db_path, (str, Path)):
+            self._persistence_detail = (
+                f"unusable db_path {type(db_path).__name__}; job state is in-memory only"
+            )
+        else:
+            try:
+                self._store = IngestionJobStore(Path(db_path))
+                for job in self._store.load_all():
+                    self._jobs[job.job_id] = job
+                    self._counter = max(self._counter, _job_sequence(job.job_id))
+            except Exception as exc:  # noqa: BLE001 — degrade to memory, never block ingestion
+                self._store = None
+                self._persistence_detail = f"job persistence unavailable: {exc}"
+                logging.warning("ingestion job persistence unavailable: %s", exc)
+
+    # ── honesty surface ──────────────────────────────────────────────────────
+    def describe(self) -> Dict[str, Any]:
+        """Whether resume state actually survives a restart, and where."""
+        return {
+            "persistent": self._store is not None,
+            "db_path": str(self._store.db_path) if self._store is not None else None,
+            "jobs": len(self._jobs),
+            "detail": self._persistence_detail,
+        }
+
+    def save(self, job: BackgroundIngestionJob) -> None:
+        """Mirror a job's current state to disk (no-op when memory-only).
+
+        A persistence failure degrades durability, never the run in progress:
+        the item work already succeeded and must not be rolled back by a
+        bookkeeping error.
+        """
+        if self._store is None:
+            return
+        try:
+            self._store.save(job)
+        except Exception as exc:  # noqa: BLE001 — durability is best-effort mid-run
+            logging.warning("ingestion job %s could not be persisted: %s", job.job_id, exc)
 
     def schedule(
         self,
@@ -114,6 +345,7 @@ class BackgroundIngestionQueue:
             it.metadata = {**it.metadata, "incremental": incremental, "bg_job": job_id}
         with self._lock:
             self._jobs[job_id] = job
+        self.save(job)
         return job
 
     def get(self, job_id: str) -> Optional[BackgroundIngestionJob]:
@@ -130,4 +362,22 @@ class BackgroundIngestionQueue:
         return list(reversed(jobs))[:limit]
 
 
-__all__ = ["JOB_ERRORS_CAP", "BackgroundIngestionJob", "BackgroundIngestionQueue"]
+def _job_sequence(job_id: str) -> int:
+    """The numeric suffix of ``bg_ingest_0007`` → 7 (0 when unparseable).
+
+    Restored jobs must not have their ids handed out again after a restart.
+    """
+    _, _, suffix = str(job_id or "").rpartition("_")
+    try:
+        return int(suffix)
+    except ValueError:
+        return 0
+
+
+__all__ = [
+    "JOB_ERRORS_CAP",
+    "JOB_STATUSES",
+    "BackgroundIngestionJob",
+    "BackgroundIngestionQueue",
+    "IngestionJobStore",
+]

package/lattice_brain/portability.py

@@ -4,8 +4,11 @@ The Knowledge Graph is the user's durable asset, so it must be portable without
 any cloud service. Two complementary mechanisms, both fully local:
 
 * **Logical export/import** (JSON): nodes/edges/chunks/sources/provenance with a
-  versioned header (schema + projection + embed-dim). Re-embeds on import, so it
-  is portable across machines.
+  versioned header (schema + projection + embed-dim). Vectors are not in the
+  artifact; the importer re-embeds with its own embedder and reports the
+  resulting index state under ``result["index"]`` (``degraded: true`` means the
+  content landed but recall is lexical-only until a rebuild succeeds). That is
+  what makes it portable across machines.
 * **Binary backup/restore** (ZIP): a faithful snapshot of the SQLite DB (incl.
   vector embeddings) plus the blob directory, integrity-checked, for
   same-machine recovery.

package/lattice_brain/runtime/multi_agent.py

@@ -47,7 +47,7 @@ from typing import Any, Callable, Dict, List, Optional
 from ..utils import now_iso as _now
 from .contracts import multi_agent_contract
 
-MULTI_AGENT_VERSION = "10.6.0"
+MULTI_AGENT_VERSION = "10.6.2"
 
 AGENT_ROLES = ("researcher", "planner", "executor", "reviewer", "release")
 CORE_PIPELINE = ("planner", "executor", "reviewer")

package/latticeai/__init__.py

@@ -1,3 +1,3 @@
 """Lattice AI - modular server package."""
 
-__version__ = "10.6.0"
+__version__ = "10.6.2"

package/latticeai/api/browser.py

@@ -28,6 +28,7 @@ from pydantic import BaseModel
 
 from lattice_brain.ingestion import IngestionItem, capture_quality_verdict
 from latticeai import __version__
+from latticeai.api.workspace_scope import resolve_workspace_scope
 from latticeai.core.quiet import quiet
 
 MAX_TAB_BYTES = 4 * 1024 * 1024          # 4 MB per captured tab payload
@@ -399,20 +400,13 @@ def create_browser_router(
             raise HTTPException(status_code=503, detail="Knowledge Graph ingestion is disabled.")
 
     def _write_workspace(request: Request, body_workspace: Optional[str], user: str) -> Optional[str]:
-        header_workspace = request.headers.get("X-Workspace-Id")
-        header_workspace = header_workspace.strip() if header_workspace and header_workspace.strip() else None
-        query_workspace = request.query_params.get("workspace_id")
-        query_workspace = query_workspace.strip() if query_workspace and query_workspace.strip() else None
-        supplied = [value for value in (body_workspace, header_workspace, query_workspace) if value]
-        if len(set(supplied)) > 1:
-            raise HTTPException(status_code=403, detail="Workspace selectors must match.")
-        requested = supplied[0] if supplied else None
-        if workspace_service is None:
-            return requested
-        try:
-            return workspace_service.resolve_write_scope(requested, user or None)
-        except PermissionError as exc:
-            raise HTTPException(status_code=403, detail=str(exc)) from exc
+        return resolve_workspace_scope(
+            request,
+            user=user,
+            workspace_service=workspace_service,
+            write=True,
+            body_workspace=body_workspace,
+        )
 
     @router.post("/api/browser/read-url")
     async def read_url(req: ReadUrlRequest, request: Request):

package/latticeai/api/knowledge_graph.py

@@ -14,6 +14,10 @@ from pydantic import BaseModel
 
 from lattice_brain.ingestion import IngestionItem
 from latticeai.api.ui_redirects import app_redirect
+from latticeai.api.workspace_scope import (
+    resolve_workspace_scope,
+    workspace_scope_from_request,
+)
 
 
 class KnowledgeGraphIngestRequest(BaseModel):
@@ -43,12 +47,9 @@ class PromotionActionRequest(BaseModel):
     ids: Optional[List[str]] = None
 
 
-def _workspace_scope_from_request(request: Request) -> Optional[str]:
-    header = request.headers.get("X-Workspace-Id")
-    if header and header.strip():
-        return header.strip()
-    query = request.query_params.get("workspace_id")
-    return query.strip() if query and query.strip() else None
+# Kept as a module-level name because callers import it from here; the
+# implementation now lives in the shared resolver.
+_workspace_scope_from_request = workspace_scope_from_request
 
 
 def _format_context(matches: list, limit: int) -> str:
@@ -127,13 +128,44 @@ def create_knowledge_graph_router(
             ]
 
     def _write_workspace(request: Request, user: str) -> Optional[str]:
-        requested = _workspace_scope_from_request(request)
-        if workspace_service is None:
-            return requested
+        return resolve_workspace_scope(
+            request,
+            user=user,
+            workspace_service=workspace_service,
+            write=True,
+        )
+
+    def _scoped_stats(request: Request) -> Dict[str, Any]:
+        """Store statistics restricted to what the caller may read.
+
+        ``stats()`` counted every row in the database, so a member of one
+        organization workspace could read another's node/edge/document volume
+        off a "harmless" metrics endpoint. Unscoped mode (single-user / no
+        auth) still gets the whole-store counts, which is the same number it
+        always was.
+        """
+        kg, allowed = _scoped(request)
+        if allowed is None:
+            return dict(kg.stats())
         try:
-            return workspace_service.resolve_write_scope(requested, user or None)
-        except PermissionError as exc:
-            raise HTTPException(status_code=403, detail=str(exc)) from exc
+            return dict(
+                kg.stats(allowed_workspaces=allowed, include_legacy_global=False)
+            )
+        except TypeError:
+            # A store predating scoped stats cannot answer the scoped
+            # question. Keep the response shape and empty the aggregates it
+            # could not restrict, rather than leaking whole-store totals.
+            payload = dict(kg.stats())
+            empty: Dict[str, Any] = {
+                "nodes": {},
+                "edges": {},
+                "local_sources": 0,
+                "local_file_status": {},
+            }
+            payload.update(
+                {key: value for key, value in empty.items() if key in payload}
+            )
+            return payload
 
     @router.get("/graph")
     async def knowledge_graph_page(request: Request):
@@ -214,13 +246,11 @@ def create_knowledge_graph_router(
 
     @router.get("/knowledge-graph/stats")
     async def knowledge_graph_stats(request: Request):
-        require_user(request)
-        return graph().stats()
+        return _scoped_stats(request)
 
     @router.get("/knowledge-graph/schema")
     async def knowledge_graph_schema(request: Request):
-        require_user(request)
-        stats = graph().stats()
+        stats = _scoped_stats(request)
         return {
             "legacy_schema_version": stats.get("schema_version"),
             "v2_schema_available": stats.get("v2_schema_available"),

package/latticeai/api/workspace.py

@@ -21,6 +21,7 @@ from fastapi import APIRouter, HTTPException, Request
 from pydantic import BaseModel
 
 from latticeai.api.ui_redirects import app_redirect
+from latticeai.api.workspace_scope import workspace_scope_from_request
 from latticeai.services.app_context import AppContext
 
 # ── Request models (workspace-only; moved verbatim from server_app) ──────────
@@ -142,17 +143,9 @@ class WorkspaceActivateRequest(BaseModel):
     workspace_id: str
 
 
-def _workspace_scope_from_request(request: Request) -> Optional[str]:
-    """Resolve a requested workspace id from header/query, or None.
-
-    ``None`` lets the service fall back to the active workspace (Personal by
-    default), preserving pre-1.1 behaviour for clients that send no header.
-    """
-    header = request.headers.get("X-Workspace-Id")
-    if header and header.strip():
-        return header.strip()
-    query = request.query_params.get("workspace_id")
-    return query.strip() if query and query.strip() else None
+# Historical name: ``app_factory`` re-exports it as part of the legacy
+# ``server_app`` surface. The implementation is the shared resolver.
+_workspace_scope_from_request = workspace_scope_from_request
 
 
 def create_workspace_router(context: AppContext) -> APIRouter:

package/latticeai/api/workspace_scope.py

@@ -0,0 +1,125 @@
+"""One answer to "which workspace is this request talking about?".
+
+Eight routers used to each re-derive this from raw headers. They did not agree:
+some accepted the ``workspace_id`` query parameter and some only the
+``X-Workspace-Id`` header, and only three of them checked that a body's
+``workspace_id`` matched the header instead of silently letting one win. A
+guard that exists in some handlers and not others is not a guard — this module
+is the single implementation they all call, so the rule is stated once and
+holds everywhere.
+
+The rule:
+
+* A caller may name a workspace in the ``X-Workspace-Id`` header, the
+  ``workspace_id`` query parameter, or the request body.
+* If more than one of those is present they must **agree**. Disagreement is a
+  ``403``, never a silent preference — a request that names two workspaces has
+  no single meaning, and picking one is how a scoped write lands in the wrong
+  vault.
+* Naming nothing resolves to ``None``, which every caller passes to
+  :class:`~latticeai.services.workspace_service.WorkspaceService`, where it
+  falls back to the active workspace (Personal by default). That is the
+  pre-1.1 single-workspace behaviour and it stays intact.
+
+Permission is *not* decided here. ``WorkspaceService`` owns read/write gating;
+this module only decides what was asked for and translates the service's
+``PermissionError`` into the HTTP boundary's ``403``.
+"""
+
+from __future__ import annotations
+
+from typing import Any, List, Optional
+
+from fastapi import HTTPException, Request
+
+__all__ = [
+    "WORKSPACE_HEADER",
+    "WORKSPACE_PARAM",
+    "requested_workspace",
+    "resolve_workspace_scope",
+    "workspace_scope_from_request",
+]
+
+WORKSPACE_HEADER = "X-Workspace-Id"
+WORKSPACE_PARAM = "workspace_id"
+
+_MISMATCH_DETAIL = "Workspace selectors must match."
+
+
+def _clean(value: Any) -> Optional[str]:
+    """Normalize one selector to a non-empty string, or ``None``."""
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text or None
+
+
+def workspace_scope_from_request(request: Request) -> Optional[str]:
+    """Resolve the workspace named by header/query, or ``None``.
+
+    ``None`` lets the service fall back to the active workspace (Personal by
+    default), preserving pre-1.1 behaviour for clients that send no header.
+    """
+    header = _clean(request.headers.get(WORKSPACE_HEADER))
+    if header:
+        return header
+    return _clean(request.query_params.get(WORKSPACE_PARAM))
+
+
+def requested_workspace(
+    request: Request,
+    *,
+    body_workspace: Any = None,
+) -> Optional[str]:
+    """The one workspace this request names, or ``None``.
+
+    Raises ``403`` when the header, query parameter, and body disagree.
+    """
+    selectors: List[str] = []
+    for value in (
+        _clean(body_workspace),
+        _clean(request.headers.get(WORKSPACE_HEADER)),
+        _clean(request.query_params.get(WORKSPACE_PARAM)),
+    ):
+        if value is not None:
+            selectors.append(value)
+    if len(set(selectors)) > 1:
+        raise HTTPException(status_code=403, detail=_MISMATCH_DETAIL)
+    return selectors[0] if selectors else None
+
+
+def resolve_workspace_scope(
+    request: Request,
+    *,
+    user: Optional[str],
+    workspace_service: Any = None,
+    write: bool = True,
+    body_workspace: Any = None,
+    allow_unscoped_anonymous: bool = False,
+) -> Optional[str]:
+    """Resolve *and authorize* the workspace a handler should act on.
+
+    ``workspace_service=None`` is the standalone/embedded router contract: the
+    named workspace passes through ungated, exactly as each local copy of this
+    logic did. With a service present, reads are gated on ``read`` and writes
+    on ``write``, and a denial surfaces as ``403``.
+
+    ``allow_unscoped_anonymous`` preserves one deliberate exception: a no-auth
+    local caller that names no workspace keeps its legacy *unscoped* records
+    instead of being resolved onto the active workspace.
+    """
+    requested = requested_workspace(request, body_workspace=body_workspace)
+    if workspace_service is None:
+        return requested
+    if allow_unscoped_anonymous and not user and requested is None:
+        return None
+    resolver = (
+        workspace_service.resolve_write_scope
+        if write
+        else workspace_service.resolve_read_scope
+    )
+    try:
+        scope: Optional[str] = resolver(requested, user or None)
+    except PermissionError as exc:
+        raise HTTPException(status_code=403, detail=str(exc)) from exc
+    return scope