ltcai 11.7.0 → 12.0.0

package/latticeai/core/security.py

@@ -1,9 +1,27 @@
-"""Password hashing, rate limiting, IP detection, file-magic validation."""
+"""Token hashing, secret redaction, and the worker's per-user rate limiter.
+
+What is *not* here any more, and why: password hashing / verification, the
+``client_ip`` resolver with its per-IP login limiter, and the file-magic
+extension check were all ported to ``lattice-auth`` / ``lattice-chat`` in
+v11.6.0, when the front door and every write moved to Rust. They were left
+behind as fully orphaned Python — no route, no service, no script called them
+— and a second copy of an authentication rule that nothing enforces is worse
+than no copy: it reads like a live guard.
+
+What stays, stays because something still calls it:
+
+* :func:`enforce_rate_limit` — charged by every ``/worker/*`` and ``/agent/*``
+  seam through ``_admit``.
+* :func:`redact_secret_text` / :func:`redact_secrets` — and they additionally
+  generate the Rust redaction fixture via ``scripts/gen_redact_fixture.py``.
+* the trusted-proxy allowlist — ``_peer_is_trusted_proxy`` is what
+  :mod:`latticeai.core.http_origin` (and through it the CSRF check) asks before
+  believing an ``X-Forwarded-Host``.
+"""
 
 import hashlib
 import ipaddress
 import re
-import secrets
 import threading
 import time
 from typing import Any, Dict, List
@@ -26,21 +44,6 @@ def sha256_hex(text: str) -> str:
     return hashlib.sha256(text.encode("utf-8")).hexdigest()
 
 
-def hash_password(password: str) -> str:
-    salt = secrets.token_hex(16)
-    key = hashlib.scrypt(password.encode(), salt=salt.encode(), n=16384, r=8, p=1)
-    return f"{salt}:{key.hex()}"
-
-
-def verify_password(password: str, hashed: str) -> bool:
-    try:
-        salt, key_hex = hashed.split(":", 1)
-        key = hashlib.scrypt(password.encode(), salt=salt.encode(), n=16384, r=8, p=1)
-        return secrets.compare_digest(key.hex(), key_hex)
-    except Exception:
-        return False
-
-
 def host_is_loopback(host: str) -> bool:
     if host in {"localhost", "127.0.0.1", "::1"}:
         return True
@@ -50,15 +53,15 @@ def host_is_loopback(host: str) -> bool:
         return False
 
 
-# ── Trusted-proxy handling ────────────────────────────────────────────────────
-# ``client_ip`` is the key used for IP rate limiting (login / register) and for
-# audit logging. A forwarded header (``X-Forwarded-For`` / ``CF-Connecting-IP``)
-# is *client-controllable*, so honoring it unconditionally lets anyone spoof
-# their source IP and bypass per-IP rate limits. We therefore trust those headers
-# ONLY when the direct peer is a configured trusted proxy (e.g. the Cloudflare /
-# Vercel edge in front of the app). Default: no trusted proxies → use the peer
-# address, which is the safe, local-first behaviour.
-_FORWARDED_HEADERS = ("CF-Connecting-IP", "X-Forwarded-For")
+# ── Trusted-proxy allowlist ──────────────────────────────────────────────────
+# The per-IP login limiter this list was born for is ``lattice-auth``'s now, and
+# ``client_ip`` went with it. The list itself is still load-bearing on this
+# side: :func:`latticeai.core.http_origin.peer_may_forward` — and through it the
+# CSRF front-door check — asks ``_peer_is_trusted_proxy`` whether an
+# ``X-Forwarded-Host`` from this peer may be believed. Off loopback the answer
+# is "only for an operator-listed proxy", which is what makes forging a front
+# door from another machine impossible. ``configure_trusted_proxies`` is the one
+# way that list is ever non-empty, so it stays with the guard it feeds.
 _trusted_proxies: List["ipaddress._BaseNetwork"] = []
 
 
@@ -96,46 +99,6 @@ def _peer_is_trusted_proxy(peer: str) -> bool:
     return any(addr in net for net in _trusted_proxies)
 
 
-def client_ip(request) -> str:
-    peer = request.client.host if request.client else ""
-    # Only a trusted proxy's forwarded headers are honoured; otherwise the
-    # client-supplied header is ignored so per-IP rate limits cannot be spoofed.
-    if _peer_is_trusted_proxy(peer):
-        for header in _FORWARDED_HEADERS:
-            val = request.headers.get(header)
-            if val:
-                candidate = val.split(",")[0].strip()
-                try:
-                    ipaddress.ip_address(candidate)
-                    return candidate
-                except ValueError:
-                    quiet()
-                    continue
-    return peer or "unknown"
-
-
-_FILE_MAGIC: Dict[str, List[bytes]] = {
-    ".pdf":  [b"%PDF-"],
-    ".docx": [b"PK\x03\x04"],
-    ".xlsx": [b"PK\x03\x04"],
-    ".pptx": [b"PK\x03\x04"],
-    ".zip":  [b"PK\x03\x04", b"PK\x05\x06", b"PK\x07\x08"],
-    ".png":  [b"\x89PNG\r\n\x1a\n"],
-    ".jpg":  [b"\xff\xd8\xff"],
-    ".jpeg": [b"\xff\xd8\xff"],
-    ".gif":  [b"GIF87a", b"GIF89a"],
-}
-
-
-def bytes_match_extension(data: bytes, ext: str) -> bool:
-    ext = (ext or "").lower()
-    signatures = _FILE_MAGIC.get(ext)
-    if not signatures:
-        return True
-    head = data[:16]
-    return any(head.startswith(sig) for sig in signatures)
-
-
 SECRET_KEY_HINTS = (
     "api_key",
     "apikey",
@@ -209,23 +172,6 @@ def redact_secrets(value: Any) -> Any:
     return value
 
 
-# ── IP-based rate limiting (registration / login) ────────────────────────────
-_ip_rate_windows: dict = {}
-_ip_rate_lock = threading.Lock()
-
-
-def check_ip_rate_limit(ip: str, action: str, max_calls: int, window_secs: float) -> None:
-    key = (ip, action)
-    now = time.time()
-    cutoff = now - window_secs
-    with _ip_rate_lock:
-        calls = [t for t in _ip_rate_windows.get(key, []) if t > cutoff]
-        if len(calls) >= max_calls:
-            raise HTTPException(status_code=429, detail="요청이 너무 많습니다. 잠시 후 다시 시도하세요.")
-        calls.append(now)
-        _ip_rate_windows[key] = calls
-
-
 # ── Per-user token-bucket rate limiting ──────────────────────────────────────
 _RATE_LIMITS = {
     "chat":   (30, 0.5),

package/latticeai/core/sessions.py

@@ -4,6 +4,21 @@ v4: bearer tokens are stored **hashed** (sha256) at rest — a process that can
 read ``sessions.json`` must not be able to hijack every session. Pre-v4 files
 holding raw tokens are migrated transparently on load (sessions survive the
 upgrade; the raw token never touches disk again).
+
+11.8.0: the in-memory map is a **cache of the file, not the file itself.**
+Since v11.6.0 the writer is ``lattice-auth``: it creates the session and
+appends it to ``sessions.json``, and this process only ever reads. Loading
+once in ``__init__`` therefore made every login that happened after worker
+boot invisible here — silently under ``trusted_local_owner`` (the anonymous
+owner path answers first), and as a flat 401 under
+``LATTICEAI_REQUIRE_AUTH=true``, for a token that is sitting in the file.
+
+A lookup that misses now re-reads the file before giving up. Two guards keep
+that from turning a token-guessing burst into a disk-read burst: the re-read
+is skipped when the file's ``mtime_ns``/``size`` are unchanged since the last
+load, and — for a file that *is* changing — throttled to one parse per
+``SESSION_RELOAD_MIN_INTERVAL``. Both are checked under the same lock that
+guards the map, so concurrent misses collapse into one read.
 """
 
 import json
@@ -19,6 +34,11 @@ from latticeai.core.security import sha256_hex
 
 SESSION_TTL = 60 * 60 * 24  # 24 hours
 SESSION_REFRESH_THRESHOLD = 60 * 15  # only persist if >15 min since last bump
+#: Floor between two miss-triggered re-reads of ``sessions.json``. A wrong or
+#: expired token is the common case for an unauthenticated probe, and each one
+#: misses; without a floor a burst of them would be a burst of file reads.
+#: Well under any human's retry, so a real login is still picked up at once.
+SESSION_RELOAD_MIN_INTERVAL = 1.0
 _lock = threading.Lock()
 
 _HEX64 = frozenset("0123456789abcdef")
@@ -79,6 +99,19 @@ def persist_sessions(sessions: Dict[str, tuple], data_dir: Optional[Path] = None
         logging.warning("persist_sessions failed: %s", e)
 
 
+def _sessions_stamp(data_dir: Optional[Path] = None) -> Optional[tuple]:
+    """``(mtime_ns, size)`` of ``sessions.json``, or ``None`` when unreadable.
+
+    ``None`` is *not* "unchanged": a file that has just appeared and one that
+    was just removed both need the map rebuilt, and both land here.
+    """
+    try:
+        stat = _sessions_file(data_dir).stat()
+    except OSError:
+        return None
+    return (stat.st_mtime_ns, stat.st_size)
+
+
 def _entry_subject(entry: tuple) -> Optional[str]:
     return entry[0] if entry else None
 
@@ -107,12 +140,63 @@ class SessionStore:
         self._ttl_seconds = int(ttl_seconds or SESSION_TTL)
         self._refresh_threshold_seconds = int(refresh_threshold_seconds or SESSION_REFRESH_THRESHOLD)
         self._sessions: Dict[str, tuple] = load_sessions(data_dir)
+        # Stamped *after* the load, so a pre-v4 file that ``load_sessions``
+        # rewrote on the way in is not mistaken for a foreign write.
+        self._loaded_stamp: Optional[tuple] = _sessions_stamp(data_dir)
+        # ``-inf`` and not "now": the first *changed* file must be free to
+        # load, or a login that lands in the same second as the worker's boot
+        # stays invisible for a second for no reason.
+        self._last_reload_at: float = float("-inf")
+
+    def _reload_if_stale(self, *, force: bool = False) -> bool:
+        """Re-read ``sessions.json`` when it has changed. Caller holds ``_lock``.
+
+        Returns whether the map was replaced. The two guards answer two
+        different costs, in the order that keeps both cheap:
+
+        * the **stamp** runs first and on every miss. It is one ``stat``, and
+          when the file has not moved — the overwhelming case, since a wrong
+          token is what an unauthenticated probe sends — that is the entire
+          cost. No parse, no allocation.
+        * the **interval** runs only once the file *has* moved, and bounds how
+          often a busy file is actually parsed. It never drops the change: the
+          stamp is recorded only when the load really happened, so the next
+          miss past the interval still sees the file as new.
+
+        ``force`` skips the interval only. It is for the two paths that are
+        about to *write* the file: a throttled read there would merge onto a
+        stale map and drop somebody else's session, and a login or a logout is
+        rare enough that the read costs nothing worth counting.
+        """
+        stamp = _sessions_stamp(self._data_dir)
+        if stamp == self._loaded_stamp:
+            return False
+        now = time.monotonic()
+        if not force and now - self._last_reload_at < SESSION_RELOAD_MIN_INTERVAL:
+            return False
+        self._last_reload_at = now
+        self._sessions = load_sessions(self._data_dir)
+        self._loaded_stamp = stamp
+        return True
+
+    def _persist(self) -> None:
+        """Write the map, then re-stamp. Caller holds ``_lock``.
+
+        Re-stamping matters: without it our own write looks like somebody
+        else's the next time a lookup misses, and the store re-reads the file
+        it just produced.
+        """
+        persist_sessions(self._sessions, self._data_dir)
+        self._loaded_stamp = _sessions_stamp(self._data_dir)
 
     def create(self, subject: str, *, email: Optional[str] = None) -> str:
         token = secrets.token_urlsafe(32)
         with _lock:
+            # Merge onto whatever is on disk rather than over it — this process
+            # is not the only writer.
+            self._reload_if_stale(force=True)
             self._sessions[_hash_token(token)] = (subject, time.time(), email or subject)
-            persist_sessions(self._sessions, self._data_dir)
+            self._persist()
         return token
 
     def get_email(self, token: str) -> Optional[str]:
@@ -128,21 +212,28 @@ class SessionStore:
         key = _hash_token(token)
         with _lock:
             entry = self._sessions.get(key)
+            if entry is None and self._reload_if_stale():
+                # A miss is the only thing that can be wrong because the map is
+                # stale: an entry we *do* hold was really issued, and one we
+                # hold that the file no longer has is handled by the TTL and by
+                # ``active_session_email``'s account check on every request.
+                entry = self._sessions.get(key)
             if entry is None:
                 return None
             created_at = _entry_created_at(entry)
             if now - created_at > self._ttl_seconds:
                 self._sessions.pop(key, None)
-                persist_sessions(self._sessions, self._data_dir)
+                self._persist()
                 return None
             if now - created_at > self._refresh_threshold_seconds:
                 refreshed = (_entry_subject(entry), now, _entry_email(entry))
                 self._sessions[key] = refreshed
-                persist_sessions(self._sessions, self._data_dir)
+                self._persist()
                 return refreshed
             return entry
 
     def invalidate(self, token: str) -> None:
         with _lock:
+            self._reload_if_stale(force=True)
             self._sessions.pop(_hash_token(token), None)
-            persist_sessions(self._sessions, self._data_dir)
+            self._persist()

package/latticeai/core/users.py

@@ -4,9 +4,7 @@ from __future__ import annotations
 
 import json
 import shutil
-import sqlite3
 import uuid
-from contextlib import closing
 from pathlib import Path
 from typing import Any, Dict, Optional
 
@@ -83,11 +81,6 @@ def load_users_file(path: Path) -> Dict[str, Any]:
     return migrated
 
 
-def save_users_file(path: Path, users: Dict[str, Any]) -> None:
-    migrated, _, _ = migrate_users(users)
-    atomic_write_json(path, migrated)
-
-
 def user_id_for_email(users: Dict[str, Any], email: Optional[str]) -> Optional[str]:
     if not email:
         return None
@@ -98,34 +91,3 @@ def user_id_for_email(users: Dict[str, Any], email: Optional[str]) -> Optional[s
     if isinstance(user, dict):
         return user.get("id") or stable_user_id(normalized)
     return stable_user_id(normalized)
-
-
-def migrate_knowledge_graph_identity(db_path: Path, email_to_id: Dict[str, str]) -> int:
-    """Rewrite KG owner/creator identity columns from email to stable UUIDs."""
-    if not db_path.exists() or not email_to_id:
-        return 0
-    changed = 0
-    with closing(sqlite3.connect(db_path)) as conn, conn:
-        tables = {
-            row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")
-        }
-        for email, user_id in email_to_id.items():
-            normalized = normalize_email(email)
-            if "nodes_v2" in tables:
-                cur = conn.execute("UPDATE nodes_v2 SET owner_id=? WHERE LOWER(owner_id)=?", (user_id, normalized))
-                changed += cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
-            if "edges_v2" in tables:
-                cur = conn.execute("UPDATE edges_v2 SET created_by=? WHERE LOWER(created_by)=?", (user_id, normalized))
-                changed += cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
-            if "ingestion_provenance" in tables:
-                cur = conn.execute("UPDATE ingestion_provenance SET owner=? WHERE LOWER(owner)=?", (user_id, normalized))
-                changed += cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
-        if changed:
-            conn.execute(
-                "CREATE TABLE IF NOT EXISTS kg_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)"
-            )
-            conn.execute(
-                "INSERT OR REPLACE INTO kg_meta(key, value) VALUES('identity_uuid_migrated_at', ?)",
-                (_now(),),
-            )
-    return changed

package/latticeai/core/vector_index/__init__.py

@@ -0,0 +1,61 @@
+"""Search-side vector index backends (HNSW append + query).
+
+``POST /worker/vector/query`` serves this sidecar. ``lattice-retrieval``
+asks for top-k candidates when ``LATTICEAI_VECTOR_INDEX=hnsw`` and
+re-scores them exactly. Default env stays the brute scan.
+
+This package is the Python HNSW sidecar: a disposable ``.hnsw`` graph
+next to the brain database, with incremental ``add_items`` so a write no
+longer rebuilds the whole graph.
+
+Rebuild (not append) when:
+
+* the embedder identity (``model_id`` / dim / space) changed
+* any previously indexed id is absent (deletion). Tombstones exist in
+  hnswlib (``mark_deleted``) but a deletion-heavy store (more than
+  :data:`DELETE_REBUILD_RATIO` of the graph, or any delete when the
+  sidecar was loaded without source vectors) is cheaper to rebuild
+  than to fragment.
+"""
+
+from .hnsw import (
+    DELETE_REBUILD_RATIO,
+    HNSW_BACKEND,
+    HNSW_META_SUFFIX,
+    HNSW_SUFFIX,
+    HNSWLIB_MODULE,
+    HnswIndex,
+    hnswlib_available,
+    load_hnswlib,
+    sidecar_paths,
+)
+from .sidecar import (
+    GRAPH_DB_NAME,
+    VECTOR_QUERY_K_CAP,
+    decode_f32le,
+    query_sidecar,
+    reset_sidecar_cache,
+    resolve_graph_db,
+    sidecar_fingerprint,
+    sidecar_freshness,
+)
+
+__all__ = [
+    "DELETE_REBUILD_RATIO",
+    "GRAPH_DB_NAME",
+    "HNSW_BACKEND",
+    "HNSW_META_SUFFIX",
+    "HNSW_SUFFIX",
+    "HNSWLIB_MODULE",
+    "HnswIndex",
+    "VECTOR_QUERY_K_CAP",
+    "decode_f32le",
+    "hnswlib_available",
+    "load_hnswlib",
+    "query_sidecar",
+    "reset_sidecar_cache",
+    "resolve_graph_db",
+    "sidecar_fingerprint",
+    "sidecar_freshness",
+    "sidecar_paths",
+]