ltcai 4.0.0 → 4.1.0

package/latticeai/core/realtime.py

@@ -28,6 +28,7 @@ from __future__ import annotations
 
 import asyncio
 import json
+import threading
 from datetime import datetime
 from typing import Any, AsyncIterator, Dict, List, Optional, Set
 
@@ -47,7 +48,7 @@ def sse_format(event: Dict[str, Any]) -> str:
 
 
 class _Subscriber:
-    __slots__ = ("id", "queue", "workspace_scope", "user", "joined_at")
+    __slots__ = ("id", "queue", "workspace_scope", "user", "joined_at", "loop")
 
     def __init__(self, sub_id: str, workspace_scope: Optional[Set[str]], user: Optional[str]):
         self.id = sub_id
@@ -55,6 +56,10 @@ class _Subscriber:
         self.workspace_scope = workspace_scope
         self.user = user
         self.joined_at = _now()
+        try:
+            self.loop: Optional[asyncio.AbstractEventLoop] = asyncio.get_running_loop()
+        except RuntimeError:
+            self.loop = None
 
     def accepts(self, workspace_id: Optional[str]) -> bool:
         # ``None`` scope = see everything the local user can (personal/unscoped).
@@ -73,6 +78,7 @@ class RealtimeBus:
         self._feed: List[Dict[str, Any]] = []
         self._presence: Dict[str, Dict[str, Any]] = {}
         self._seq = 0
+        self._lock = threading.RLock()
 
     # ── publishing ────────────────────────────────────────────────────────
 
@@ -82,34 +88,41 @@ class RealtimeBus:
         Safe to call from sync code (e.g. the store's timeline hook). Never
         raises and never blocks the caller.
         """
-        self._seq += 1
-        enriched = {
-            "seq": self._seq,
-            "received_at": _now(),
-            "area": event.get("area", "workspace"),
-            "event_type": event.get("event_type", "event"),
-            "workspace_id": event.get("workspace_id"),
-            "payload": event.get("payload", {}),
-            **{k: v for k, v in event.items() if k not in {"area", "event_type", "workspace_id", "payload"}},
-        }
-        self._feed.append(enriched)
-        if len(self._feed) > _FEED_LIMIT:
-            self._feed = self._feed[-_FEED_LIMIT:]
-
-        workspace_id = enriched.get("workspace_id")
-        for sub in list(self._subscribers.values()):
-            if not sub.accepts(workspace_id):
-                continue
-            try:
-                sub.queue.put_nowait(enriched)
-            except asyncio.QueueFull:
-                try:
-                    sub.queue.get_nowait()  # drop oldest
-                    sub.queue.put_nowait(enriched)
-                except Exception:
-                    pass
+        with self._lock:
+            self._seq += 1
+            enriched = {
+                "seq": self._seq,
+                "received_at": _now(),
+                "area": event.get("area", "workspace"),
+                "event_type": event.get("event_type", "event"),
+                "workspace_id": event.get("workspace_id"),
+                "payload": event.get("payload", {}),
+                **{k: v for k, v in event.items() if k not in {"area", "event_type", "workspace_id", "payload"}},
+            }
+            self._feed.append(enriched)
+            if len(self._feed) > _FEED_LIMIT:
+                self._feed = self._feed[-_FEED_LIMIT:]
+
+            workspace_id = enriched.get("workspace_id")
+            subscribers = [sub for sub in self._subscribers.values() if sub.accepts(workspace_id)]
+        for sub in subscribers:
+            if sub.loop is not None and sub.loop.is_running():
+                sub.loop.call_soon_threadsafe(self._enqueue, sub, enriched)
+            else:
+                self._enqueue(sub, enriched)
         return enriched
 
+    @staticmethod
+    def _enqueue(sub: _Subscriber, event: Dict[str, Any]) -> None:
+        try:
+            sub.queue.put_nowait(event)
+        except asyncio.QueueFull:
+            try:
+                sub.queue.get_nowait()  # drop oldest
+                sub.queue.put_nowait(event)
+            except Exception:
+                pass
+
     # The store calls ``event_sink(event)`` positionally; expose a stable alias.
     def __call__(self, event: Dict[str, Any]) -> Dict[str, Any]:
         return self.publish(event)
@@ -118,11 +131,13 @@ class RealtimeBus:
 
     def add_subscriber(self, sub_id: str, *, workspace_scope: Optional[Set[str]] = None, user: Optional[str] = None) -> _Subscriber:
         sub = _Subscriber(sub_id, workspace_scope, user)
-        self._subscribers[sub_id] = sub
+        with self._lock:
+            self._subscribers[sub_id] = sub
         return sub
 
     def remove_subscriber(self, sub_id: str) -> None:
-        self._subscribers.pop(sub_id, None)
+        with self._lock:
+            self._subscribers.pop(sub_id, None)
 
     async def stream(self, sub: _Subscriber, *, heartbeat: float = 15.0) -> AsyncIterator[str]:
         """Yield SSE frames for a subscriber until the client disconnects.
@@ -146,7 +161,8 @@ class RealtimeBus:
     # ── feed + presence ─────────────────────────────────────────────────────
 
     def recent(self, *, limit: int = 50, workspace_scope: Optional[Set[str]] = None) -> List[Dict[str, Any]]:
-        events = self._feed
+        with self._lock:
+            events = list(self._feed)
         if workspace_scope is not None:
             events = [e for e in events if e.get("workspace_id") is None or e.get("workspace_id") in workspace_scope]
         return list(reversed(events[-max(1, min(limit, _FEED_LIMIT)):]))
@@ -159,32 +175,37 @@ class RealtimeBus:
             "joined_at": _now(),
             "last_seen": _now(),
         }
-        self._presence[client_id] = record
+        with self._lock:
+            self._presence[client_id] = record
         self.publish({"area": "presence", "event_type": "join", "workspace_id": workspace_id, "payload": {"user": user, "client_id": client_id}})
         return record
 
     def heartbeat(self, client_id: str) -> Optional[Dict[str, Any]]:
-        record = self._presence.get(client_id)
-        if record:
-            record["last_seen"] = _now()
-        return record
+        with self._lock:
+            record = self._presence.get(client_id)
+            if record:
+                record["last_seen"] = _now()
+            return record
 
     def leave(self, client_id: str) -> None:
-        record = self._presence.pop(client_id, None)
+        with self._lock:
+            record = self._presence.pop(client_id, None)
         if record:
             self.publish({"area": "presence", "event_type": "leave", "workspace_id": record.get("workspace_id"), "payload": {"client_id": client_id}})
 
     def presence(self, *, workspace_scope: Optional[Set[str]] = None) -> List[Dict[str, Any]]:
-        records = list(self._presence.values())
+        with self._lock:
+            records = list(self._presence.values())
         if workspace_scope is not None:
             records = [r for r in records if r.get("workspace_id") is None or r.get("workspace_id") in workspace_scope]
         return records
 
     def stats(self) -> Dict[str, Any]:
-        return {
-            "version": REALTIME_VERSION,
-            "subscribers": len(self._subscribers),
-            "presence": len(self._presence),
-            "feed_size": len(self._feed),
-            "transport": "sse",
-        }
+        with self._lock:
+            return {
+                "version": REALTIME_VERSION,
+                "subscribers": len(self._subscribers),
+                "presence": len(self._presence),
+                "feed_size": len(self._feed),
+                "transport": "sse",
+            }

package/latticeai/core/sessions.py

@@ -67,34 +67,60 @@ def persist_sessions(sessions: Dict[str, tuple], data_dir: Optional[Path] = None
         logging.warning("persist_sessions failed: %s", e)
 
 
+def _entry_subject(entry: tuple) -> Optional[str]:
+    return entry[0] if entry else None
+
+
+def _entry_email(entry: tuple) -> Optional[str]:
+    if len(entry) >= 3 and entry[2]:
+        return entry[2]
+    return entry[0] if entry else None
+
+
+def _entry_created_at(entry: tuple) -> float:
+    if len(entry) >= 2:
+        return float(entry[1])
+    return 0.0
+
+
 class SessionStore:
     def __init__(self, data_dir: Optional[Path] = None):
         self._data_dir = data_dir
         self._sessions: Dict[str, tuple] = load_sessions(data_dir)
 
-    def create(self, email: str) -> str:
+    def create(self, subject: str, *, email: Optional[str] = None) -> str:
         token = secrets.token_urlsafe(32)
         with _lock:
-            self._sessions[_hash_token(token)] = (email, time.time())
+            self._sessions[_hash_token(token)] = (subject, time.time(), email or subject)
             persist_sessions(self._sessions, self._data_dir)
         return token
 
     def get_email(self, token: str) -> Optional[str]:
+        entry = self._get_entry(token)
+        return _entry_email(entry) if entry else None
+
+    def get_subject(self, token: str) -> Optional[str]:
+        entry = self._get_entry(token)
+        return _entry_subject(entry) if entry else None
+
+    def _get_entry(self, token: str) -> Optional[tuple]:
         now = time.time()
         key = _hash_token(token)
         with _lock:
             entry = self._sessions.get(key)
             if entry is None:
                 return None
-            email, created_at = entry
+            created_at = _entry_created_at(entry)
             if now - created_at > SESSION_TTL:
                 self._sessions.pop(key, None)
                 persist_sessions(self._sessions, self._data_dir)
                 return None
             if now - created_at > SESSION_REFRESH_THRESHOLD:
-                self._sessions[key] = (email, now)
+                refreshed = (_entry_subject(entry), now, _entry_email(entry))
+                self._sessions[key] = refreshed
                 persist_sessions(self._sessions, self._data_dir)
-            return email
+                return refreshed
+            return entry
 
     def invalidate(self, token: str) -> None:
         with _lock:

package/latticeai/core/users.py

@@ -0,0 +1,147 @@
+"""User identity store and v4 UUID migration helpers."""
+
+from __future__ import annotations
+
+import json
+import shutil
+import sqlite3
+import uuid
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+
+USER_NAMESPACE = uuid.UUID("5d6d4480-cf79-49c3-a6d0-4c6eec3224d6")
+
+
+def _now() -> str:
+    return datetime.now().isoformat(timespec="seconds")
+
+
+def _atomic_write_json(path: Path, data: Dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
+    tmp.replace(path)
+
+
+def normalize_email(email: str) -> str:
+    return str(email or "").strip().lower()
+
+
+def stable_user_id(email: str) -> str:
+    return f"user:{uuid.uuid5(USER_NAMESPACE, normalize_email(email))}"
+
+
+def ensure_user_identity(email: str, user: Dict[str, Any]) -> bool:
+    changed = False
+    normalized = normalize_email(email or user.get("email") or "")
+    if not user.get("id"):
+        user["id"] = stable_user_id(normalized)
+        changed = True
+    if user.get("email") != normalized:
+        user["email"] = normalized
+        changed = True
+    return changed
+
+
+def migrate_users(users: Dict[str, Any]) -> tuple[Dict[str, Any], Dict[str, str], bool]:
+    migrated: Dict[str, Any] = {}
+    email_to_id: Dict[str, str] = {}
+    changed = False
+    for raw_email, raw_user in (users or {}).items():
+        if not isinstance(raw_user, dict):
+            continue
+        email = normalize_email(raw_user.get("email") or raw_email)
+        user = dict(raw_user)
+        changed = ensure_user_identity(email, user) or changed
+        if raw_email != email:
+            changed = True
+        if email in migrated:
+            existing = migrated[email]
+            merged = {**existing, **user}
+            merged["id"] = existing.get("id") or user.get("id") or stable_user_id(email)
+            if isinstance(existing.get("api_keys"), dict) or isinstance(user.get("api_keys"), dict):
+                merged["api_keys"] = {**(existing.get("api_keys") or {}), **(user.get("api_keys") or {})}
+            user = merged
+            changed = True
+        migrated[email] = user
+        email_to_id[email] = user["id"]
+    return migrated, email_to_id, changed
+
+
+def load_users_file(path: Path) -> Dict[str, Any]:
+    if not path.exists():
+        return {}
+    try:
+        loaded = json.loads(path.read_text(encoding="utf-8"))
+        if not isinstance(loaded, dict):
+            loaded = {}
+    except Exception:
+        loaded = {}
+    migrated, _, changed = migrate_users(loaded)
+    if changed:
+        backup = path.with_name(f"{path.name}.pre-user-uuid.{_now().replace(':', '-')}.json")
+        try:
+            shutil.copy2(path, backup)
+        except Exception:
+            pass
+        _atomic_write_json(path, migrated)
+    return migrated
+
+
+def save_users_file(path: Path, users: Dict[str, Any]) -> None:
+    migrated, _, _ = migrate_users(users)
+    _atomic_write_json(path, migrated)
+
+
+def user_id_for_email(users: Dict[str, Any], email: Optional[str]) -> Optional[str]:
+    if not email:
+        return None
+    if str(email).startswith("user:"):
+        return str(email)
+    normalized = normalize_email(email)
+    user = (users or {}).get(normalized)
+    if isinstance(user, dict):
+        return user.get("id") or stable_user_id(normalized)
+    return stable_user_id(normalized)
+
+
+def email_for_user_id(users: Dict[str, Any], user_id: Optional[str]) -> Optional[str]:
+    if not user_id:
+        return None
+    for email, user in (users or {}).items():
+        if isinstance(user, dict) and user.get("id") == user_id:
+            return email
+    return None
+
+
+def migrate_knowledge_graph_identity(db_path: Path, email_to_id: Dict[str, str]) -> int:
+    """Rewrite KG owner/creator identity columns from email to stable UUIDs."""
+    if not db_path.exists() or not email_to_id:
+        return 0
+    changed = 0
+    with sqlite3.connect(db_path) as conn:
+        tables = {
+            row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")
+        }
+        for email, user_id in email_to_id.items():
+            normalized = normalize_email(email)
+            if "nodes_v2" in tables:
+                cur = conn.execute("UPDATE nodes_v2 SET owner_id=? WHERE LOWER(owner_id)=?", (user_id, normalized))
+                changed += cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
+            if "edges_v2" in tables:
+                cur = conn.execute("UPDATE edges_v2 SET created_by=? WHERE LOWER(created_by)=?", (user_id, normalized))
+                changed += cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
+            if "ingestion_provenance" in tables:
+                cur = conn.execute("UPDATE ingestion_provenance SET owner=? WHERE LOWER(owner)=?", (user_id, normalized))
+                changed += cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
+        if changed:
+            conn.execute(
+                "CREATE TABLE IF NOT EXISTS kg_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)"
+            )
+            conn.execute(
+                "INSERT OR REPLACE INTO kg_meta(key, value) VALUES('identity_uuid_migrated_at', ?)",
+                (_now(),),
+            )
+    return changed