ltcai 0.1.30 → 0.2.0

package/latticeai/core/audit.py

@@ -0,0 +1,245 @@
+"""Audit logging, sensitivity analysis, and admin reporting."""
+
+import json
+import logging
+import os
+import re
+import threading
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional
+
+_history_lock = threading.Lock()
+
+SENSITIVE_PATTERNS = [
+    {"key": "rrn", "label": "주민등록번호", "severity": "high", "pattern": r"\b\d{6}[- ]?[1-4]\d{6}\b"},
+    {"key": "card", "label": "카드번호", "severity": "high", "pattern": r"\b(?:\d[ -]?){13,19}\b"},
+    {"key": "account", "label": "계좌번호", "severity": "medium", "pattern": r"(?:계좌|account|bank).{0,12}\d[\d -]{8,24}"},
+    {"key": "password", "label": "비밀번호/인증정보", "severity": "high", "pattern": r"(?:password|passwd|비밀번호|암호|token|api[_ -]?key|secret)\s*[:=]\s*[^\s,;]{4,}"},
+    {"key": "email", "label": "이메일", "severity": "low", "pattern": r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b"},
+    {"key": "phone", "label": "전화번호", "severity": "medium", "pattern": r"\b(?:01[016789]|02|0[3-6][1-5])[- ]?\d{3,4}[- ]?\d{4}\b"},
+    {"key": "address", "label": "주소", "severity": "medium", "pattern": r"(?:[가-힣]+(?:시|도)\s*)?[가-힣]+(?:시|군|구)\s+[가-힣0-9\s-]+(?:로|길)\s*\d*"},
+    {"key": "health", "label": "건강/의료정보", "severity": "medium", "pattern": r"(?:진단|병명|처방|복용|수술|장애|임신|혈액형|알레르기|medical|diagnosis)"},
+]
+SEVERITY_SCORE = {"low": 1, "medium": 2, "high": 3}
+AUDIT_DELETE_EVENTS = {"conversation_delete", "history_delete", "user_delete"}
+
+
+def get_audit_log(audit_file: Path) -> List[Dict]:
+    if not os.path.exists(audit_file):
+        return []
+    try:
+        with open(audit_file, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        return data if isinstance(data, list) else []
+    except Exception as e:
+        logging.warning("get_audit_log failed: %s", e)
+        return []
+
+
+def append_audit_event(audit_file: Path, event_type: str, **payload) -> None:
+    try:
+        event = {
+            "event_type": event_type,
+            "timestamp": datetime.now().isoformat(),
+            **payload,
+        }
+        with _history_lock:
+            events = get_audit_log(audit_file)
+            events.append(event)
+            if len(events) > 5000:
+                events = events[-5000:]
+            tmp_path = str(audit_file) + ".tmp"
+            with open(tmp_path, "w", encoding="utf-8") as f:
+                json.dump(events, f, ensure_ascii=False, indent=2)
+            os.replace(tmp_path, audit_file)
+    except Exception as e:
+        logging.warning("append_audit_event failed: %s", e)
+
+
+def mask_sensitive_text(text: str, matches: List[Dict]) -> str:
+    masked = text
+    for item in sorted(matches, key=lambda m: m["start"], reverse=True):
+        value = masked[item["start"]:item["end"]]
+        if len(value) <= 4:
+            replacement = "*" * len(value)
+        else:
+            replacement = value[:2] + "*" * min(len(value) - 4, 12) + value[-2:]
+        masked = masked[:item["start"]] + replacement + masked[item["end"]:]
+    return masked
+
+
+def classify_sensitive_message(item: Dict, index: int) -> Dict:
+    content = str(item.get("content", ""))
+    found = []
+    seen: set = set()
+    for rule in SENSITIVE_PATTERNS:
+        for match in re.finditer(rule["pattern"], content, flags=re.IGNORECASE):
+            key = (rule["key"], match.start(), match.end())
+            if key in seen:
+                continue
+            seen.add(key)
+            found.append({
+                "type": rule["key"],
+                "label": rule["label"],
+                "severity": rule["severity"],
+                "start": match.start(),
+                "end": match.end(),
+            })
+    severity = "none"
+    if found:
+        severity = max(found, key=lambda m: SEVERITY_SCORE[m["severity"]])["severity"]
+    preview_text = content[:240]
+    preview_matches = [m for m in found if m["start"] < len(preview_text)]
+    return {
+        "index": index,
+        "role": item.get("role", ""),
+        "user_email": item.get("user_email"),
+        "user_nickname": item.get("user_nickname") or item.get("user_email") or "Unknown",
+        "timestamp": item.get("timestamp"),
+        "sensitivity": severity,
+        "labels": sorted({m["label"] for m in found}),
+        "risk_fields": found,
+        "compliance_fields": [] if found else ["민감정보 미검출"],
+        "preview": mask_sensitive_text(preview_text, preview_matches),
+    }
+
+
+def build_sensitivity_report(history: List[Dict]) -> Dict:
+    items = [classify_sensitive_message(item, i) for i, item in enumerate(history)]
+    risky = [x for x in items if x["risk_fields"]]
+    compliant = [x for x in items if not x["risk_fields"]]
+    field_counts: Dict[str, int] = {}
+    user_counts: Dict[str, int] = {}
+    severity_counts = {"high": 0, "medium": 0, "low": 0, "none": len(compliant)}
+    for item in risky:
+        severity_counts[item["sensitivity"]] += 1
+        user_key = item.get("user_email") or item.get("user_nickname") or "Unknown"
+        user_counts[user_key] = user_counts.get(user_key, 0) + 1
+        for field in item["risk_fields"]:
+            field_counts[field["label"]] = field_counts.get(field["label"], 0) + 1
+    return {
+        "summary": {
+            "total_messages": len(items),
+            "risky_messages": len(risky),
+            "compliant_messages": len(compliant),
+            "risk_rate": round((len(risky) / len(items)) * 100, 1) if items else 0,
+            "severity_counts": severity_counts,
+            "field_counts": field_counts,
+            "user_counts": user_counts,
+        },
+        "risk_fields": risky[-30:],
+        "compliance_fields": compliant[-30:],
+    }
+
+
+def build_admin_audit_report(
+    audit_file: Path,
+    users: Dict,
+    *,
+    get_user_role: Callable[[str, Optional[Dict]], str],
+    graph_stats: Optional[Dict] = None,
+) -> Dict:
+    events = get_audit_log(audit_file)
+
+    def _user_bucket(email: Optional[str], nickname: Optional[str] = None) -> Dict:
+        user = users.get(email or "", {})
+        return {
+            "email": email or "Unknown",
+            "nickname": nickname or user.get("nickname") or user.get("name") or email or "Unknown",
+            "role": get_user_role(email, users) if email else "unknown",
+            "disabled": bool(user.get("disabled")) if user else False,
+            "user_messages": 0, "assistant_messages": 0, "document_uploads": 0,
+            "clear_events": 0, "delete_events": 0, "sensitive_events": 0,
+            "high_sensitive_events": 0, "total_content_chars": 0, "last_activity_at": None,
+        }
+
+    per_user: Dict[str, Dict] = {}
+
+    def ensure(email: Optional[str], nickname: Optional[str] = None) -> Dict:
+        key = email or nickname or "Unknown"
+        if key not in per_user:
+            per_user[key] = _user_bucket(email, nickname)
+        elif nickname and per_user[key].get("nickname") in {"Unknown", email, None}:
+            per_user[key]["nickname"] = nickname
+        return per_user[key]
+
+    for email, user in users.items():
+        ensure(email, user.get("nickname") or user.get("name"))
+
+    summary: Dict[str, Any] = {
+        "total_events": len(events), "chat_events": 0, "user_messages": 0,
+        "assistant_messages": 0, "document_uploads": 0, "clear_events": 0,
+        "delete_events": 0, "sensitive_events": 0, "high_sensitive_events": 0,
+    }
+    sensitive_events: List[Dict] = []
+    deletion_events: List[Dict] = []
+
+    for event in events:
+        event_type = event.get("event_type")
+        email = event.get("user_email")
+        u = ensure(email, event.get("user_nickname"))
+        ts = event.get("timestamp")
+        if ts and (not u["last_activity_at"] or ts > u["last_activity_at"]):
+            u["last_activity_at"] = ts
+        u["total_content_chars"] += int(event.get("content_chars") or event.get("extracted_chars") or 0)
+        sensitivity = event.get("sensitivity") or "none"
+        labels = event.get("sensitive_labels") or []
+        is_sensitive = sensitivity != "none" or bool(labels)
+
+        if event_type == "chat_message":
+            summary["chat_events"] += 1
+            if event.get("role") == "user":
+                summary["user_messages"] += 1
+                u["user_messages"] += 1
+            elif event.get("role") == "assistant":
+                summary["assistant_messages"] += 1
+                u["assistant_messages"] += 1
+        elif event_type == "document_upload":
+            summary["document_uploads"] += 1
+            u["document_uploads"] += 1
+        elif event_type == "clear_command":
+            summary["clear_events"] += 1
+            u["clear_events"] += 1
+        elif event_type in AUDIT_DELETE_EVENTS:
+            summary["delete_events"] += 1
+            u["delete_events"] += 1
+            deletion_events.append(_public_audit_event(event))
+
+        if is_sensitive:
+            summary["sensitive_events"] += 1
+            u["sensitive_events"] += 1
+            if sensitivity == "high":
+                summary["high_sensitive_events"] += 1
+                u["high_sensitive_events"] += 1
+            sensitive_events.append(_public_audit_event(event))
+
+    allowed_keys = {
+        "event_type", "timestamp", "role", "user_email", "user_nickname", "source",
+        "conversation_id", "command", "scope", "target_email", "filename", "mime_type",
+        "ext", "bytes", "extracted_chars", "graph_node", "keep_last", "removed", "kept",
+        "started_at", "sensitivity", "sensitive_labels", "content_preview", "content_chars",
+    }
+    recent = [_public_audit_event(e) for e in events[-50:]]
+
+    result: Dict[str, Any] = {
+        "summary": summary,
+        "per_user": sorted(per_user.values(), key=lambda u: u.get("last_activity_at") or "", reverse=True),
+        "recent_events": list(reversed(recent)),
+        "sensitive_events": sensitive_events[-30:],
+        "deletion_events": deletion_events[-30:],
+    }
+    if graph_stats:
+        result["summary"]["graph_nodes"] = graph_stats.get("total_nodes", 0)
+        result["summary"]["graph_edges"] = graph_stats.get("total_edges", 0)
+    return result
+
+
+def _public_audit_event(event: Dict) -> Dict:
+    allowed = {
+        "event_type", "timestamp", "role", "user_email", "user_nickname", "source",
+        "conversation_id", "command", "scope", "target_email", "filename", "mime_type",
+        "ext", "bytes", "extracted_chars", "graph_node", "keep_last", "removed", "kept",
+        "started_at", "sensitivity", "sensitive_labels", "content_preview", "content_chars",
+    }
+    return {k: event.get(k) for k in allowed if k in event}

package/latticeai/core/security.py

@@ -0,0 +1,131 @@
+"""Password hashing, rate limiting, IP detection, file-magic validation."""
+
+import hashlib
+import ipaddress
+import re
+import secrets
+import threading
+import time
+from typing import Dict, List, Optional
+
+from fastapi import HTTPException
+
+
+def hash_password(password: str) -> str:
+    salt = secrets.token_hex(16)
+    key = hashlib.scrypt(password.encode(), salt=salt.encode(), n=16384, r=8, p=1)
+    return f"{salt}:{key.hex()}"
+
+
+def verify_password(password: str, hashed: str) -> bool:
+    try:
+        salt, key_hex = hashed.split(":", 1)
+        key = hashlib.scrypt(password.encode(), salt=salt.encode(), n=16384, r=8, p=1)
+        return secrets.compare_digest(key.hex(), key_hex)
+    except Exception:
+        return False
+
+
+def host_is_loopback(host: str) -> bool:
+    if host in {"localhost", "127.0.0.1", "::1"}:
+        return True
+    try:
+        return ipaddress.ip_address(host).is_loopback
+    except ValueError:
+        return False
+
+
+def client_ip(request) -> str:
+    for header in ("CF-Connecting-IP", "X-Forwarded-For"):
+        val = request.headers.get(header)
+        if val:
+            return val.split(",")[0].strip()
+    return request.client.host if request.client else "unknown"
+
+
+_FILE_MAGIC: Dict[str, List[bytes]] = {
+    ".pdf":  [b"%PDF-"],
+    ".docx": [b"PK\x03\x04"],
+    ".xlsx": [b"PK\x03\x04"],
+    ".pptx": [b"PK\x03\x04"],
+    ".zip":  [b"PK\x03\x04", b"PK\x05\x06", b"PK\x07\x08"],
+    ".png":  [b"\x89PNG\r\n\x1a\n"],
+    ".jpg":  [b"\xff\xd8\xff"],
+    ".jpeg": [b"\xff\xd8\xff"],
+    ".gif":  [b"GIF87a", b"GIF89a"],
+}
+
+
+def bytes_match_extension(data: bytes, ext: str) -> bool:
+    ext = (ext or "").lower()
+    signatures = _FILE_MAGIC.get(ext)
+    if not signatures:
+        return True
+    head = data[:16]
+    return any(head.startswith(sig) for sig in signatures)
+
+
+def redact_secret_text(text: str) -> str:
+    if not text:
+        return ""
+    patterns = [
+        r"(?i)(api[_ -]?key|secret|token|password|passwd)\s*[:=]\s*['\"]?([A-Za-z0-9_\-\.]{12,})['\"]?",
+        r"\b(sk-[A-Za-z0-9_\-]{16,})\b",
+        r"\b(xai-[A-Za-z0-9_\-]{16,})\b",
+        r"\b(gsk_[A-Za-z0-9_\-]{16,})\b",
+    ]
+    redacted = str(text)
+    for pattern in patterns:
+        redacted = re.sub(pattern, lambda m: f"{m.group(1)}=[REDACTED]" if len(m.groups()) > 1 else "[REDACTED]", redacted)
+    return redacted
+
+
+# ── IP-based rate limiting (registration / login) ────────────────────────────
+_ip_rate_windows: dict = {}
+_ip_rate_lock = threading.Lock()
+
+
+def check_ip_rate_limit(ip: str, action: str, max_calls: int, window_secs: float) -> None:
+    key = (ip, action)
+    now = time.time()
+    cutoff = now - window_secs
+    with _ip_rate_lock:
+        calls = [t for t in _ip_rate_windows.get(key, []) if t > cutoff]
+        if len(calls) >= max_calls:
+            raise HTTPException(status_code=429, detail="요청이 너무 많습니다. 잠시 후 다시 시도하세요.")
+        calls.append(now)
+        _ip_rate_windows[key] = calls
+
+
+# ── Per-user token-bucket rate limiting ──────────────────────────────────────
+_RATE_LIMITS = {
+    "chat":   (30, 0.5),
+    "agent":  (10, 0.1),
+    "upload": (20, 0.2),
+}
+_rate_buckets: Dict[str, Dict[str, float]] = {}
+_user_rate_lock = threading.Lock()
+
+
+def enforce_rate_limit(email: str, bucket_key: str, *, enabled: bool = True) -> None:
+    if not enabled or not email:
+        return
+    cap, refill = _RATE_LIMITS.get(bucket_key, (60, 1.0))
+    key = f"{email}:{bucket_key}"
+    now = time.time()
+    with _user_rate_lock:
+        bucket = _rate_buckets.get(key)
+        if bucket is None:
+            _rate_buckets[key] = {"tokens": cap - 1, "ts": now}
+            return
+        elapsed = now - bucket["ts"]
+        bucket["tokens"] = min(cap, bucket["tokens"] + elapsed * refill)
+        bucket["ts"] = now
+        if bucket["tokens"] < 1:
+            retry_after = max(1, int((1 - bucket["tokens"]) / refill))
+            raise HTTPException(
+                status_code=429,
+                detail=f"Rate limit exceeded for {bucket_key}. Retry after {retry_after}s.",
+                headers={"Retry-After": str(retry_after)},
+            )
+        bucket["tokens"] -= 1

package/latticeai/core/sessions.py

@@ -0,0 +1,72 @@
+"""File-backed session store with sliding-window TTL."""
+
+import json
+import logging
+import os
+import secrets
+import threading
+import time
+from pathlib import Path
+from typing import Dict, Optional
+
+SESSION_TTL = 60 * 60 * 24  # 24 hours
+SESSION_REFRESH_THRESHOLD = 60 * 15  # only persist if >15 min since last bump
+_lock = threading.Lock()
+
+
+def _sessions_file(data_dir: Optional[Path] = None) -> Path:
+    d = data_dir or Path(os.getenv("LATTICEAI_DATA_DIR") or (Path.home() / ".ltcai"))
+    d.mkdir(parents=True, exist_ok=True)
+    return d / "sessions.json"
+
+
+def load_sessions(data_dir: Optional[Path] = None) -> Dict[str, tuple]:
+    try:
+        f = _sessions_file(data_dir)
+        if f.exists():
+            raw = json.loads(f.read_text())
+            return {k: tuple(v) for k, v in raw.items()}
+    except Exception as e:
+        logging.warning("load_sessions failed (starting empty): %s", e)
+    return {}
+
+
+def persist_sessions(sessions: Dict[str, tuple], data_dir: Optional[Path] = None) -> None:
+    try:
+        _sessions_file(data_dir).write_text(json.dumps({k: list(v) for k, v in sessions.items()}, ensure_ascii=False))
+    except Exception as e:
+        logging.warning("persist_sessions failed: %s", e)
+
+
+class SessionStore:
+    def __init__(self, data_dir: Optional[Path] = None):
+        self._data_dir = data_dir
+        self._sessions: Dict[str, tuple] = load_sessions(data_dir)
+
+    def create(self, email: str) -> str:
+        token = secrets.token_urlsafe(32)
+        with _lock:
+            self._sessions[token] = (email, time.time())
+            persist_sessions(self._sessions, self._data_dir)
+        return token
+
+    def get_email(self, token: str) -> Optional[str]:
+        now = time.time()
+        with _lock:
+            entry = self._sessions.get(token)
+            if entry is None:
+                return None
+            email, created_at = entry
+            if now - created_at > SESSION_TTL:
+                self._sessions.pop(token, None)
+                persist_sessions(self._sessions, self._data_dir)
+                return None
+            if now - created_at > SESSION_REFRESH_THRESHOLD:
+                self._sessions[token] = (email, now)
+                persist_sessions(self._sessions, self._data_dir)
+            return email
+
+    def invalidate(self, token: str) -> None:
+        with _lock:
+            self._sessions.pop(token, None)
+            persist_sessions(self._sessions, self._data_dir)

package/llm_router.py

@@ -100,7 +100,7 @@ OPENAI_COMPATIBLE_PROVIDERS = {
         "env_key": "VLLM_API_KEY",
         "base_url_env": "VLLM_BASE_URL",
         "base_url": "http://localhost:8000/v1",
-        "default_model": "Qwen/Qwen2.5-7B-Instruct",
+        "default_model": "meta-llama/Llama-3.1-8B-Instruct",
         "api_key_fallback": "vllm",
     },
     "lmstudio": {
@@ -121,29 +121,35 @@ OPENAI_COMPATIBLE_PROVIDERS = {
 
 PROVIDER_MODEL_CATALOG = {
     "openai": [
+        {"id": "gpt-5.5", "name": "GPT-5.5", "family": "GPT"},
+        {"id": "gpt-5.4", "name": "GPT-5.4", "family": "GPT"},
+        {"id": "gpt-5.4-mini", "name": "GPT-5.4 Mini", "family": "GPT"},
+        {"id": "gpt-5.4-nano", "name": "GPT-5.4 Nano", "family": "GPT"},
         {"id": "gpt-4o-mini", "name": "GPT-4o Mini", "family": "GPT"},
         {"id": "gpt-4o", "name": "GPT-4o", "family": "GPT"},
         {"id": "gpt-4.1-mini", "name": "GPT-4.1 Mini", "family": "GPT"},
         {"id": "gpt-4.1", "name": "GPT-4.1", "family": "GPT"},
     ],
     "openrouter": [
+        {"id": "openai/gpt-5.5", "name": "GPT-5.5 via OpenRouter", "family": "GPT"},
         {"id": "openai/gpt-4o-mini", "name": "GPT-4o Mini via OpenRouter", "family": "GPT"},
-        {"id": "anthropic/claude-sonnet-4-6", "name": "Claude Sonnet 4.6 via OpenRouter", "family": "Claude"},
-        {"id": "anthropic/claude-haiku-4-5", "name": "Claude Haiku 4.5 via OpenRouter", "family": "Claude"},
+        {"id": "anthropic/claude-opus-4.7", "name": "Claude Opus 4.7 via OpenRouter", "family": "Claude"},
+        {"id": "anthropic/claude-sonnet-4.6", "name": "Claude Sonnet 4.6 via OpenRouter", "family": "Claude"},
+        {"id": "anthropic/claude-haiku-4.5", "name": "Claude Haiku 4.5 via OpenRouter", "family": "Claude"},
+        {"id": "qwen/qwen3-vl-235b-a22b-instruct", "name": "Qwen3-VL 235B A22B via OpenRouter", "family": "Qwen"},
+        {"id": "qwen/qwen3-coder", "name": "Qwen3 Coder via OpenRouter", "family": "Qwen"},
         {"id": "x-ai/grok-2", "name": "Grok 2 via OpenRouter", "family": "Grok"},
         {"id": "meta-llama/llama-3.3-70b-instruct", "name": "Llama 3.3 70B via OpenRouter", "family": "Llama"},
-        {"id": "qwen/qwen-2.5-72b-instruct", "name": "Qwen 2.5 72B via OpenRouter", "family": "Qwen"},
         {"id": "google/gemini-2.5-flash", "name": "Gemini 2.5 Flash via OpenRouter", "family": "Gemini"},
     ],
     "groq": [
+        {"id": "qwen/qwen3-32b", "name": "Qwen3 32B", "family": "Qwen"},
         {"id": "llama-3.1-8b-instant", "name": "Llama 3.1 8B Instant", "family": "Llama"},
         {"id": "llama-3.3-70b-versatile", "name": "Llama 3.3 70B Versatile", "family": "Llama"},
-        {"id": "qwen-qwq-32b", "name": "Qwen QwQ 32B", "family": "Qwen"},
     ],
     "together": [
+        {"id": "Qwen/Qwen3-VL-32B-Instruct", "name": "Qwen3-VL 32B", "family": "Qwen"},
         {"id": "meta-llama/Llama-3.3-70B-Instruct-Turbo", "name": "Llama 3.3 70B Turbo", "family": "Llama"},
-        {"id": "Qwen/Qwen2.5-72B-Instruct-Turbo", "name": "Qwen 2.5 72B Turbo", "family": "Qwen"},
-        {"id": "deepseek-ai/DeepSeek-R1", "name": "DeepSeek R1", "family": "DeepSeek"},
         {"id": "mistralai/Mixtral-8x22B-Instruct-v0.1", "name": "Mixtral 8x22B", "family": "Mistral"},
     ],
     "xai": [