ltcai 3.5.0 → 4.0.0

package/knowledge_graph_api.py

@@ -1,130 +1,14 @@
-"""Knowledge graph page and API routes."""
+"""Deprecation shim — the knowledge graph router moved in v4.
 
-from pathlib import Path
-from typing import Any, Callable, Dict, Optional
+The ``/knowledge-graph/*`` data router (and legacy ``/graph`` page routes)
+now live in :mod:`latticeai.api.knowledge_graph`. This root module remains
+importable for the deprecation window and will be removed in a future major
+release.
+"""
 
-from fastapi import APIRouter, HTTPException, Request
-from fastapi.responses import FileResponse
-from pydantic import BaseModel
+from latticeai.api.knowledge_graph import (  # noqa: F401
+    KnowledgeGraphIngestRequest,
+    create_knowledge_graph_router,
+)
 
-
-class KnowledgeGraphIngestRequest(BaseModel):
-    type: str
-    content: str = ""
-    role: Optional[str] = None
-    title: Optional[str] = None
-    source: Optional[str] = None
-    conversation_id: Optional[str] = None
-    user_email: Optional[str] = None
-    user_nickname: Optional[str] = None
-    metadata: Optional[Dict[str, Any]] = None
-
-
-def create_knowledge_graph_router(
-    *,
-    get_graph: Callable[[], Any],
-    require_graph: Callable[[], None],
-    require_user: Callable[[Request], str],
-    static_dir: Path,
-) -> APIRouter:
-    router = APIRouter()
-
-    def graph():
-        require_graph()
-        return get_graph()
-
-    @router.get("/graph")
-    async def knowledge_graph_page(request: Request):
-        """Serve the interactive knowledge graph canvas UI."""
-        graph()
-        require_user(request)
-        response = FileResponse(static_dir / "graph.html")
-        response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
-        response.headers["Pragma"] = "no-cache"
-        response.headers["Expires"] = "0"
-        return response
-
-    @router.get("/knowledge-graph")
-    async def knowledge_graph_legacy_page(request: Request):
-        """Backward-compatible route for the graph page."""
-        graph()
-        require_user(request)
-        response = FileResponse(static_dir / "graph.html")
-        response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
-        response.headers["Pragma"] = "no-cache"
-        response.headers["Expires"] = "0"
-        return response
-
-    @router.get("/knowledge-graph/stats")
-    async def knowledge_graph_stats(request: Request):
-        require_user(request)
-        return graph().stats()
-
-    @router.get("/knowledge-graph/schema")
-    async def knowledge_graph_schema(request: Request):
-        require_user(request)
-        stats = graph().stats()
-        return {
-            "legacy_schema_version": stats.get("schema_version"),
-            "v2_schema_available": stats.get("v2_schema_available"),
-            "v2": stats.get("v2"),
-        }
-
-    @router.get("/knowledge-graph/graph")
-    async def knowledge_graph_data(request: Request, limit: int = 300):
-        require_user(request)
-        return graph().graph(limit)
-
-    @router.get("/knowledge-graph/documents")
-    async def knowledge_graph_documents(request: Request, limit: int = 200):
-        """Ingested documents (uploads + indexed local docs) with index state.
-
-        Backs the Files view so uploaded content is visible end-to-end:
-        upload → Files → Knowledge Graph → Hybrid Search → Chat.
-        """
-        require_user(request)
-        return graph().list_documents(limit)
-
-    @router.get("/knowledge-graph/search")
-    async def knowledge_graph_search(q: str, request: Request, limit: int = 30):
-        require_user(request)
-        if not q or not q.strip():
-            return {"query": q, "matches": []}
-        return graph().search(q, limit)
-
-    @router.get("/knowledge-graph/context")
-    async def knowledge_graph_context(q: str, request: Request, limit: int = 6):
-        require_user(request)
-        return {"query": q, "context": graph().context_for_query(q, limit)}
-
-    @router.get("/knowledge-graph/neighbors/{node_id:path}")
-    async def knowledge_graph_neighbors(node_id: str, request: Request):
-        require_user(request)
-        if not node_id:
-            raise HTTPException(status_code=400, detail="node_id required")
-        return graph().neighbors(node_id)
-
-    @router.post("/knowledge-graph/ingest")
-    async def knowledge_graph_ingest(req: KnowledgeGraphIngestRequest, request: Request):
-        current_user = require_user(request)
-        kg = graph()
-        event_type = (req.type or "").strip().lower()
-        if event_type not in {"message", "ai_response", "note"}:
-            raise HTTPException(status_code=400, detail="지원하는 type: message, ai_response, note")
-        role = req.role or ("assistant" if event_type == "ai_response" else "user")
-        return kg.ingest_message(
-            role,
-            req.content,
-            user_email=req.user_email or current_user,
-            user_nickname=req.user_nickname,
-            source=req.source or "mcp",
-            conversation_id=req.conversation_id,
-            raw={
-                "type": req.type,
-                "title": req.title,
-                "content": req.content,
-                "metadata": req.metadata or {},
-            },
-        )
-
-    return router
+__all__ = ["KnowledgeGraphIngestRequest", "create_knowledge_graph_router"]

package/latticeai/__init__.py

@@ -1,3 +1,3 @@
 """Lattice AI - modular server package."""
 
-__version__ = "3.5.0"
+__version__ = "4.0.0"

package/latticeai/api/admin.py

@@ -2,7 +2,7 @@
 
 import logging
 from collections import defaultdict
-from typing import Any, Callable, Dict, List, Optional
+from typing import Callable, Dict, List, Optional
 
 from fastapi import APIRouter, HTTPException, Request
 from pydantic import BaseModel

package/latticeai/api/agents.py

@@ -163,7 +163,13 @@ def create_agents_router(
         current_user = require_user(request)
         scope = gate_write(request)
         try:
-            return runtime.start(
+            # Worker thread: an LLM-backed run blocks on model generation and
+            # must not stall the event loop (the sync model bridge also
+            # requires a loop-free thread).
+            import asyncio
+
+            return await asyncio.to_thread(
+                runtime.start,
                 req.goal,
                 user_email=current_user or None,
                 scope=scope,

package/latticeai/api/auth.py

@@ -1,5 +1,7 @@
 """Authentication API router: register, login, logout, SSO, profile."""
 
+import base64
+import hashlib
 import logging
 import secrets
 import time
@@ -69,11 +71,22 @@ def create_auth_router(
 ) -> APIRouter:
     router = APIRouter()
 
+    def _enforce_password_policy(password: str) -> None:
+        # Real policy (v4): length >= 8 with letters AND digits. A 4-char
+        # minimum was not a policy.
+        pw = str(password or "")
+        if len(pw) < 8 or not any(c.isalpha() for c in pw) or not any(c.isdigit() for c in pw):
+            raise HTTPException(
+                status_code=400,
+                detail="비밀번호는 8자 이상이며 영문자와 숫자를 모두 포함해야 합니다.",
+            )
+
     @router.post("/register")
     async def register(req: UserRegister, request: Request):
         check_ip_rate_limit(client_ip(request), "register", max_calls=5, window_secs=3600)
         if not open_registration:
             raise HTTPException(status_code=403, detail="회원가입이 비활성화되어 있습니다. 관리자에게 문의하세요.")
+        _enforce_password_policy(req.password)
         users = load_users()
         if req.email in users:
             raise HTTPException(status_code=400, detail="이미 존재하는 이메일입니다.")
@@ -123,7 +136,15 @@ def create_auth_router(
             raise HTTPException(status_code=503, detail="SSO가 설정되지 않았습니다.")
         state = secrets.token_urlsafe(16)
         nonce = secrets.token_urlsafe(16)
-        _sso_states[state] = (time.time(), nonce)
+        # PKCE (S256): bind the token exchange to this login, so an
+        # intercepted authorization code is useless without the verifier.
+        code_verifier = secrets.token_urlsafe(48)
+        code_challenge = (
+            base64.urlsafe_b64encode(hashlib.sha256(code_verifier.encode("ascii")).digest())
+            .rstrip(b"=")
+            .decode("ascii")
+        )
+        _sso_states[state] = (time.time(), nonce, code_verifier)
         params = urlencode({
             "client_id": settings["client_id"],
             "response_type": "code",
@@ -131,6 +152,8 @@ def create_auth_router(
             "scope": settings.get("scopes") or "openid email profile",
             "state": state,
             "nonce": nonce,
+            "code_challenge": code_challenge,
+            "code_challenge_method": "S256",
         })
         return RedirectResponse(f"{discovery['authorization_endpoint']}?{params}")
 
@@ -141,7 +164,7 @@ def create_auth_router(
         entry = _sso_states.pop(state, None)
         if entry is None or time.time() - entry[0] > 300:
             raise HTTPException(status_code=400, detail="유효하지 않은 SSO 상태입니다.")
-        _, nonce = entry
+        _, nonce, code_verifier = entry
         settings = get_sso_settings()
         discovery = await get_sso_discovery()
         if not settings.get("enabled") or not discovery:
@@ -154,6 +177,7 @@ def create_auth_router(
                 "redirect_uri": settings["redirect_uri"],
                 "client_id": settings["client_id"],
                 "client_secret": settings["client_secret"],
+                "code_verifier": code_verifier,
             }, headers={"Accept": "application/json"}, timeout=15)
             tokens = r.json()
         id_token = tokens.get("id_token")
@@ -214,8 +238,7 @@ def create_auth_router(
         email = require_user(request)
         if not email:
             raise HTTPException(status_code=401, detail="인증이 필요합니다.")
-        if len(req.new_password) < 4:
-            raise HTTPException(status_code=400, detail="새 비밀번호는 4자 이상이어야 합니다.")
+        _enforce_password_policy(req.new_password)
         users = load_users()
         user = users.get(email)
         if not user:

package/latticeai/api/browser.py

@@ -0,0 +1,217 @@
+"""Browser & web ingestion — Knowledge Graph inputs, not standalone features.
+
+v3.6.0 Knowledge Graph First: a public URL or an open browser tab is just another
+source that converges into the Knowledge Graph through the unified ingestion
+pipeline. Everything runs on the **local runtime** — the server fetches/reads
+locally, stores into local SQLite, and never uploads to a cloud service.
+
+Two layers, both feeding ``IngestionPipeline.ingest``:
+
+* ``POST /api/browser/read-url`` — the runtime fetches a public URL locally,
+  extracts readable text, stores it as ``source_type=web_url``. Fails gracefully
+  on blocked / login-required pages.
+* ``POST /api/browser/ingest-current-tab`` — accepts a payload from the local
+  browser extension (url/title/text/selected_text/html), sanitizes + size-limits
+  it, stores it as ``source_type=browser_tab``.
+"""
+
+from __future__ import annotations
+
+from html.parser import HTMLParser
+from typing import Any, Callable, Optional, Tuple
+from urllib.parse import urlparse
+
+from fastapi import APIRouter, HTTPException, Request
+from pydantic import BaseModel
+
+from latticeai.services.ingestion import IngestionItem
+
+MAX_TAB_BYTES = 4 * 1024 * 1024          # 4 MB per captured tab payload
+MAX_URL_FETCH_BYTES = 4 * 1024 * 1024    # 4 MB cap on a fetched page
+URL_FETCH_TIMEOUT = 12.0                  # seconds
+
+
+class BrowserFetchError(Exception):
+    """A URL could not be fetched (blocked, login-required, timeout, too big)."""
+
+
+# ── readable-text extraction ─────────────────────────────────────────────────
+_SKIP_TAGS = {"script", "style", "noscript", "template", "svg", "head"}
+
+
+class _TextExtractor(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self._skip_depth = 0
+        self._chunks: list[str] = []
+        self.title: str = ""
+        self._in_title = False
+
+    def handle_starttag(self, tag, attrs):
+        if tag in _SKIP_TAGS:
+            self._skip_depth += 1
+        if tag == "title":
+            self._in_title = True
+
+    def handle_endtag(self, tag):
+        if tag in _SKIP_TAGS and self._skip_depth > 0:
+            self._skip_depth -= 1
+        if tag == "title":
+            self._in_title = False
+        if tag in {"p", "div", "br", "li", "h1", "h2", "h3", "h4", "section", "article"}:
+            self._chunks.append("\n")
+
+    def handle_data(self, data):
+        if self._in_title:
+            self.title += data
+        if self._skip_depth == 0:
+            text = data.strip()
+            if text:
+                self._chunks.append(text)
+
+    def text(self) -> str:
+        raw = " ".join(self._chunks)
+        # collapse runs of whitespace while keeping paragraph breaks
+        lines = [ln.strip() for ln in raw.replace("\r", "").split("\n")]
+        return "\n".join([ln for ln in lines if ln]).strip()
+
+
+def extract_readable_text(html: str) -> Tuple[str, str]:
+    """Return (title, readable_text) from an HTML string. Never raises."""
+    parser = _TextExtractor()
+    try:
+        parser.feed(html or "")
+    except Exception:  # noqa: BLE001 — malformed HTML must still yield best-effort text
+        pass
+    return parser.title.strip(), parser.text()
+
+
+def _default_fetch_url(url: str) -> Tuple[str, str]:
+    """Fetch a public URL on the local runtime and extract readable text.
+
+    Raises :class:`BrowserFetchError` on any non-success (blocked, login wall,
+    timeout, oversized, non-HTML) so the route can fail gracefully.
+    """
+    import httpx
+
+    try:
+        with httpx.Client(
+            follow_redirects=True, timeout=URL_FETCH_TIMEOUT,
+            headers={"User-Agent": "LatticeAI-local/3.6 (+local-first knowledge graph)"},
+        ) as client:
+            resp = client.get(url)
+    except httpx.HTTPError as exc:
+        raise BrowserFetchError(f"Could not reach the page: {exc}") from exc
+
+    if resp.status_code in (401, 403):
+        raise BrowserFetchError("The page is login-required or blocked (HTTP %s)." % resp.status_code)
+    if resp.status_code >= 400:
+        raise BrowserFetchError(f"The page returned HTTP {resp.status_code}.")
+    content_type = resp.headers.get("content-type", "")
+    if content_type and "html" not in content_type and "text" not in content_type:
+        raise BrowserFetchError(f"Unsupported content type: {content_type or 'unknown'}.")
+    body = resp.text or ""
+    if len(body.encode("utf-8", "ignore")) > MAX_URL_FETCH_BYTES:
+        body = body.encode("utf-8", "ignore")[:MAX_URL_FETCH_BYTES].decode("utf-8", "ignore")
+    title, text = extract_readable_text(body)
+    return (title or url, text)
+
+
+def _validate_http_url(url: str) -> str:
+    url = (url or "").strip()
+    if not url:
+        raise HTTPException(status_code=400, detail="url is required.")
+    parsed = urlparse(url)
+    if parsed.scheme not in ("http", "https"):
+        raise HTTPException(status_code=400, detail="Only http(s) URLs are supported.")
+    if not parsed.netloc:
+        raise HTTPException(status_code=400, detail="Malformed URL.")
+    return url
+
+
+# ── request models ───────────────────────────────────────────────────────────
+class ReadUrlRequest(BaseModel):
+    url: str
+    workspace_id: Optional[str] = None
+
+
+class IngestTabRequest(BaseModel):
+    url: str
+    title: Optional[str] = None
+    text: Optional[str] = None
+    selected_text: Optional[str] = None
+    html: Optional[str] = None
+    captured_at: Optional[str] = None
+    workspace_id: Optional[str] = None
+
+
+def create_browser_router(
+    *,
+    pipeline: Any,
+    require_user: Callable[[Request], str],
+    fetch_url: Optional[Callable[[str], Tuple[str, str]]] = None,
+    max_tab_bytes: int = MAX_TAB_BYTES,
+) -> APIRouter:
+    router = APIRouter()
+    _fetch = fetch_url or _default_fetch_url
+
+    def _require_pipeline():
+        if pipeline is None or not pipeline.available():
+            raise HTTPException(status_code=503, detail="Knowledge Graph ingestion is disabled.")
+
+    @router.post("/api/browser/read-url")
+    async def read_url(req: ReadUrlRequest, request: Request):
+        """Fetch a public URL locally and ingest it as a web_url source."""
+        user = require_user(request)
+        _require_pipeline()
+        url = _validate_http_url(req.url)
+        try:
+            title, text = _fetch(url)
+        except BrowserFetchError as exc:
+            # Graceful failure — not a 5xx; the page was simply unreadable.
+            raise HTTPException(status_code=422, detail=str(exc))
+        if not (text or "").strip():
+            return {"status": "empty", "source_type": "web_url", "url": url,
+                    "detail": "No readable text was extracted from the page."}
+        res = pipeline.ingest(
+            IngestionItem(
+                source_type="web_url", title=title, text=text, source_uri=url,
+                owner=user, workspace_id=req.workspace_id,
+            ),
+            user_email=user,
+        )
+        return res.as_dict()
+
+    @router.post("/api/browser/ingest-current-tab")
+    async def ingest_current_tab(req: IngestTabRequest, request: Request):
+        """Ingest a payload captured from the local browser extension."""
+        user = require_user(request)
+        _require_pipeline()
+        url = _validate_http_url(req.url)
+        # Sanitize: reject an oversized payload before doing any work.
+        for value in (req.text, req.html, req.selected_text):
+            if value and len(value.encode("utf-8", "ignore")) > max_tab_bytes:
+                raise HTTPException(status_code=413, detail="Captured payload is too large.")
+        text = (req.text or "").strip()
+        if not text and req.html:
+            _title, text = extract_readable_text(req.html)
+        if not text:
+            text = (req.selected_text or "").strip()
+        if not text:
+            raise HTTPException(status_code=400, detail="No text, html, or selected_text provided.")
+        res = pipeline.ingest(
+            IngestionItem(
+                source_type="browser_tab",
+                title=req.title or url,
+                text=text,
+                source_uri=url,
+                captured_at=req.captured_at,
+                owner=user,
+                workspace_id=req.workspace_id,
+                metadata={"has_selection": bool(req.selected_text)},
+            ),
+            user_email=user,
+        )
+        return res.as_dict()
+
+    return router