ltcai 0.1.29 → 0.1.31

package/knowledge_graph_api.py

@@ -0,0 +1,112 @@
+"""Knowledge graph page and API routes."""
+
+from pathlib import Path
+from typing import Any, Callable, Dict, Optional
+
+from fastapi import APIRouter, HTTPException, Request
+from fastapi.responses import FileResponse
+from pydantic import BaseModel
+
+
+class KnowledgeGraphIngestRequest(BaseModel):
+    type: str
+    content: str = ""
+    role: Optional[str] = None
+    title: Optional[str] = None
+    source: Optional[str] = None
+    conversation_id: Optional[str] = None
+    user_email: Optional[str] = None
+    user_nickname: Optional[str] = None
+    metadata: Optional[Dict[str, Any]] = None
+
+
+def create_knowledge_graph_router(
+    *,
+    get_graph: Callable[[], Any],
+    require_graph: Callable[[], None],
+    require_user: Callable[[Request], str],
+    static_dir: Path,
+) -> APIRouter:
+    router = APIRouter()
+
+    def graph():
+        require_graph()
+        return get_graph()
+
+    @router.get("/graph")
+    async def knowledge_graph_page(request: Request):
+        """Serve the interactive knowledge graph canvas UI."""
+        graph()
+        require_user(request)
+        return FileResponse(static_dir / "graph.html")
+
+    @router.get("/knowledge-graph")
+    async def knowledge_graph_legacy_page(request: Request):
+        """Backward-compatible route for the graph page."""
+        graph()
+        require_user(request)
+        return FileResponse(static_dir / "graph.html")
+
+    @router.get("/knowledge-graph/stats")
+    async def knowledge_graph_stats(request: Request):
+        require_user(request)
+        return graph().stats()
+
+    @router.get("/knowledge-graph/schema")
+    async def knowledge_graph_schema(request: Request):
+        require_user(request)
+        stats = graph().stats()
+        return {
+            "legacy_schema_version": stats.get("schema_version"),
+            "v2_schema_available": stats.get("v2_schema_available"),
+            "v2": stats.get("v2"),
+        }
+
+    @router.get("/knowledge-graph/graph")
+    async def knowledge_graph_data(request: Request, limit: int = 300):
+        require_user(request)
+        return graph().graph(limit)
+
+    @router.get("/knowledge-graph/search")
+    async def knowledge_graph_search(q: str, request: Request, limit: int = 30):
+        require_user(request)
+        if not q or not q.strip():
+            return {"query": q, "matches": []}
+        return graph().search(q, limit)
+
+    @router.get("/knowledge-graph/context")
+    async def knowledge_graph_context(q: str, request: Request, limit: int = 6):
+        require_user(request)
+        return {"query": q, "context": graph().context_for_query(q, limit)}
+
+    @router.get("/knowledge-graph/neighbors/{node_id:path}")
+    async def knowledge_graph_neighbors(node_id: str, request: Request):
+        require_user(request)
+        if not node_id:
+            raise HTTPException(status_code=400, detail="node_id required")
+        return graph().neighbors(node_id)
+
+    @router.post("/knowledge-graph/ingest")
+    async def knowledge_graph_ingest(req: KnowledgeGraphIngestRequest, request: Request):
+        current_user = require_user(request)
+        kg = graph()
+        event_type = (req.type or "").strip().lower()
+        if event_type not in {"message", "ai_response", "note"}:
+            raise HTTPException(status_code=400, detail="지원하는 type: message, ai_response, note")
+        role = req.role or ("assistant" if event_type == "ai_response" else "user")
+        return kg.ingest_message(
+            role,
+            req.content,
+            user_email=req.user_email or current_user,
+            user_nickname=req.user_nickname,
+            source=req.source or "mcp",
+            conversation_id=req.conversation_id,
+            raw={
+                "type": req.type,
+                "title": req.title,
+                "content": req.content,
+                "metadata": req.metadata or {},
+            },
+        )
+
+    return router

package/llm_router.py

@@ -100,7 +100,7 @@ OPENAI_COMPATIBLE_PROVIDERS = {
         "env_key": "VLLM_API_KEY",
         "base_url_env": "VLLM_BASE_URL",
         "base_url": "http://localhost:8000/v1",
-        "default_model": "Qwen/Qwen2.5-7B-Instruct",
+        "default_model": "meta-llama/Llama-3.1-8B-Instruct",
         "api_key_fallback": "vllm",
     },
     "lmstudio": {
@@ -121,29 +121,35 @@ OPENAI_COMPATIBLE_PROVIDERS = {
 
 PROVIDER_MODEL_CATALOG = {
     "openai": [
+        {"id": "gpt-5.5", "name": "GPT-5.5", "family": "GPT"},
+        {"id": "gpt-5.4", "name": "GPT-5.4", "family": "GPT"},
+        {"id": "gpt-5.4-mini", "name": "GPT-5.4 Mini", "family": "GPT"},
+        {"id": "gpt-5.4-nano", "name": "GPT-5.4 Nano", "family": "GPT"},
         {"id": "gpt-4o-mini", "name": "GPT-4o Mini", "family": "GPT"},
         {"id": "gpt-4o", "name": "GPT-4o", "family": "GPT"},
         {"id": "gpt-4.1-mini", "name": "GPT-4.1 Mini", "family": "GPT"},
         {"id": "gpt-4.1", "name": "GPT-4.1", "family": "GPT"},
     ],
     "openrouter": [
+        {"id": "openai/gpt-5.5", "name": "GPT-5.5 via OpenRouter", "family": "GPT"},
         {"id": "openai/gpt-4o-mini", "name": "GPT-4o Mini via OpenRouter", "family": "GPT"},
-        {"id": "anthropic/claude-3.5-sonnet", "name": "Claude 3.5 Sonnet via OpenRouter", "family": "Claude"},
-        {"id": "anthropic/claude-3.5-haiku", "name": "Claude 3.5 Haiku via OpenRouter", "family": "Claude"},
+        {"id": "anthropic/claude-opus-4.7", "name": "Claude Opus 4.7 via OpenRouter", "family": "Claude"},
+        {"id": "anthropic/claude-sonnet-4.6", "name": "Claude Sonnet 4.6 via OpenRouter", "family": "Claude"},
+        {"id": "anthropic/claude-haiku-4.5", "name": "Claude Haiku 4.5 via OpenRouter", "family": "Claude"},
+        {"id": "qwen/qwen3-vl-235b-a22b-instruct", "name": "Qwen3-VL 235B A22B via OpenRouter", "family": "Qwen"},
+        {"id": "qwen/qwen3-coder", "name": "Qwen3 Coder via OpenRouter", "family": "Qwen"},
         {"id": "x-ai/grok-2", "name": "Grok 2 via OpenRouter", "family": "Grok"},
         {"id": "meta-llama/llama-3.3-70b-instruct", "name": "Llama 3.3 70B via OpenRouter", "family": "Llama"},
-        {"id": "qwen/qwen-2.5-72b-instruct", "name": "Qwen 2.5 72B via OpenRouter", "family": "Qwen"},
-        {"id": "google/gemini-2.0-flash-exp", "name": "Gemini 2 Flash via OpenRouter", "family": "Gemini"},
+        {"id": "google/gemini-2.5-flash", "name": "Gemini 2.5 Flash via OpenRouter", "family": "Gemini"},
     ],
     "groq": [
+        {"id": "qwen/qwen3-32b", "name": "Qwen3 32B", "family": "Qwen"},
         {"id": "llama-3.1-8b-instant", "name": "Llama 3.1 8B Instant", "family": "Llama"},
         {"id": "llama-3.3-70b-versatile", "name": "Llama 3.3 70B Versatile", "family": "Llama"},
-        {"id": "qwen-qwq-32b", "name": "Qwen QwQ 32B", "family": "Qwen"},
     ],
     "together": [
+        {"id": "Qwen/Qwen3-VL-32B-Instruct", "name": "Qwen3-VL 32B", "family": "Qwen"},
         {"id": "meta-llama/Llama-3.3-70B-Instruct-Turbo", "name": "Llama 3.3 70B Turbo", "family": "Llama"},
-        {"id": "Qwen/Qwen2.5-72B-Instruct-Turbo", "name": "Qwen 2.5 72B Turbo", "family": "Qwen"},
-        {"id": "deepseek-ai/DeepSeek-R1", "name": "DeepSeek R1", "family": "DeepSeek"},
         {"id": "mistralai/Mixtral-8x22B-Instruct-v0.1", "name": "Mixtral 8x22B", "family": "Mistral"},
     ],
     "xai": [
@@ -174,7 +180,7 @@ def parse_model_ref(model_id: str) -> tuple[str, str]:
         return "local_mlx", model_id.split(":", 1)[1]
     return "local_mlx", model_id
 
-HF_MODELS_ROOT = Path.home() / ".latticeai" / "hf-models"
+HF_MODELS_ROOT = Path.home() / ".ltcai" / "hf-models"
 
 def hf_model_dir(repo_id: str) -> Path:
     return HF_MODELS_ROOT / repo_id.replace("/", "__")

package/local_knowledge_api.py

@@ -0,0 +1,319 @@
+"""Local folder knowledge-source API and optional filesystem watcher."""
+
+import logging
+import threading
+from pathlib import Path
+from typing import Any, Callable, Dict, Optional
+
+from fastapi import APIRouter, HTTPException, Request
+from pydantic import BaseModel, Field
+
+
+class LocalTreeRequest(BaseModel):
+    path: str
+    max_items: int = 200
+    approved: bool = False
+    approval_token: Optional[str] = None
+
+
+class LocalKnowledgeAuditRequest(BaseModel):
+    path: str
+    include_ocr: bool = False
+    max_files: int = 50_000
+    approved: bool = False
+    approval_token: Optional[str] = None
+
+
+class LocalKnowledgeIndexRequest(BaseModel):
+    path: str
+    include_ocr: bool = False
+    watch_enabled: bool = False
+    max_files: int = 5_000
+    consent: Dict[str, Any] = Field(default_factory=dict)
+    approved: bool = False
+    approval_token: Optional[str] = None
+
+
+class LocalKnowledgeWatchRequest(BaseModel):
+    source_id: str
+
+
+class _LocalWatchHandler:
+    def __init__(self, schedule_change: Callable[[], None]):
+        self._schedule_change = schedule_change
+
+    def on_any_event(self, event):  # pragma: no cover - exercised by OS watcher
+        if getattr(event, "is_directory", False):
+            return
+        self._schedule_change()
+
+
+class LocalKnowledgeWatcher:
+    """Debounced watchdog wrapper for approved local knowledge sources."""
+
+    def __init__(self, get_graph: Callable[[], Any], *, debounce_seconds: float = 5.0):
+        self._get_graph = get_graph
+        self._debounce_seconds = debounce_seconds
+        self._lock = threading.Lock()
+        self._watched: Dict[str, Dict[str, Any]] = {}
+        self._observer_cls = None
+        self._event_handler_base = None
+        self._import_error = ""
+        try:
+            from watchdog.events import FileSystemEventHandler
+            from watchdog.observers import Observer
+
+            self._observer_cls = Observer
+            self._event_handler_base = FileSystemEventHandler
+        except Exception as exc:  # pragma: no cover - depends on optional dependency
+            self._import_error = str(exc)
+
+    @property
+    def available(self) -> bool:
+        return self._observer_cls is not None and self._event_handler_base is not None
+
+    def status(self) -> Dict[str, Any]:
+        with self._lock:
+            active = {
+                source_id: {
+                    "root_path": item["source"].get("root_path"),
+                    "last_event_at": item.get("last_event_at"),
+                    "last_indexed_at": item.get("last_indexed_at"),
+                    "last_error": item.get("last_error"),
+                }
+                for source_id, item in self._watched.items()
+            }
+        return {
+            "available": self.available,
+            "error": "" if self.available else self._import_error or "watchdog is not installed",
+            "debounce_seconds": self._debounce_seconds,
+            "active": active,
+        }
+
+    def restore_enabled_sources(self) -> Dict[str, Any]:
+        graph = self._get_graph()
+        if graph is None:
+            return {"restored": 0, "available": self.available}
+        restored = 0
+        try:
+            for source in graph.local_sources().get("sources", []):
+                if source.get("watch_enabled"):
+                    result = self.start_source(source)
+                    if result.get("watching"):
+                        restored += 1
+        except Exception as exc:
+            logging.warning("local knowledge watcher restore failed: %s", exc)
+        return {"restored": restored, "available": self.available}
+
+    def start_source(self, source: Dict[str, Any]) -> Dict[str, Any]:
+        source_id = str(source.get("id") or "")
+        root_path = str(source.get("root_path") or "")
+        if not source_id or not root_path:
+            return {"watching": False, "error": "source_id and root_path are required"}
+        if not self.available:
+            return {"watching": False, "source_id": source_id, "error": self._import_error or "watchdog is not installed"}
+        root = Path(root_path).expanduser().resolve()
+        if not root.exists() or not root.is_dir():
+            return {"watching": False, "source_id": source_id, "error": "source folder is not available"}
+
+        self.stop_source(source_id)
+
+        class Handler(_LocalWatchHandler, self._event_handler_base):  # type: ignore[misc, valid-type]
+            def __init__(handler_self):
+                self._event_handler_base.__init__(handler_self)
+                _LocalWatchHandler.__init__(handler_self, lambda: self._schedule(source_id))
+
+        observer = self._observer_cls()
+        try:
+            observer.schedule(Handler(), str(root), recursive=True)
+            observer.start()
+        except Exception as exc:
+            logging.warning("local knowledge watcher start failed for %s: %s", root, exc)
+            return {"watching": False, "source_id": source_id, "error": str(exc)}
+
+        with self._lock:
+            self._watched[source_id] = {
+                "observer": observer,
+                "timer": None,
+                "source": dict(source),
+                "last_event_at": None,
+                "last_indexed_at": None,
+                "last_error": None,
+            }
+        return {"watching": True, "source_id": source_id, "root_path": str(root)}
+
+    def stop_source(self, source_id: str) -> Dict[str, Any]:
+        with self._lock:
+            item = self._watched.pop(source_id, None)
+        if not item:
+            return {"stopped": False, "source_id": source_id}
+        timer = item.get("timer")
+        if timer:
+            timer.cancel()
+        observer = item.get("observer")
+        try:
+            observer.stop()
+            observer.join(timeout=3)
+        except Exception as exc:
+            logging.warning("local knowledge watcher stop failed for %s: %s", source_id, exc)
+        return {"stopped": True, "source_id": source_id}
+
+    def stop_all(self) -> None:
+        for source_id in list(self.status().get("active", {}).keys()):
+            self.stop_source(source_id)
+
+    def _schedule(self, source_id: str) -> None:
+        with self._lock:
+            item = self._watched.get(source_id)
+            if not item:
+                return
+            timer = item.get("timer")
+            if timer:
+                timer.cancel()
+            item["last_event_at"] = _now_seconds()
+            timer = threading.Timer(self._debounce_seconds, self._run_index, args=(source_id,))
+            timer.daemon = True
+            item["timer"] = timer
+            timer.start()
+
+    def _run_index(self, source_id: str) -> None:
+        with self._lock:
+            item = self._watched.get(source_id)
+            if not item:
+                return
+            source = dict(item["source"])
+            item["timer"] = None
+        graph = self._get_graph()
+        if graph is None:
+            return
+        consent = source.get("consent") or {}
+        try:
+            graph.index_local_folder(
+                Path(source["root_path"]),
+                include_ocr=bool(source.get("include_ocr")),
+                watch_enabled=True,
+                user_email=consent.get("approved_by"),
+                consent=consent,
+            )
+            with self._lock:
+                if source_id in self._watched:
+                    self._watched[source_id]["last_indexed_at"] = _now_seconds()
+                    self._watched[source_id]["last_error"] = None
+        except Exception as exc:
+            logging.warning("local knowledge watcher reindex failed for %s: %s", source_id, exc)
+            with self._lock:
+                if source_id in self._watched:
+                    self._watched[source_id]["last_error"] = str(exc)
+
+
+def _now_seconds() -> float:
+    import time
+
+    return time.time()
+
+
+def create_local_knowledge_router(
+    *,
+    get_graph: Callable[[], Any],
+    require_graph: Callable[[], None],
+    require_user: Callable[[Request], str],
+    require_local_user: Callable[[Request], str],
+    local_permission_response: Callable[..., dict],
+    require_local_approval: Callable[..., None],
+    watcher: Optional[LocalKnowledgeWatcher] = None,
+) -> APIRouter:
+    router = APIRouter()
+
+    def graph():
+        require_graph()
+        return get_graph()
+
+    @router.get("/knowledge-graph/local/roots")
+    async def knowledge_graph_local_roots(request: Request):
+        require_user(request)
+        return graph().discover_local_roots()
+
+    @router.get("/knowledge-graph/local/sources")
+    async def knowledge_graph_local_sources(request: Request):
+        require_user(request)
+        payload = graph().local_sources()
+        watch_status = watcher.status() if watcher else {"available": False, "active": {}}
+        active = watch_status.get("active", {})
+        for source in payload.get("sources", []):
+            source["watch_active"] = source.get("id") in active
+            source["watch_status"] = active.get(source.get("id"))
+        payload["watch"] = watch_status
+        return payload
+
+    @router.get("/knowledge-graph/local/watch/status")
+    async def knowledge_graph_local_watch_status(request: Request):
+        require_user(request)
+        graph()
+        return watcher.status() if watcher else {"available": False, "active": {}, "error": "watcher unavailable"}
+
+    @router.post("/knowledge-graph/local/watch/stop")
+    async def knowledge_graph_local_watch_stop(req: LocalKnowledgeWatchRequest, request: Request):
+        require_user(request)
+        kg = graph()
+        try:
+            kg.set_local_source_watch(req.source_id, False)
+        except ValueError as exc:
+            raise HTTPException(status_code=404, detail=str(exc)) from exc
+        result = watcher.stop_source(req.source_id) if watcher else {"stopped": False, "source_id": req.source_id}
+        return {"status": "ok", "watch": result}
+
+    @router.post("/knowledge-graph/local/tree")
+    async def knowledge_graph_local_tree(req: LocalTreeRequest, request: Request):
+        current_user = require_local_user(request)
+        kg = graph()
+        if not req.approved:
+            return local_permission_response(req.path, "list", current_user)
+        require_local_approval(token=req.approval_token, path=req.path, action="list", user_email=current_user)
+        try:
+            return kg.preview_local_tree(Path(req.path), max_items=req.max_items)
+        except ValueError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @router.post("/knowledge-graph/local/audit")
+    async def knowledge_graph_local_audit(req: LocalKnowledgeAuditRequest, request: Request):
+        current_user = require_local_user(request)
+        kg = graph()
+        if not req.approved:
+            return local_permission_response(req.path, "list", current_user)
+        require_local_approval(token=req.approval_token, path=req.path, action="list", user_email=current_user)
+        try:
+            return kg.audit_local_folder(Path(req.path), include_ocr=req.include_ocr, max_files=req.max_files)
+        except ValueError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @router.post("/knowledge-graph/local/index")
+    async def knowledge_graph_local_index(req: LocalKnowledgeIndexRequest, request: Request):
+        current_user = require_local_user(request)
+        kg = graph()
+        if not req.approved:
+            return local_permission_response(req.path, "read", current_user)
+        require_local_approval(token=req.approval_token, path=req.path, action="read", user_email=current_user)
+        try:
+            result = kg.index_local_folder(
+                Path(req.path),
+                include_ocr=req.include_ocr,
+                watch_enabled=req.watch_enabled,
+                user_email=current_user,
+                consent=req.consent or {},
+                max_files=req.max_files,
+            )
+        except ValueError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+        if watcher:
+            if req.watch_enabled:
+                source_payload = {
+                    **result.get("source", {}),
+                    "consent": {"approved_by": current_user, **(req.consent or {})},
+                }
+                result["watch"] = watcher.start_source(source_payload)
+            else:
+                result["watch"] = watcher.stop_source(result.get("source", {}).get("id", ""))
+        return result
+
+    return router