ltcai 8.8.0 → 9.0.0

package/lattice_brain/graph/_kg_common.py

@@ -34,502 +34,43 @@ except Exception:  # pragma: no cover - v2 schema is optional at import time
     _exec_script = None  # type: ignore[assignment]
 
 from ..embeddings import LocalEmbeddingModel
+from .json_utils import _json, _safe_loads
+from .runtime import get_llm_router, set_llm_router
 
 # Default read source for the graph queries: v2 reconstruction views.
 # Override with LATTICEAI_KG_READ_V2=0 to fall back to the legacy tables.
 _READ_FROM_V2_DEFAULT = os.getenv("LATTICEAI_KG_READ_V2", "1") != "0"
 
-# Bump when the v2 projection layout changes (columns, normalization rules).
-# On init, a stale projection is dropped and rebuilt from the authoritative
-# legacy tables — safe because nodes_v2/edges_v2 only ever hold a derived view.
-# v4: summary nullable + verbatim (byte-faithful) projection of legacy values.
-_PROJECTION_VERSION = 4
-_KG_DB_FORMAT_VERSION = 4
-_KG_DB_FORMAT_KEY = "db_format_version"
-_V2_WRITE_MASTER_KEY = "v2_write_mastered_at"
-
-_llm_router_ref = None
-
-
-def set_llm_router(router_instance):
-    global _llm_router_ref
-    _llm_router_ref = router_instance
-
-
-GRAPH_SCHEMA_VERSION = 1
-
-LOCAL_TEXT_EXTENSIONS = {".txt", ".md"}
-LOCAL_CODE_EXTENSIONS = {
-    ".py",
-    ".js",
-    ".ts",
-    ".tsx",
-    ".jsx",
-    ".html",
-    ".css",
-    ".json",
-    ".yaml",
-    ".yml",
-    ".xml",
-    ".sql",
-    ".sh",
-    ".zsh",
-    ".toml",
-    ".ini",
-}
-LOCAL_DOCUMENT_EXTENSIONS = {".pdf", ".docx"}
-LOCAL_SPREADSHEET_EXTENSIONS = {".xlsx", ".csv"}
-LOCAL_SLIDE_EXTENSIONS = {".pptx"}
-LOCAL_IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp"}
-LOCAL_SUPPORTED_EXTENSIONS = (
-    LOCAL_TEXT_EXTENSIONS
-    | LOCAL_CODE_EXTENSIONS
-    | LOCAL_DOCUMENT_EXTENSIONS
-    | LOCAL_SPREADSHEET_EXTENSIONS
-    | LOCAL_SLIDE_EXTENSIONS
-    | LOCAL_IMAGE_EXTENSIONS
-)
-
-LOCAL_SIZE_LIMITS = {
-    "text": 4_000_000,
-    "code": 4_000_000,
-    "pdf": 50_000_000,
-    "document": 50_000_000,
-    "spreadsheet": 50_000_000,
-    "slide_deck": 50_000_000,
-    "image": 100_000_000,
-}
-
-COMMON_EXCLUDED_DIRS = {
-    ".git",
-    "node_modules",
-    ".venv",
-    "venv",
-    "env",
-    "__pycache__",
-    ".pytest_cache",
-    ".mypy_cache",
-    ".ruff_cache",
-    ".next",
-    ".nuxt",
-    ".turbo",
-    "dist",
-    "build",
-    "target",
-    "out",
-    "coverage",
-    ".cache",
-    ".config",
-    ".ssh",
-    ".gnupg",
-    ".docker",
-    ".kube",
-    ".aws",
-    ".azure",
-    ".npm",
-    ".pnpm-store",
-    ".yarn",
-    ".bun",
-    ".cargo",
-    ".rustup",
-    ".pyenv",
-    ".conda",
-    ".local",
-    ".claude",
-    ".codex",
-    ".cursor",
-    ".copilot",
-    ".antigravity",
-    ".antigravity-ide",
-}
-
-COMMON_EXCLUDED_FILE_NAMES = {
-    ".env",
-    ".env.local",
-    ".env.production",
-    ".env.development",
-    "id_rsa",
-    "id_ed25519",
-    "authorized_keys",
-    "known_hosts",
-    "credentials.json",
-    "service-account.json",
-    "token.json",
-    "secrets.json",
-    "cookies",
-    "login data",
-    "history",
-    "web data",
-    ".ds_store",
-    "thumbs.db",
-}
-COMMON_EXCLUDED_FILE_SUFFIXES = {
-    ".pem",
-    ".key",
-    ".p12",
-    ".pfx",
-    ".kdbx",
-    ".wallet",
-    ".sqlite",
-    ".db",
-    ".exe",
-    ".dll",
-    ".sys",
-    ".msi",
-    ".dmg",
-    ".pkg",
-    ".app",
-    ".zip",
-    ".tar",
-    ".gz",
-    ".7z",
-    ".rar",
-    ".mp4",
-    ".mov",
-    ".mp3",
-    ".wav",
-    ".tmp",
-    ".bak",
-    ".lock",
-}
-SENSITIVE_PATH_KEYWORDS = {
-    "secret",
-    "secrets",
-    "token",
-    "password",
-    "passwd",
-    "credential",
-    "credentials",
-    "private",
-    "key",
-    "wallet",
-    "recovery",
-    "seed",
-    "mnemonic",
-    "cookie",
-    "session",
-    "auth",
-    "oauth",
-    "certificate",
-    "cert",
-    "api_key",
-    "apikey",
-}
-
-MACOS_EXCLUDED_PREFIXES = (
-    "/System",
-    "/Library",
-    "/Applications",
-    "/private",
-    "/tmp",
-    "/var",
-)
-WINDOWS_EXCLUDED_NAMES = {
-    "windows",
-    "program files",
-    "program files (x86)",
-    "programdata",
-    "appdata",
-    "$recycle.bin",
-    "system volume information",
-    "recovery",
-    "perflogs",
-    "intel",
-    "amd",
-    "nvidia",
-}
-LINUX_EXCLUDED_PREFIXES = (
-    "/bin",
-    "/boot",
-    "/dev",
-    "/etc",
-    "/lib",
-    "/lib64",
-    "/proc",
-    "/root",
-    "/run",
-    "/sbin",
-    "/sys",
-    "/tmp",
-    "/usr",
-    "/var",
-    "/snap",
-    "/lost+found",
+# Static constants (projection/format versions, local-ingestion classification
+# tables, OS exclusion lists) live in ._kg_constants; re-exported here so every
+# existing ``from ._kg_common import <CONST>`` site is unaffected.
+from ._kg_constants import (  # noqa: E402
+    _KG_DB_FORMAT_KEY,
+    _KG_DB_FORMAT_VERSION,
+    _PROJECTION_VERSION,
+    _V2_WRITE_MASTER_KEY,
+    COMMON_EXCLUDED_DIRS,
+    COMMON_EXCLUDED_FILE_NAMES,
+    COMMON_EXCLUDED_FILE_SUFFIXES,
+    GRAPH_SCHEMA_VERSION,
+    LINUX_EXCLUDED_PREFIXES,
+    LOCAL_CODE_EXTENSIONS,
+    LOCAL_DOCUMENT_EXTENSIONS,
+    LOCAL_IMAGE_EXTENSIONS,
+    LOCAL_SIZE_LIMITS,
+    LOCAL_SLIDE_EXTENSIONS,
+    LOCAL_SPREADSHEET_EXTENSIONS,
+    LOCAL_SUPPORTED_EXTENSIONS,
+    LOCAL_TEXT_EXTENSIONS,
+    MACOS_EXCLUDED_PREFIXES,
+    SENSITIVE_PATH_KEYWORDS,
+    WINDOWS_EXCLUDED_NAMES,
 )
 
 
-def _now() -> str:
-    return datetime.now().isoformat()
-
-
-def _parse_iso(raw: Optional[str]) -> Optional[datetime]:
-    if not raw:
-        return None
-    try:
-        return datetime.fromisoformat(str(raw))
-    except (TypeError, ValueError):
-        return None
-
-
-def _recency_score(
-    updated_at: Optional[str],
-    *,
-    now: Optional[datetime] = None,
-    half_life_days: float = 14.0,
-) -> float:
-    stamp = _parse_iso(updated_at)
-    if not stamp:
-        return 0.0
-    now = now or datetime.now()
-    age_days = max(0.0, (now - stamp).total_seconds() / 86400.0)
-    decay = math.log(2) / max(0.1, half_life_days)
-    return math.exp(-decay * age_days)
-
-
-def _json(data: Optional[Dict[str, Any]]) -> str:
-    return json.dumps(data or {}, ensure_ascii=False, sort_keys=True)
-
-
-def _safe_loads(raw: Optional[str]) -> Dict[str, Any]:
-    """Tolerantly parse a metadata_json column — returns {} on corrupt rows."""
-    if not raw:
-        return {}
-    try:
-        value = json.loads(raw)
-        return value if isinstance(value, dict) else {}
-    except (json.JSONDecodeError, TypeError) as e:
-        logging.warning(
-            "knowledge_graph: corrupt metadata_json (%s) — using empty dict", e
-        )
-        return {}
-
-
-def _slug(text: str, max_len: int = 96) -> str:
-    value = re.sub(r"\s+", " ", str(text or "")).strip().lower()
-    value = re.sub(r"[^0-9a-zA-Z가-힣._:@/-]+", "-", value).strip("-")
-    return (value or "untitled")[:max_len]
-
-
-def _sha256_bytes(data: bytes) -> str:
-    return hashlib.sha256(data).hexdigest()
-
-
-def _sha256_text(text: str) -> str:
-    return hashlib.sha256(text.encode("utf-8", errors="replace")).hexdigest()
-
-
-def _safe_iso_from_stat_mtime(mtime: float) -> str:
-    try:
-        return datetime.fromtimestamp(float(mtime)).isoformat()
-    except (TypeError, ValueError, OSError):
-        return ""
-
-
-def _path_fingerprint(path: Path) -> str:
-    return _sha256_text(str(path.expanduser().resolve()))[:24]
-
-
-def _is_relative_to(path: Path, base: Path) -> bool:
-    try:
-        path.relative_to(base)
-        return True
-    except ValueError:
-        return False
-
-
-def _path_parts_lower(path: Path) -> List[str]:
-    return [
-        part.lower()
-        for part in path.parts
-        if part and part not in {os.sep, path.anchor}
-    ]
-
-
-def _current_os_type() -> str:
-    system = platform.system().lower()
-    if system.startswith("darwin"):
-        return "macos"
-    if system.startswith("windows"):
-        return "windows"
-    if system.startswith("linux"):
-        return "linux"
-    return system or "unknown"
-
-
-def _drive_id_for_path(path: Path) -> str:
-    resolved = path.expanduser().resolve()
-    if resolved.drive:
-        return resolved.drive.upper()
-    parts = resolved.parts
-    if len(parts) >= 3 and parts[1] == "Volumes":
-        return f"/Volumes/{parts[2]}"
-    if len(parts) >= 3 and parts[1] == "media":
-        return f"/media/{parts[2]}"
-    if len(parts) >= 3 and parts[1] == "mnt":
-        return f"/mnt/{parts[2]}"
-    return resolved.anchor or "/"
-
-
-def _file_category(ext: str) -> str:
-    ext = (ext or "").lower()
-    if ext in LOCAL_CODE_EXTENSIONS:
-        return "code"
-    if ext in LOCAL_TEXT_EXTENSIONS:
-        return "text"
-    if ext == ".pdf":
-        return "pdf"
-    if ext in LOCAL_DOCUMENT_EXTENSIONS:
-        return "document"
-    if ext in LOCAL_SPREADSHEET_EXTENSIONS:
-        return "spreadsheet"
-    if ext in LOCAL_SLIDE_EXTENSIONS:
-        return "slide_deck"
-    if ext in LOCAL_IMAGE_EXTENSIONS:
-        return "image"
-    return "unsupported"
-
-
-def _node_type_for_category(category: str) -> str:
-    return {
-        "code": "CodeFile",
-        "spreadsheet": "Spreadsheet",
-        "slide_deck": "SlideDeck",
-        "image": "Image",
-        "unsupported": "File",
-    }.get(category, "Document")
-
-
-def _parser_type_for_category(category: str, ext: str) -> str:
-    if category in {"text", "code"}:
-        return "plain_text"
-    if category == "spreadsheet" and ext == ".csv":
-        return "csv_text"
-    if category == "image":
-        return "image_ocr"
-    return ext.lstrip(".") or category
-
-
-def _size_limit_for_category(category: str) -> int:
-    return LOCAL_SIZE_LIMITS.get(category, LOCAL_SIZE_LIMITS["document"])
-
-
-def _is_hidden_path(path: Path, root: Optional[Path] = None) -> bool:
-    parts: Iterable[str]
-    if root is not None:
-        try:
-            parts = path.relative_to(root).parts
-        except ValueError:
-            parts = path.parts
-    else:
-        parts = path.parts
-    return any(part.startswith(".") and part not in {".", ".."} for part in parts)
-
-
-def _excluded_directory_reason(
-    path: Path, *, root: Optional[Path] = None, os_type: Optional[str] = None
-) -> Optional[str]:
-    os_type = os_type or _current_os_type()
-    name = path.name.lower()
-    if name in COMMON_EXCLUDED_DIRS:
-        return "excluded_folder"
-    if _is_hidden_path(path, root):
-        return "hidden_folder"
-    parts = _path_parts_lower(path)
-    if os_type == "windows" and any(part in WINDOWS_EXCLUDED_NAMES for part in parts):
-        return "system_folder"
-    normalized = path.as_posix()
-    root_normalized = root.as_posix() if root else ""
-
-    def _prefix_blocks(prefixes: Tuple[str, ...]) -> bool:
-        for prefix in prefixes:
-            path_under_prefix = normalized == prefix or normalized.startswith(
-                f"{prefix}/"
-            )
-            root_under_prefix = bool(root_normalized) and (
-                root_normalized == prefix or root_normalized.startswith(f"{prefix}/")
-            )
-            if path_under_prefix and not root_under_prefix:
-                return True
-        return False
-
-    if os_type == "macos":
-        home_library = Path.home() / "Library"
-        try:
-            root_is_library = bool(root) and _is_relative_to(
-                root.expanduser().resolve(), home_library.expanduser().resolve()
-            )
-            if (
-                _is_relative_to(
-                    path.expanduser().resolve(), home_library.expanduser().resolve()
-                )
-                and not root_is_library
-            ):
-                return "user_library"
-        except OSError:
-            pass
-        if _prefix_blocks(MACOS_EXCLUDED_PREFIXES):
-            return "system_folder"
-    if os_type == "linux":
-        if _prefix_blocks(LINUX_EXCLUDED_PREFIXES):
-            return "system_folder"
-    return None
-
-
-def _sensitive_file_reason(path: Path, *, root: Optional[Path] = None) -> Optional[str]:
-    name = path.name.lower()
-    suffix = path.suffix.lower()
-    if name in COMMON_EXCLUDED_FILE_NAMES or suffix in COMMON_EXCLUDED_FILE_SUFFIXES:
-        return "sensitive_or_excluded_file"
-    try:
-        rel_text = (
-            path.relative_to(root).as_posix().lower()
-            if root
-            else path.as_posix().lower()
-        )
-    except ValueError:
-        rel_text = path.as_posix().lower()
-    tokens = re.split(r"[^0-9a-zA-Z_가-힣]+", rel_text)
-    if any(token in SENSITIVE_PATH_KEYWORDS for token in tokens):
-        return "sensitive_name"
-    return None
-
-
-def _root_warning(path: Path, os_type: str) -> Optional[str]:
-    resolved = path.expanduser().resolve()
-    home = Path.home().expanduser().resolve()
-    if os_type == "macos" and resolved == home:
-        return "홈 전체에는 설정/숨김 폴더가 포함될 수 있습니다. 문서, 데스크탑, 다운로드, 프로젝트 폴더부터 추가하는 것을 권장합니다."
-    if os_type == "linux" and resolved.as_posix() == "/":
-        return "루트 디렉터리에는 시스템 파일이 포함되어 있습니다. 일반 사용자 폴더나 마운트된 데이터 폴더를 권장합니다."
-    if os_type == "windows" and str(resolved).rstrip("\\/").upper() in {"C:", "C:\\"}:
-        return "C드라이브에는 Windows 시스템 파일과 앱 설정 파일이 포함되어 있습니다. 하위 폴더를 선택하는 것을 권장합니다."
-    return None
-
-
-def _sample_file(
-    path: Path, root: Path, status: str, reason: str = ""
-) -> Dict[str, Any]:
-    try:
-        rel = path.relative_to(root).as_posix()
-    except ValueError:
-        rel = path.name
-    try:
-        stat = path.stat()
-        size = stat.st_size if path.is_file() else None
-        modified_at = _safe_iso_from_stat_mtime(stat.st_mtime)
-    except OSError:
-        size = None
-        modified_at = ""
-    return {
-        "path": str(path),
-        "relative_path": rel,
-        "name": path.name,
-        "extension": path.suffix.lower(),
-        "status": status,
-        "reason": reason,
-        "size_bytes": size,
-        "modified_at": modified_at,
-    }
+# Pure fs/path/hash/classification helpers → ._kg_fsutil (re-exported so the
+# computed __all__ below still forwards them to the graph mixins).
+from ._kg_fsutil import *  # noqa: E402,F401,F403
 
 
 def _clean_text(text: str) -> str:
@@ -586,9 +127,10 @@ ENABLE_LLM_EXTRACTION = os.getenv("LATTICEAI_LLM_EXTRACTION", "true").lower() in
 
 
 def _llm_extract_concepts(text: str, limit: int = 12) -> Optional[List[str]]:
-    if not ENABLE_LLM_EXTRACTION or not _llm_router_ref:
+    router = get_llm_router()
+    if not ENABLE_LLM_EXTRACTION or not router:
         return None
-    if not _llm_router_ref.current_model_id:
+    if not router.current_model_id:
         return None
     prompt = _LLM_EXTRACT_CONCEPT_PROMPT.format(text=text[:3000], limit=limit)
     try:
@@ -599,12 +141,12 @@ def _llm_extract_concepts(text: str, limit: int = 12) -> Optional[List[str]]:
             with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
                 future = pool.submit(
                     asyncio.run,
-                    _llm_router_ref.generate(prompt, max_tokens=1024, temperature=0.1),
+                    router.generate(prompt, max_tokens=1024, temperature=0.1),
                 )
                 raw = future.result(timeout=30)
         else:
             raw = asyncio.run(
-                _llm_router_ref.generate(prompt, max_tokens=1024, temperature=0.1)
+                router.generate(prompt, max_tokens=1024, temperature=0.1)
             )
         raw = raw.strip()
         if raw.startswith("```"):
@@ -627,9 +169,10 @@ def _llm_extract_concepts(text: str, limit: int = 12) -> Optional[List[str]]:
 def _llm_extract_triples(
     text: str, concepts: List[str], limit: int = 20
 ) -> Optional[List[Dict[str, str]]]:
-    if not ENABLE_LLM_EXTRACTION or not _llm_router_ref:
+    router = get_llm_router()
+    if not ENABLE_LLM_EXTRACTION or not router:
         return None
-    if not _llm_router_ref.current_model_id:
+    if not router.current_model_id:
         return None
     prompt = _LLM_EXTRACT_TRIPLE_PROMPT.format(
         text=text[:3000],
@@ -644,12 +187,12 @@ def _llm_extract_triples(
             with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
                 future = pool.submit(
                     asyncio.run,
-                    _llm_router_ref.generate(prompt, max_tokens=2048, temperature=0.1),
+                    router.generate(prompt, max_tokens=2048, temperature=0.1),
                 )
                 raw = future.result(timeout=30)
         else:
             raw = asyncio.run(
-                _llm_router_ref.generate(prompt, max_tokens=2048, temperature=0.1)
+                router.generate(prompt, max_tokens=2048, temperature=0.1)
             )
         raw = raw.strip()
         if raw.startswith("```"):