ltcai 11.5.2 → 11.7.0

package/latticeai/core/file_generation/validation.py

@@ -1,201 +0,0 @@
-"""Per-extension structural checks — step 3 of the pipeline.
-
-Answers one question about an already-extracted payload: *is this a file of the
-requested type?* HTML document shape, JSON parses, CSS has rule blocks, Python
-parses, braced code is balanced, components close their blocks — and, for prose
-types with no grammar to check, that the reply is not merely talk about the
-file.
-"""
-
-from __future__ import annotations
-
-import ast
-import json
-import re
-from typing import List, Tuple
-
-from .extraction import _ext
-
-_REFUSAL_RE = re.compile(
-    r"(i can('|no)?t|i'?m (sorry|unable)|as an ai|cannot assist"
-    r"|죄송(하지만|합니다)|할 수 없|불가능합니다|도와드릴 수 없)",
-    re.IGNORECASE,
-)
-
-
-def looks_like_refusal(content: str) -> bool:
-    head = content[:300]
-    return bool(_REFUSAL_RE.search(head)) and len(content) < 600
-
-
-# Braced code types validated structurally (balanced delimiters, no fences).
-_BRACED_CODE_EXTENSIONS = frozenset({".js", ".jsx", ".ts", ".tsx"})
-# Single-file components validated by their block tags being closed.
-_COMPONENT_EXTENSIONS = frozenset({".vue", ".svelte"})
-
-
-def _strip_code_literals(text: str) -> str:
-    """Remove string literals and comments so delimiter counting stays honest.
-
-    A cheap single-pass scanner (not a parser): quotes ('', "", ``), line
-    comments (//) and block comments (/* */) commonly contain lone braces
-    that would otherwise false-flag valid JS/TS as unbalanced.
-    """
-    out: List[str] = []
-    i, n = 0, len(text)
-    while i < n:
-        ch = text[i]
-        nxt = text[i + 1] if i + 1 < n else ""
-        if ch == "\\":
-            i += 2  # escaped char (inside or outside a literal — always skip)
-            continue
-        if ch in ("'", '"', "`"):
-            quote = ch
-            i += 1
-            while i < n:
-                if text[i] == "\\":
-                    i += 2
-                    continue
-                if text[i] == quote:
-                    i += 1
-                    break
-                i += 1
-            continue
-        if ch == "/" and nxt == "/":
-            while i < n and text[i] != "\n":
-                i += 1
-            continue
-        if ch == "/" and nxt == "*":
-            end = text.find("*/", i + 2)
-            i = n if end < 0 else end + 2
-            continue
-        out.append(ch)
-        i += 1
-    return "".join(out)
-
-
-def _check_balanced_delimiters(content: str) -> Tuple[bool, str]:
-    """Lenient count-based balance check for braced code (js/ts family).
-
-    Only *counts* are compared, never ordering, so valid-but-unusual code is
-    not rejected; a truncated file with a dangling ``{`` still fails.
-    """
-    stripped = _strip_code_literals(content)
-    for opener, closer, label in (("{", "}", "braces"), ("(", ")", "parentheses"), ("[", "]", "brackets")):
-        if stripped.count(opener) != stripped.count(closer):
-            return False, f"unbalanced {label} ({opener}{closer}) — the file looks truncated"
-    return True, "ok"
-
-
-def _check_component_blocks(content: str) -> Tuple[bool, str]:
-    """Vue/Svelte SFC sanity: every opened block tag must be closed."""
-    lower = content.lower()
-    for tag in ("template", "script", "style"):
-        opened = len(re.findall(rf"<{tag}(?:\s[^>]*)?>", lower))
-        closed = lower.count(f"</{tag}>")
-        if opened != closed:
-            return False, f"<{tag}> block is not closed — the component looks truncated"
-    return True, "ok"
-
-
-def validate_file_content(content: str, target_path: str) -> Tuple[bool, str]:
-    """Structural sanity check per file type. Returns (ok, reason)."""
-    if not content.strip():
-        return False, "empty output"
-    if looks_like_refusal(content):
-        return False, "the reply was a refusal/chat message, not file content"
-
-    ext = _ext(target_path)
-    if ext in (".html", ".htm"):
-        lower = content.lower()
-        if "<html" not in lower and "<!doctype" not in lower:
-            return False, "not a complete HTML document (missing <!DOCTYPE html>/<html>)"
-        if "</html>" not in lower:
-            return False, "HTML document is truncated (missing </html>)"
-        # A document that merely *contains* html somewhere is not a valid
-        # file payload: fenced/chat-wrapped replies must fail here so the
-        # extraction pass gets a chance to slice out the real document.
-        stripped = content.lstrip()
-        if not (stripped.lower().startswith("<!doctype") or stripped.lower().startswith("<html")):
-            return False, "HTML document is wrapped in prose or fences"
-        if "```" in content:
-            return False, "output still contains Markdown fences"
-        return True, "ok"
-    if ext == ".json":
-        try:
-            json.loads(content)
-        except (ValueError, TypeError) as exc:
-            return False, f"invalid JSON: {exc}"
-        return True, "ok"
-    if ext == ".css":
-        if "```" in content:
-            return False, "output still contains Markdown fences"
-        if "{" not in content or "}" not in content:
-            return False, "no CSS rule blocks found"
-        return True, "ok"
-    if ext == ".py":
-        if "```" in content:
-            return False, "output still contains Markdown fences"
-        try:
-            ast.parse(content)
-        except SyntaxError as exc:
-            return False, f"invalid Python syntax: {exc.msg} (line {exc.lineno})"
-        return True, "ok"
-    if ext in _BRACED_CODE_EXTENSIONS:
-        if "```" in content:
-            return False, "output still contains Markdown fences"
-        return _check_balanced_delimiters(content)
-    if ext in _COMPONENT_EXTENSIONS:
-        if "```" in content:
-            return False, "output still contains Markdown fences"
-        return _check_component_blocks(content)
-    if ext in (".sh", ".sql"):
-        if "```" in content:
-            return False, "output still contains Markdown fences"
-        return True, "ok"
-    # Prose types (.md, .txt, .csv, …) have no grammar to check, which used to
-    # mean *nothing* was checked: a 1–4B model that answered "Sure! Here is the
-    # document you asked for:" and stopped had its sentence saved as the file,
-    # because the fence stripper only removes conversational lines it can
-    # recognise and the length guard on `looks_like_refusal` lets a wordy
-    # refusal through. The two checks below are the only ones that generalise
-    # without inventing a grammar: it must not still be wearing fences, and it
-    # must not be *only* an answer about the file.
-    if "```" in content:
-        return False, "output still contains Markdown fences"
-    if _looks_like_commentary(content):
-        return False, "the reply talks about the file instead of being the file"
-    return True, "ok"
-
-
-# Openers that mean "I am about to give you the thing" — if the whole reply is
-# one of these, the thing never arrived.
-_COMMENTARY_RE = re.compile(
-    r"^\s*("
-    r"(sure|of course|certainly|okay|ok|alright|here|below|the following)\b"
-    r"|i('ve| have| will|'ll)\b"
-    r"|(물론|네[,!. ]|알겠|다음은|아래(는|의)?|요청하신|원하시는)"
-    r")",
-    re.IGNORECASE,
-)
-
-
-def _looks_like_commentary(content: str) -> bool:
-    """True when the reply reads as an answer *about* a file, not the file.
-
-    Deliberately conservative — a long document that merely opens with "The
-    following" is a document. Only a short reply that both opens
-    conversationally and never grows into content is rejected, so a real file
-    is never thrown away to catch a chat line.
-    """
-    stripped = content.strip()
-    if len(stripped) > 400:
-        return False
-    if not _COMMENTARY_RE.match(stripped):
-        return False
-    # Structure means content arrived after the preamble: a heading, a list, a
-    # table row, a delimiter, or simply several lines of body text.
-    body = stripped.split("\n", 1)[1].strip() if "\n" in stripped else ""
-    if re.search(r"^\s*(#{1,6}\s|[-*+]\s|\d+[.)]\s|\||>)", body, re.MULTILINE):
-        return False
-    return len(body) < 120

package/latticeai/core/invitations.py

@@ -1,132 +0,0 @@
-"""Local-first invitation tokens for workspace membership."""
-
-from __future__ import annotations
-
-import json
-import secrets
-from datetime import datetime, timedelta
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-
-from latticeai.core.quiet import quiet
-from latticeai.core.security import sha256_hex
-
-from .timeutil import local_now as _now
-
-
-def _iso(dt: datetime) -> str:
-    return dt.isoformat(timespec="seconds")
-
-
-def _hash_token(token: str) -> str:
-    """Strict by design: an invite code is always a string by the time it lands here."""
-    return sha256_hex(token)
-
-
-def _atomic_write(path: Path, data: Dict[str, Any]) -> None:
-    path.parent.mkdir(parents=True, exist_ok=True)
-    tmp = path.with_suffix(path.suffix + ".tmp")
-    tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
-    tmp.replace(path)
-
-
-class InvitationStore:
-    def __init__(self, path: Path | str):
-        self.path = Path(path)
-
-    def _load(self) -> Dict[str, Any]:
-        if not self.path.exists():
-            return {"version": 1, "invitations": []}
-        try:
-            data = json.loads(self.path.read_text(encoding="utf-8"))
-            if isinstance(data, dict):
-                data.setdefault("invitations", [])
-                return data
-        except Exception:
-            quiet()
-        return {"version": 1, "invitations": []}
-
-    def _save(self, data: Dict[str, Any]) -> None:
-        data["version"] = 1
-        _atomic_write(self.path, data)
-
-    def create(
-        self,
-        *,
-        email: Optional[str],
-        workspace_id: Optional[str],
-        role: str,
-        created_by: Optional[str],
-        expires_hours: int = 168,
-    ) -> Dict[str, Any]:
-        token = secrets.token_urlsafe(32)
-        now = _now()
-        record = {
-            "id": f"invite-{secrets.token_hex(8)}",
-            "token_hash": _hash_token(token),
-            "email": (email or "").strip().lower() or None,
-            "workspace_id": workspace_id,
-            "role": role,
-            "created_by": created_by,
-            "created_at": _iso(now),
-            "expires_at": _iso(now + timedelta(hours=max(1, min(int(expires_hours or 168), 24 * 30)))),
-            "status": "pending",
-            "accepted_by": None,
-            "accepted_at": None,
-        }
-        data = self._load()
-        data.setdefault("invitations", []).append(record)
-        self._save(data)
-        public = self.public(record)
-        public["token"] = token
-        return public
-
-    def list(self) -> List[Dict[str, Any]]:
-        data = self._load()
-        changed = False
-        records = []
-        for record in data.get("invitations") or []:
-            if self._expire_if_needed(record):
-                changed = True
-            records.append(self.public(record))
-        if changed:
-            self._save(data)
-        return records
-
-    def accept(self, token: str, *, accepted_by: str, email: Optional[str]) -> Dict[str, Any]:
-        data = self._load()
-        token_hash = _hash_token(token)
-        record = next((item for item in data.get("invitations") or [] if item.get("token_hash") == token_hash), None)
-        if record is None:
-            raise FileNotFoundError("invitation not found")
-        if self._expire_if_needed(record):
-            self._save(data)
-            raise PermissionError("invitation expired")
-        if record.get("status") != "pending":
-            raise PermissionError(f"invitation is {record.get('status')}")
-        invited_email = (record.get("email") or "").lower()
-        if invited_email and (not email or invited_email != email.lower()):
-            raise PermissionError("invitation was issued for a different email")
-        record["status"] = "accepted"
-        record["accepted_by"] = accepted_by
-        record["accepted_at"] = _iso(_now())
-        self._save(data)
-        return self.public(record)
-
-    @staticmethod
-    def _expire_if_needed(record: Dict[str, Any]) -> bool:
-        if record.get("status") != "pending":
-            return False
-        try:
-            expires_at = datetime.fromisoformat(str(record.get("expires_at")))
-        except Exception:
-            expires_at = _now() - timedelta(seconds=1)
-        if expires_at >= _now():
-            return False
-        record["status"] = "expired"
-        record["expired_at"] = _iso(_now())
-        return True
-
-    @staticmethod
-    def public(record: Dict[str, Any]) -> Dict[str, Any]:
-        return {k: v for k, v in record.items() if k != "token_hash"}

package/latticeai/core/legacy_compatibility.py

@@ -1,243 +0,0 @@
-"""Managed legacy compatibility surface for legacy import shims.
-
-Compatibility modules are intentionally still present for old scripts, VS Code
-extension paths, historical integrations, and pre-graph-package imports. 8.4.0
-stops treating them as vague technical debt: every tracked shim has an owner,
-migration target, removal phase, and replacement import that can be surfaced in
-docs and tests.
-"""
-
-from __future__ import annotations
-
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any, Dict, List
-
-LEGACY_COMPATIBILITY_VERSION = "11.5.2"
-
-
-@dataclass(frozen=True)
-class LegacyShim:
-    path: str
-    owner: str
-    replacement: str
-    reason: str
-    removal_phase: str
-    layer: str = "root"
-    status: str = "managed"
-
-    def as_dict(self) -> Dict[str, str]:
-        return {
-            "path": self.path,
-            "owner": self.owner,
-            "replacement": self.replacement,
-            "reason": self.reason,
-            "removal_phase": self.removal_phase,
-            "layer": self.layer,
-            "status": self.status,
-        }
-
-
-LEGACY_SHIMS: List[LegacyShim] = [
-    LegacyShim(
-        path="server.py",
-        owner="latticeai.server_app",
-        replacement="from latticeai.server_app import app, main",
-        reason="Deployment docs, Docker CMD, start scripts, and uvicorn targets still use server:app.",
-        removal_phase="major-release-after-9.x",
-    ),
-]
-
-# Shim layers that have completed their compatibility window and were
-# physically deleted. Kept in the report so consumers (docs, tests, release
-# notes) can tell "removed on purpose" apart from "missing by accident".
-# Removing the internal-only layers is the first step of extracting
-# ``lattice_brain`` as a standalone Brain Core package: the package now has
-# exactly one import surface (``lattice_brain.*`` physical paths).
-REMOVED_SHIMS: List[LegacyShim] = [
-    LegacyShim(
-        path="knowledge_graph.py",
-        owner="lattice_brain.graph.store",
-        replacement="from lattice_brain.graph.store import KnowledgeGraphStore",
-        reason="Root compatibility shims completed their window; removed in 9.9.1.",
-        removal_phase="removed-9.9.1",
-        layer="root",
-        status="removed",
-    ),
-    LegacyShim(
-        path="knowledge_graph_api.py",
-        owner="latticeai.api.knowledge_graph",
-        replacement="from latticeai.api.knowledge_graph import create_knowledge_graph_router",
-        reason="Root compatibility shims completed their window; removed in 9.9.1.",
-        removal_phase="removed-9.9.1",
-        layer="root",
-        status="removed",
-    ),
-    LegacyShim(
-        path="kg_schema.py",
-        owner="lattice_brain.graph.schema",
-        replacement="from lattice_brain.graph.schema import ...",
-        reason="Root compatibility shims completed their window; removed in 9.9.1.",
-        removal_phase="removed-9.9.1",
-        layer="root",
-        status="removed",
-    ),
-    LegacyShim(
-        path="ltcai_cli.py",
-        owner="latticeai.cli.entrypoint",
-        replacement="from latticeai.cli.entrypoint import main",
-        reason="Root compatibility shims completed their window; removed in 9.9.1.",
-        removal_phase="removed-9.9.1",
-        layer="root",
-        status="removed",
-    ),
-    LegacyShim(
-        path="telegram_bot.py",
-        owner="latticeai.integrations.telegram_bot",
-        replacement="from latticeai.integrations.telegram_bot import run_bot",
-        reason="Root compatibility shims completed their window; removed in 9.9.1.",
-        removal_phase="removed-9.9.1",
-        layer="root",
-        status="removed",
-    ),
-    LegacyShim(
-        path="p_reinforce.py",
-        owner="latticeai.services.p_reinforce",
-        replacement="from latticeai.services.p_reinforce import PReinforceGardener",
-        reason="Root compatibility shims completed their window; removed in 9.9.1.",
-        removal_phase="removed-9.9.1",
-        layer="root",
-        status="removed",
-    ),
-    LegacyShim(
-        path="mcp_registry.py",
-        owner="latticeai.core.mcp_registry",
-        replacement="import latticeai.core.mcp_registry",
-        reason="Root compatibility shims completed their window; removed in 9.9.1.",
-        removal_phase="removed-9.9.1",
-        layer="root",
-        status="removed",
-    ),
-    LegacyShim(
-        path="llm_router.py",
-        owner="latticeai.models.router",
-        replacement="import latticeai.models.router",
-        reason="Root compatibility shims completed their window; removed in 9.9.1.",
-        removal_phase="removed-9.9.1",
-        layer="root",
-        status="removed",
-    ),
-    LegacyShim(
-        path="local_knowledge_api.py",
-        owner="latticeai.services.local_knowledge",
-        replacement="from latticeai.services.local_knowledge import create_local_knowledge_router",
-        reason="Root compatibility shims completed their window; removed in 9.9.1.",
-        removal_phase="removed-9.9.1",
-        layer="root",
-        status="removed",
-    ),
-    LegacyShim(
-        path="auto_setup.py",
-        owner="latticeai.setup.auto_setup",
-        replacement="from latticeai.setup.auto_setup import probe, recommend",
-        reason="Root compatibility shims completed their window; removed in 9.9.1.",
-        removal_phase="removed-9.9.1",
-        layer="root",
-        status="removed",
-    ),
-    LegacyShim(
-        path="setup_wizard.py",
-        owner="latticeai.setup.wizard",
-        replacement="from latticeai.setup.wizard import scan_environment",
-        reason="Root compatibility shims completed their window; removed in 9.9.1.",
-        removal_phase="removed-9.9.1",
-        layer="root",
-        status="removed",
-    ),
-    LegacyShim(
-        path="tools/",
-        owner="latticeai.tools",
-        replacement="from latticeai.tools import execute_tool",
-        reason="Root compatibility shims completed their window; removed in 9.9.1.",
-        removal_phase="removed-9.9.1",
-        layer="root",
-        status="removed",
-    ),
-    LegacyShim(
-        path="lattice_brain/store.py",
-        owner="lattice_brain.graph.store",
-        replacement="from lattice_brain.graph.store import KnowledgeGraphStore",
-        reason="Pre-graph-package flat modules were internal-only; removed in 8.8.0.",
-        removal_phase="removed-8.8.0",
-        layer="brain-flat",
-        status="removed",
-    ),
-    LegacyShim(
-        path="lattice_brain/ingest.py",
-        owner="lattice_brain.graph.ingest",
-        replacement="from lattice_brain.graph.ingest import KnowledgeGraphIngestMixin",
-        reason="Pre-graph-package flat modules were internal-only; removed in 8.8.0.",
-        removal_phase="removed-8.8.0",
-        layer="brain-flat",
-        status="removed",
-    ),
-    LegacyShim(
-        path="lattice_brain/retrieval.py",
-        owner="lattice_brain.graph.retrieval",
-        replacement="from lattice_brain.graph.retrieval import KnowledgeGraphRetrievalMixin",
-        reason="Pre-graph-package flat modules were internal-only; removed in 8.8.0.",
-        removal_phase="removed-8.8.0",
-        layer="brain-flat",
-        status="removed",
-    ),
-    LegacyShim(
-        path="latticeai/brain/",
-        owner="lattice_brain",
-        replacement="import lattice_brain",
-        reason="The deprecated latticeai.brain namespace completed its window; removed in 8.8.0.",
-        removal_phase="removed-8.8.0",
-        layer="deprecated-namespace",
-        status="removed",
-    ),
-    LegacyShim(
-        path="latticeai/services/agent_runtime.py",
-        owner="lattice_brain.runtime.agent_runtime",
-        replacement="from lattice_brain.runtime.agent_runtime import AgentRuntime",
-        reason="The service-layer alias completed its window; removed in 8.8.0.",
-        removal_phase="removed-8.8.0",
-        layer="service-alias",
-        status="removed",
-    ),
-]
-
-
-def legacy_shim_report(root: Path | None = None) -> Dict[str, Any]:
-    if root is None:
-        root = Path(__file__).resolve().parents[2]
-    entries = [shim.as_dict() for shim in LEGACY_SHIMS]
-    missing = [shim.path for shim in LEGACY_SHIMS if not (root / shim.path).exists()]
-    lingering = [shim.path for shim in REMOVED_SHIMS if (root / shim.path).exists()]
-    phases = sorted({shim.removal_phase for shim in LEGACY_SHIMS})
-    layers = sorted({shim.layer for shim in LEGACY_SHIMS})
-    return {
-        "schema_version": "legacy-compatibility/v1",
-        "version_target": LEGACY_COMPATIBILITY_VERSION,
-        "status": "managed" if not missing and not lingering else "incomplete",
-        "remaining_count": len(LEGACY_SHIMS),
-        "removed_count": len(REMOVED_SHIMS),
-        "missing": missing,
-        "lingering": lingering,
-        "removal_phases": phases,
-        "layers": layers,
-        "shims": entries,
-        "removed": [shim.as_dict() for shim in REMOVED_SHIMS],
-    }
-
-
-__all__ = [
-    "LEGACY_COMPATIBILITY_VERSION",
-    "LEGACY_SHIMS",
-    "REMOVED_SHIMS",
-    "LegacyShim",
-    "legacy_shim_report",
-]

package/latticeai/core/logging_safety.py

@@ -1,46 +0,0 @@
-"""Helpers for keeping sensitive values out of logs."""
-
-from __future__ import annotations
-
-import logging
-from typing import Any
-
-from latticeai.core.security import redact_secret_text, redact_secrets
-
-_LOG_FILTER_INSTALLED = False
-
-
-def mask_telegram_bot_token(value: Any) -> str:
-    """Return ``value`` as text with Telegram bot token and other secrets redacted."""
-
-    return redact_secret_text(str(value))
-
-
-def safe_log_text(value: Any) -> str:
-    """Sanitize text before it is sent to application logs."""
-
-    return mask_telegram_bot_token(value)
-
-
-def _safe_log_arg(value: Any) -> Any:
-    return redact_secrets(value)
-
-
-def install_sensitive_log_filter() -> None:
-    """Install a process-wide log-record sanitizer for known secret shapes."""
-
-    global _LOG_FILTER_INSTALLED
-    if _LOG_FILTER_INSTALLED:
-        return
-
-    original_factory = logging.getLogRecordFactory()
-
-    def factory(*args: Any, **kwargs: Any) -> logging.LogRecord:
-        record = original_factory(*args, **kwargs)
-        record.msg = _safe_log_arg(record.msg)
-        if record.args:
-            record.args = _safe_log_arg(record.args)
-        return record
-
-    logging.setLogRecordFactory(factory)
-    _LOG_FILTER_INSTALLED = True