ltcai 10.6.3 → 10.8.0

package/latticeai/core/file_generation.py

@@ -312,9 +312,54 @@ def validate_file_content(content: str, target_path: str) -> Tuple[bool, str]:
         if "```" in content:
             return False, "output still contains Markdown fences"
         return True, "ok"
+    # Prose types (.md, .txt, .csv, …) have no grammar to check, which used to
+    # mean *nothing* was checked: a 1–4B model that answered "Sure! Here is the
+    # document you asked for:" and stopped had its sentence saved as the file,
+    # because the fence stripper only removes conversational lines it can
+    # recognise and the length guard on `looks_like_refusal` lets a wordy
+    # refusal through. The two checks below are the only ones that generalise
+    # without inventing a grammar: it must not still be wearing fences, and it
+    # must not be *only* an answer about the file.
+    if "```" in content:
+        return False, "output still contains Markdown fences"
+    if _looks_like_commentary(content):
+        return False, "the reply talks about the file instead of being the file"
     return True, "ok"
 
 
+# Openers that mean "I am about to give you the thing" — if the whole reply is
+# one of these, the thing never arrived.
+_COMMENTARY_RE = re.compile(
+    r"^\s*("
+    r"(sure|of course|certainly|okay|ok|alright|here|below|the following)\b"
+    r"|i('ve| have| will|'ll)\b"
+    r"|(물론|네[,!. ]|알겠|다음은|아래(는|의)?|요청하신|원하시는)"
+    r")",
+    re.IGNORECASE,
+)
+
+
+def _looks_like_commentary(content: str) -> bool:
+    """True when the reply reads as an answer *about* a file, not the file.
+
+    Deliberately conservative — a long document that merely opens with "The
+    following" is a document. Only a short reply that both opens
+    conversationally and never grows into content is rejected, so a real file
+    is never thrown away to catch a chat line.
+    """
+    stripped = content.strip()
+    if len(stripped) > 400:
+        return False
+    if not _COMMENTARY_RE.match(stripped):
+        return False
+    # Structure means content arrived after the preamble: a heading, a list, a
+    # table row, a delimiter, or simply several lines of body text.
+    body = stripped.split("\n", 1)[1].strip() if "\n" in stripped else ""
+    if re.search(r"^\s*(#{1,6}\s|[-*+]\s|\d+[.)]\s|\||>)", body, re.MULTILINE):
+        return False
+    return len(body) < 120
+
+
 # ── prompting ───────────────────────────────────────────────────────────
 
 _FIRST_LINE_HINTS: Dict[str, str] = {
@@ -869,14 +914,28 @@ async def generate_file_content(
     """Generate validated file content with any LLM.
 
     ``generate`` is an async callable ``context -> raw model text``. Runs up
-    to ``max_attempts`` model calls (the second with corrective feedback),
+    to ``max_attempts`` model calls (each retry carrying corrective feedback),
     then falls back to deterministic repair, so the returned content is
     always non-empty and structurally valid for the target type.
+
+    One extra call beyond ``max_attempts`` is spent — at most once per
+    request — when the model has returned a byte-identical rejected reply.
+    That is the one case where the ordinary retry is known to be dead on
+    arrival: the corrective feedback did not change the reply, so the budget
+    is better spent on a prompt that names the repetition than on a third
+    identical round trip. Small local models hit this constantly; large ones
+    never do, so the extra call is not charged to models that do not need it.
     """
     attempts: List[Dict[str, Any]] = []
     feedback: Optional[str] = None
-    last_candidate = ""
-    for attempt in range(1, max_attempts + 1):
+    best_candidate = ""
+    best_score = (-1, -1)
+    seen: set[str] = set()
+    escalations_left = 1
+    attempt = 0
+    budget = max_attempts
+    while attempt < budget:
+        attempt += 1
         context = build_file_generation_context(
             target_path, user_request, feedback=feedback, bundle_files=bundle_files,
         )
@@ -888,16 +947,90 @@ async def generate_file_content(
             continue
         candidate = extract_file_content(str(raw or ""), target_path)
         ok, reason = validate_file_content(candidate, target_path)
-        attempts.append({"attempt": attempt, "valid": ok, "reason": reason})
+        record: Dict[str, Any] = {"attempt": attempt, "valid": ok, "reason": reason}
         if ok:
+            attempts.append(record)
             return candidate, {"attempts": attempts, "repaired": False}
-        if len(candidate) > len(last_candidate):
-            last_candidate = candidate
-        feedback = reason
-    repaired = repair_file_content(last_candidate, target_path, user_request)
+
+        # A small model handed the same corrective feedback often replays the
+        # same reply verbatim. Saying "you sent this before" is the only signal
+        # left that has any chance of moving it, and it makes the wasted retry
+        # visible in the trace instead of looking like two genuine tries.
+        fingerprint = candidate.strip()
+        repeated = fingerprint in seen and bool(fingerprint)
+        record["repeated"] = repeated
+        seen.add(fingerprint)
+        if repeated and escalations_left and attempt >= budget:
+            # The retry budget is exhausted and the last thing it bought was a
+            # duplicate. Buy one more, but only with a prompt that says so.
+            escalations_left -= 1
+            budget += 1
+            record["escalated"] = True
+        attempts.append(record)
+
+        # Keep the candidate that is *closest to a file*, not the longest one.
+        # Longest-wins handed repair a 900-character apology in preference to a
+        # 300-character HTML document that only needed its </html> closing —
+        # and repair can finish the document but can only bury the apology.
+        score = _salvage_score(candidate, target_path)
+        if score > best_score:
+            best_score, best_candidate = score, candidate
+
+        feedback = (
+            f"{reason}. You already sent exactly this reply and it was rejected "
+            "for the same reason — do not repeat it. Output the file itself, "
+            "starting at its first character."
+            if repeated else reason
+        )
+    repaired = repair_file_content(best_candidate, target_path, user_request)
     return repaired, {"attempts": attempts, "repaired": True}
 
 
+def _salvage_score(candidate: str, target_path: str) -> Tuple[int, int]:
+    """How useful an invalid candidate is as raw material for repair.
+
+    ``(tier, length)`` — tier first, so a short real document always beats a
+    long non-document; length breaks ties within a tier.
+
+    Tier 2  something of the right shape that repair can finish (an HTML
+            document missing its close tag, parseable-ish JSON, Python that
+            at least tokenises).
+    Tier 1  ordinary text: no structure, but the words may be the content.
+    Tier 0  a refusal — repair should prefer literally anything else, because
+            an apology written into the file is worse than an empty stub.
+    """
+    text = candidate.strip()
+    if not text:
+        return (0, 0)
+    if looks_like_refusal(text):
+        return (0, len(text))
+
+    ext = _ext(target_path)
+    lower = text.lower()
+    if ext in (".html", ".htm"):
+        if lower.startswith("<!doctype") or lower.startswith("<html"):
+            return (2, len(text))
+    elif ext == ".json":
+        if _slice_json_document(text) is not None:
+            return (2, len(text))
+    elif ext == ".py":
+        try:
+            ast.parse(text)
+        except SyntaxError:
+            pass
+        else:
+            return (2, len(text))
+    elif ext in _BRACED_CODE_EXTENSIONS:
+        if _check_balanced_delimiters(text)[0]:
+            return (2, len(text))
+    elif ext in _COMPONENT_EXTENSIONS:
+        if _check_component_blocks(text)[0]:
+            return (2, len(text))
+    elif ext == ".css" and "{" in text and "}" in text:
+        return (2, len(text))
+    return (1, len(text))
+
+
 __all__ = [
     "PREVIEWABLE_EXTENSIONS",
     "build_file_generation_context",

package/latticeai/core/legacy_compatibility.py

@@ -13,7 +13,7 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, Dict, List
 
-LEGACY_COMPATIBILITY_VERSION = "10.6.3"
+LEGACY_COMPATIBILITY_VERSION = "10.8.0"
 
 
 @dataclass(frozen=True)

package/latticeai/core/marketplace.py

@@ -10,7 +10,7 @@ from __future__ import annotations
 from copy import deepcopy
 from typing import Any, Dict, List, Optional
 
-MARKETPLACE_VERSION = "10.6.3"
+MARKETPLACE_VERSION = "10.8.0"
 TEMPLATE_KINDS = ("plugin", "workflow", "agent", "ingestion_bridge")
 
 

package/latticeai/core/messages.py

@@ -0,0 +1,230 @@
+"""Server-side message catalog.
+
+Every string the API hands a person — an ``HTTPException`` detail, a status
+line, a refusal — used to be a literal at the raise site, and the literals were
+written in whichever language the author happened to be thinking in. The result
+was one API that answered ``"사용자를 찾을 수 없습니다."`` and, two files over,
+``"Knowledge Graph ingestion is disabled."``, so *every* user read half the
+product in a language they had not chosen.
+
+The rules here are the same three the frontend catalog follows:
+
+1. **One id per message.** The id is what code refers to; the wording is data.
+2. **Both languages or neither.** ``test_messages.py`` fails on a key present
+   in one language and missing from the other, so a new message cannot ship
+   half-translated.
+3. **The caller never picks the language.** It comes from the request, via
+   :func:`resolve_language`, so a message cannot be localized to whoever wrote
+   the endpoint.
+
+Usage::
+
+    from latticeai.core.messages import http_error, resolve_language
+
+    lang = resolve_language(request)
+    raise http_error(404, "auth.user_not_found", lang)
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Mapping, Optional
+
+from fastapi import HTTPException
+
+__all__ = [
+    "DEFAULT_LANGUAGE",
+    "SUPPORTED_LANGUAGES",
+    "LANGUAGE_HEADER",
+    "MESSAGES",
+    "http_error",
+    "resolve_language",
+    "translate",
+]
+
+DEFAULT_LANGUAGE = "ko"
+SUPPORTED_LANGUAGES = ("ko", "en")
+
+#: Explicit override the frontend sends. Preferred over ``Accept-Language``
+#: because the person picked this language *in the product*, whereas
+#: ``Accept-Language`` is whatever their browser was installed with.
+LANGUAGE_HEADER = "x-lattice-language"
+
+
+MESSAGES: Dict[str, Dict[str, str]] = {
+    # ── auth ────────────────────────────────────────────────────────────
+    "auth.password_too_weak": {
+        "ko": "비밀번호는 8자 이상이며 영문자와 숫자를 모두 포함해야 합니다.",
+        "en": "Your password needs at least 8 characters, including letters and numbers.",
+    },
+    "auth.invitation_required": {
+        "ko": "유효한 서명 초대 권한이 필요합니다.",
+        "en": "A valid signed invitation is required.",
+    },
+    "auth.registration_disabled": {
+        "ko": "회원가입이 비활성화되어 있습니다. 관리자에게 문의하세요.",
+        "en": "Sign-up is turned off. Ask an administrator to enable it.",
+    },
+    "auth.email_taken": {
+        "ko": "이미 존재하는 이메일입니다.",
+        "en": "That email address is already registered.",
+    },
+    "auth.bad_credentials": {
+        "ko": "이메일 또는 비밀번호가 틀렸습니다.",
+        "en": "That email address or password is not correct.",
+    },
+    "auth.account_disabled": {
+        "ko": "비활성화된 계정입니다.",
+        "en": "This account has been disabled.",
+    },
+    "auth.login_required": {
+        "ko": "인증이 필요합니다.",
+        "en": "You need to sign in first.",
+    },
+    "auth.user_not_found": {
+        "ko": "사용자를 찾을 수 없습니다.",
+        "en": "No such user.",
+    },
+    "auth.current_password_wrong": {
+        "ko": "현재 비밀번호가 틀렸습니다.",
+        "en": "Your current password is not correct.",
+    },
+    "auth.name_required": {
+        "ko": "이름을 입력해주세요.",
+        "en": "Please enter a name.",
+    },
+    "auth.nickname_required": {
+        "ko": "닉네임을 입력해주세요.",
+        "en": "Please enter a nickname.",
+    },
+    # ── SSO ─────────────────────────────────────────────────────────────
+    "sso.not_configured": {
+        "ko": "SSO가 설정되지 않았습니다.",
+        "en": "Single sign-on is not set up.",
+    },
+    "sso.invalid_state": {
+        "ko": "유효하지 않은 SSO 상태입니다.",
+        "en": "That sign-on request is no longer valid. Please start again.",
+    },
+    "sso.config_error": {
+        "ko": "SSO 설정 오류입니다.",
+        "en": "Single sign-on is misconfigured.",
+    },
+    "sso.no_id_token": {
+        "ko": "ID 토큰을 받지 못했습니다.",
+        "en": "The identity provider did not return an ID token.",
+    },
+    "sso.token_verification_failed": {
+        "ko": "SSO 토큰 검증에 실패했습니다.",
+        "en": "The sign-on token could not be verified.",
+    },
+    "sso.provider_verification_failed": {
+        "ko": "SSO 공급자 검증에 실패했습니다.",
+        "en": "The sign-on provider could not be verified.",
+    },
+    "sso.email_unavailable": {
+        "ko": "이메일을 확인할 수 없습니다.",
+        "en": "The identity provider did not share an email address.",
+    },
+    "sso.invitation_required": {
+        "ko": "신규 SSO 계정에는 유효한 서명 초대 권한이 필요합니다.",
+        "en": "New single sign-on accounts need a valid signed invitation.",
+    },
+    # ── admin ───────────────────────────────────────────────────────────
+    "admin.invalid_role": {
+        "ko": "role은 admin 또는 user만 가능합니다.",
+        "en": "Role must be either 'admin' or 'user'.",
+    },
+    "admin.cannot_disable_self": {
+        "ko": "자기 자신은 비활성화할 수 없습니다.",
+        "en": "You cannot disable your own account.",
+    },
+    "admin.cannot_delete_self": {
+        "ko": "자기 자신은 삭제할 수 없습니다.",
+        "en": "You cannot delete your own account.",
+    },
+    # ── capture / ingestion ─────────────────────────────────────────────
+    "capture.ingestion_disabled": {
+        "ko": "지식 그래프 수집이 꺼져 있습니다.",
+        "en": "Knowledge Graph ingestion is turned off.",
+    },
+    "capture.payload_too_large": {
+        "ko": "보낸 내용이 너무 큽니다.",
+        "en": "That capture is too large to accept.",
+    },
+    "capture.nothing_to_capture": {
+        "ko": "저장할 내용이 없습니다. 텍스트나 페이지 내용을 함께 보내주세요.",
+        "en": "Nothing to capture — send text, html, or a selection.",
+    },
+}
+
+
+def resolve_language(request: Any, default: str = DEFAULT_LANGUAGE) -> str:
+    """Language for this request: the product's choice, then the browser's.
+
+    Never raises and never returns an unsupported language — an endpoint that
+    cannot resolve a language must still be able to answer.
+    """
+    headers: Optional[Mapping[str, str]] = getattr(request, "headers", None)
+    if headers is None:
+        return default
+
+    explicit = _normalize(headers.get(LANGUAGE_HEADER))
+    if explicit:
+        return explicit
+
+    # `Accept-Language: en-GB,en;q=0.9,ko;q=0.8` — take the first tag we
+    # support, honouring the order the browser sent (which is the priority
+    # order) rather than re-sorting by q-value, since the two agree in every
+    # real header and disagreeing ones are malformed anyway.
+    for part in (headers.get("accept-language") or "").split(","):
+        candidate = _normalize(part.split(";", 1)[0])
+        if candidate:
+            return candidate
+    return default
+
+
+def _normalize(value: Optional[str]) -> Optional[str]:
+    """``"en-GB"`` → ``"en"`` when supported, else None."""
+    if not value:
+        return None
+    tag = value.strip().lower().replace("_", "-")
+    if not tag:
+        return None
+    base = tag.split("-", 1)[0]
+    return base if base in SUPPORTED_LANGUAGES else None
+
+
+def translate(key: str, language: str = DEFAULT_LANGUAGE, **params: Any) -> str:
+    """Localized text for ``key``.
+
+    An unknown key returns the key itself rather than raising: a missing
+    message must not turn a 404 into a 500. The namespaced key is also
+    recognisable in a bug report, which a generic "an error occurred" is not.
+    """
+    entry = MESSAGES.get(key)
+    if entry is None:
+        return key
+    text = entry.get(language) or entry.get(DEFAULT_LANGUAGE) or key
+    for name, value in params.items():
+        text = text.replace("{" + name + "}", str(value))
+    return text
+
+
+def http_error(
+    status_code: int,
+    key: str,
+    language: str = DEFAULT_LANGUAGE,
+    *,
+    headers: Optional[Dict[str, str]] = None,
+    **params: Any,
+) -> HTTPException:
+    """An ``HTTPException`` whose detail is localized for this request.
+
+    Returned, not raised, so call sites keep reading ``raise http_error(...)``
+    and the traceback still points at the endpoint.
+    """
+    return HTTPException(
+        status_code=status_code,
+        detail=translate(key, language, **params),
+        headers=headers,
+    )

package/latticeai/core/workspace_computer_memory.py

@@ -0,0 +1,84 @@
+"""Local Computer Memory: the opt-in that lets the Brain watch this machine.
+
+Extracted from ``WorkspaceOSStore``. Off by default and impossible to enable
+without a recorded consent — the check lives here so there is one place to read
+when asking "what did this machine agree to observe".
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from .timeutil import now_iso as _now
+from .workspace_os_utils import _json_hash
+
+__all__ = ["WorkspaceComputerMemory", "DEFAULT_COMPUTER_MEMORY_SCOPES"]
+
+#: Folders the feature offers to watch when the person names none. Chosen to be
+#: recognisable rather than exhaustive: the point of the list is that a person
+#: reading the consent dialog knows what it means.
+DEFAULT_COMPUTER_MEMORY_SCOPES = ("Downloads", "Documents", "Repositories")
+
+
+class WorkspaceComputerMemory:
+    """Consent state and activity log for watching the local computer."""
+
+    def __init__(self, store: Any) -> None:
+        self.store = store
+
+    def configure(
+        self,
+        *,
+        enabled: bool,
+        approved_by: Optional[str],
+        consent: Optional[Dict[str, Any]] = None,
+        scopes: Optional[List[str]] = None,
+    ) -> Dict[str, Any]:
+        consent = consent or {}
+        if enabled and not consent.get("approved"):
+            raise PermissionError("Local Computer Memory requires explicit approval.")
+        state = self.store.load_state()
+        config = state.setdefault("computer_memory", {})
+        config.update({
+            "enabled": bool(enabled),
+            "approved": bool(enabled),
+            "approved_at": _now() if enabled else config.get("approved_at"),
+            "approved_by": approved_by if enabled else config.get("approved_by"),
+            "scopes": scopes or config.get("scopes") or list(DEFAULT_COMPUTER_MEMORY_SCOPES),
+            "consent": consent,
+        })
+        state.setdefault("feature_flags", {})["local_computer_memory"] = bool(enabled)
+        self.store.save_state(state)
+        self.store.record_timeline_event(
+            "memory",
+            "computer_memory_configured",
+            {"enabled": bool(enabled), "approved_by": approved_by},
+        )
+        return config
+
+    def record_activity(self, activity: Dict[str, Any], graph: Any = None) -> Dict[str, Any]:
+        state = self.store.load_state()
+        config = state.setdefault("computer_memory", {})
+        if not config.get("enabled"):
+            return {"status": "ignored", "reason": "local computer memory is disabled"}
+        record = {
+            "id": f"activity-{_json_hash([activity, _now()])[:16]}",
+            "timestamp": _now(),
+            **activity,
+        }
+        config.setdefault("activities", []).append(record)
+        if graph is not None:
+            # A graph that refuses the event must not lose the activity: the
+            # record is kept either way and carries the reason it did not land.
+            try:
+                graph.ingest_event(
+                    "ComputerActivity",
+                    str(activity.get("summary") or activity.get("path") or "Computer activity")[:120],
+                    source="workspace_os",
+                    metadata=record,
+                )
+            except Exception as exc:
+                record["graph_error"] = str(exc)
+        self.store.save_state(state)
+        self.store.record_timeline_event("memory", "computer_activity", {"activity_id": record["id"]})
+        return {"status": "ok", "activity": record}

package/latticeai/core/workspace_indexing.py

@@ -0,0 +1,102 @@
+"""Indexing dashboard and per-source watch control.
+
+Extracted from ``WorkspaceOSStore`` so the store stays a façade over composed
+managers rather than a class that also knows how a file watcher reports itself.
+Everything here reads the graph and the watcher and writes only timeline
+events, so it needs the store for ``record_timeline_event`` and nothing else.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Optional
+
+__all__ = ["WorkspaceIndexing"]
+
+
+class WorkspaceIndexing:
+    """Reads index state out of the graph; pauses, resumes, and removes sources."""
+
+    def __init__(self, store: Any) -> None:
+        self.store = store
+
+    def build_dashboard(
+        self,
+        graph: Any,
+        watcher_status: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        if graph is None:
+            return {
+                "sources": [],
+                "watcher": watcher_status or {"available": False, "active": {}},
+                "totals": {"success": 0, "failed": 0, "nodes": 0, "edges": 0},
+            }
+        stats = graph.stats()
+        sources = graph.local_sources().get("sources", [])
+        watcher_status = watcher_status or {"available": False, "active": {}}
+        active = watcher_status.get("active", {})
+        dashboard_sources = []
+        total_success = 0
+        total_failed = 0
+        for source in sources:
+            file_status = source.get("file_status") or {}
+            success = int(file_status.get("indexed") or 0)
+            failed = sum(
+                int(file_status.get(key) or 0)
+                for key in ("failed", "inaccessible", "skipped_empty_text")
+            )
+            total_success += success
+            total_failed += failed
+            watch = active.get(source.get("id")) or {}
+            dashboard_sources.append({
+                "id": source.get("id"),
+                "label": source.get("label"),
+                "root_path": source.get("root_path"),
+                "status": source.get("status"),
+                "watch_enabled": bool(source.get("watch_enabled")),
+                "watch_active": source.get("id") in active,
+                "watch_status": watch,
+                "success_count": success,
+                "failure_count": failed,
+                "last_run_at": source.get("last_scanned_at") or source.get("updated_at"),
+                "file_status": file_status,
+                "include_ocr": bool(source.get("include_ocr")),
+            })
+        return {
+            "sources": dashboard_sources,
+            "watcher": watcher_status,
+            "totals": {
+                "success": total_success,
+                "failed": total_failed,
+                "nodes": sum(int(v or 0) for v in (stats.get("nodes") or {}).values()),
+                "edges": sum(int(v or 0) for v in (stats.get("edges") or {}).values()),
+                "local_sources": stats.get("local_sources", len(sources)),
+            },
+            "graph_stats": stats,
+        }
+
+    def pause(self, graph: Any, source_id: str, watcher: Any = None) -> Dict[str, Any]:
+        result = graph.set_local_source_watch(source_id, False)
+        watch = watcher.stop_source(source_id) if watcher else {"stopped": False, "source_id": source_id}
+        self.store.record_timeline_event("graph", "indexing_paused", {"source_id": source_id})
+        return {"status": "ok", "source": result, "watch": watch}
+
+    def resume(self, graph: Any, source_id: str, watcher: Any = None) -> Dict[str, Any]:
+        result = graph.set_local_source_watch(source_id, True)
+        watch = {"watching": False, "source_id": source_id}
+        source = next(
+            (item for item in graph.local_sources().get("sources", []) if item.get("id") == source_id),
+            None,
+        )
+        if watcher and source:
+            watch = watcher.start_source(source)
+        self.store.record_timeline_event("graph", "indexing_resumed", {"source_id": source_id})
+        return {"status": "ok", "source": result, "watch": watch}
+
+    def remove_source(self, graph: Any, source_id: str, watcher: Any = None) -> Dict[str, Any]:
+        if watcher:
+            watcher.stop_source(source_id)
+        if not hasattr(graph, "remove_local_source"):
+            raise ValueError("graph store does not support removing local sources")
+        result = graph.remove_local_source(source_id)
+        self.store.record_timeline_event("graph", "indexing_removed", {"source_id": source_id})
+        return {"status": "ok", **result}