ltcai 11.5.2 → 11.7.0

package/latticeai/core/enterprise_admin.py

@@ -1,158 +0,0 @@
-"""Enterprise PoC surfaces (admin policies, audit export, SIEM stub, org settings).
-
-This module is **structure only** — it prepares concrete, discoverable shapes for
-Enterprise governance features while keeping the open-source Community edition
-fully functional and ungated. Every capability here is consulted through
-:data:`latticeai.core.enterprise.capability_registry`; in the Community build
-each is reported ``enabled=False`` and the Community behaviour (local audit
-export, the four base roles, single-tenant local storage) is always available.
-
-Nothing in this module restricts a Community feature. It answers "what *would*
-an Enterprise provider light up, and is it active?" so the admin UI can show an
-honest edition/capability matrix and a SIEM export *preview envelope* without
-shipping any Enterprise implementation.
-"""
-
-from __future__ import annotations
-
-from typing import Any, Dict, List, Optional
-
-from latticeai.core.enterprise import (
-    EnterpriseCapability,
-    capability_registry,
-)
-
-COMMUNITY_NOTICE = (
-    "Community edition: this is an Enterprise extension point and is not "
-    "enforced. Local-first behaviour is always available. See "
-    "docs/ENTERPRISE.md and docs/EDITION_STRATEGY.md."
-)
-
-
-def _cap(capability: EnterpriseCapability) -> bool:
-    return capability_registry.is_capability_enabled(capability)
-
-
-def admin_policies() -> Dict[str, Any]:
-    """Admin policy-pack status + the effective (open) Community policy."""
-    enabled = _cap(EnterpriseCapability.ADMIN_POLICY_PACKS)
-    return {
-        "capability": EnterpriseCapability.ADMIN_POLICY_PACKS.value,
-        "enabled": enabled,
-        "enforced": enabled,
-        "effective_policy": {
-            # Community defaults — descriptive, not enforced by a policy engine.
-            "base_roles": ["owner", "admin", "member", "viewer"],
-            "local_file_access": "approval-token gated (per path/user/action)",
-            "package_install": "admin-only with audit trail",
-            "network_binding": "127.0.0.1 by default",
-            "managed_policy_packs": [] if not enabled else "provided-by-enterprise",
-        },
-        "note": COMMUNITY_NOTICE,
-    }
-
-
-def audit_export_descriptor() -> Dict[str, Any]:
-    """What audit export is available locally vs. via Enterprise SIEM streaming."""
-    siem_enabled = _cap(EnterpriseCapability.SIEM_EXPORT)
-    retention_enabled = _cap(EnterpriseCapability.COMPLIANCE_RETENTION)
-    return {
-        "local_export": {
-            "available": True,
-            "endpoint": "/admin/security/export",
-            "formats": ["json", "csv", "xlsx", "txt", "pdf"],
-            "note": "Community local audit export is always available to admins.",
-        },
-        "siem_streaming": {
-            "capability": EnterpriseCapability.SIEM_EXPORT.value,
-            "enabled": siem_enabled,
-            "note": COMMUNITY_NOTICE,
-        },
-        "compliance_retention": {
-            "capability": EnterpriseCapability.COMPLIANCE_RETENTION.value,
-            "enabled": retention_enabled,
-            "note": COMMUNITY_NOTICE,
-        },
-    }
-
-
-def siem_export_stub(events: Optional[List[Dict[str, Any]]] = None) -> Dict[str, Any]:
-    """A preview of the envelope an Enterprise SIEM exporter would emit.
-
-    In the Community build this is a *stub*: it returns the envelope *shape*
-    (so integrators can see the contract) but ``streamed=False`` and no events
-    are actually pushed to an external SIEM.
-    """
-    enabled = _cap(EnterpriseCapability.SIEM_EXPORT)
-    sample = events or [
-        {
-            "id": "evt_sample",
-            "type": "audit_event",
-            "timestamp": "1970-01-01T00:00:00Z",
-            "actor": "admin@example.com",
-            "action": "model_load",
-            "severity": "informational",
-        }
-    ]
-    envelope = {
-        "format": "ltcai.siem.v1",
-        "encoding": "ndjson",
-        "vendor": "LatticeAI",
-        "product": "Workspace OS",
-        "records": [
-            {
-                "ts": e.get("timestamp"),
-                "actor": e.get("actor"),
-                "act": e.get("action"),
-                "sev": e.get("severity", "informational"),
-                "kind": e.get("type"),
-                "id": e.get("id"),
-            }
-            for e in sample
-        ],
-    }
-    return {
-        "capability": EnterpriseCapability.SIEM_EXPORT.value,
-        "enabled": enabled,
-        "streamed": False if not enabled else True,
-        "destination": None if not enabled else "configured-by-enterprise",
-        "preview_envelope": envelope,
-        "note": COMMUNITY_NOTICE,
-    }
-
-
-def organization_settings() -> Dict[str, Any]:
-    """Org-scale governance capabilities and their (Community=off) state."""
-    governance_caps = [
-        EnterpriseCapability.TENANT_ISOLATION,
-        EnterpriseCapability.RBAC_ABAC_ADVANCED,
-        EnterpriseCapability.SCIM,
-        EnterpriseCapability.IDP_PROVISIONING,
-        EnterpriseCapability.SSO_ADVANCED,
-        EnterpriseCapability.DLP_POLICY,
-        EnterpriseCapability.EDISCOVERY,
-        EnterpriseCapability.PRIVATE_VPC,
-        EnterpriseCapability.AIR_GAPPED_DEPLOYMENT,
-    ]
-    return {
-        "community_baseline": {
-            "workspaces": ["personal", "organization"],
-            "roles": ["owner", "admin", "member", "viewer"],
-            "data_isolation": "single-tenant local storage (~/.ltcai)",
-        },
-        "governance_capabilities": {
-            cap.value: _cap(cap) for cap in governance_caps
-        },
-        "note": COMMUNITY_NOTICE,
-    }
-
-
-def poc_overview() -> Dict[str, Any]:
-    """Combined Enterprise PoC surface for the admin dashboard."""
-    return {
-        "edition": capability_registry.describe(),
-        "admin_policies": admin_policies(),
-        "audit_export": audit_export_descriptor(),
-        "siem_export": siem_export_stub(),
-        "organization_settings": organization_settings(),
-    }

package/latticeai/core/file_generation/__init__.py

@@ -1,115 +0,0 @@
-"""Model-agnostic file content generation pipeline.
-
-Small local models (gemma/qwen/llama 7B class) asked to "generate an HTML
-file" commonly wrap the payload in chat noise: leading commentary ("Sure!
-Here is your page:"), Markdown fences, ``<think>`` reasoning blocks,
-trailing explanations, or an incomplete document. The previous direct-write
-path saved that reply nearly verbatim, so weak models produced broken files.
-
-This package makes the file-creation flow robust regardless of which LLM is
-loaded, by treating the model as an untrusted content source:
-
-1. Prompt   — extension-aware instructions anchored with the exact first
-   line the reply must start with (small models follow examples, not rules).
-   → :mod:`.prompting`
-2. Extract  — strip reasoning blocks and conversational framing, pick the
-   best fenced block, slice known document boundaries. → :mod:`.extraction`
-3. Validate — per-extension structural checks (HTML document shape, JSON
-   parses, CSS has rule blocks, refusal/chat detection). → :mod:`.validation`
-4. Retry    — one corrective attempt that tells the model what was wrong.
-   → :mod:`.orchestration`
-5. Repair   — deterministic scaffolds guarantee the user still gets a valid
-   file even when the model never produces usable output. → :mod:`.repair`
-
-Alongside the five steps: :mod:`.sanitize` is the single write-side door every
-entry point runs untrusted content through, :mod:`.inference` guesses a
-filename or a whole project manifest when the user named neither, and
-:mod:`.bundles` checks that a multi-file project's links hold together.
-
-The pipeline is pure (no I/O, no FastAPI); the chat layer injects an async
-``generate(context) -> str`` callable.
-
-Split into these submodules in v11.3.0 with no behaviour change. Every name
-the single module exposed still resolves from ``latticeai.core.file_generation``.
-
-Stubbing note: a name rebound *here* changes only this module's binding — the
-submodule that calls it holds its own. A test standing in for one of the
-regex/helper seams patches the submodule that reads it (for example
-``latticeai.core.file_generation.inference._PKG_NAME_RE``).
-"""
-
-from __future__ import annotations
-
-# The single module bound this for its own use, which made it part of the
-# surface callers could reach. Re-exported in the redundant-alias form so it
-# reads as deliberate rather than as a leftover import.
-from latticeai.core.quiet import quiet as quiet
-
-from .bundles import _EXTERNAL_REF_PREFIXES as _EXTERNAL_REF_PREFIXES
-from .bundles import _HTML_LOCAL_REF_RE as _HTML_LOCAL_REF_RE
-from .bundles import _local_bundle_refs as _local_bundle_refs
-from .bundles import repair_bundle_references as repair_bundle_references
-from .bundles import validate_project_bundle as validate_project_bundle
-from .extraction import _CHAT_LINE_RE as _CHAT_LINE_RE
-from .extraction import _EXT_FENCE_LANGS as _EXT_FENCE_LANGS
-from .extraction import _FENCE_RE as _FENCE_RE
-from .extraction import _THINK_BLOCK_RE as _THINK_BLOCK_RE
-from .extraction import _THINK_OPEN_RE as _THINK_OPEN_RE
-from .extraction import _ext as _ext
-from .extraction import _slice_html_document as _slice_html_document
-from .extraction import _slice_json_document as _slice_json_document
-from .extraction import _strip_chat_lines as _strip_chat_lines
-from .extraction import extract_file_content as extract_file_content
-from .inference import _CREATE_VERB_RE as _CREATE_VERB_RE
-from .inference import _CSS_HINT_RE as _CSS_HINT_RE
-from .inference import _EXPLICIT_FILENAME_RE as _EXPLICIT_FILENAME_RE
-from .inference import _HTML_HINT_RE as _HTML_HINT_RE
-from .inference import _JS_HINT_RE as _JS_HINT_RE
-from .inference import _PACKAGE_HINT_RE as _PACKAGE_HINT_RE
-from .inference import _PKG_NAME_RE as _PKG_NAME_RE
-from .inference import _PROJECT_NAME_RE as _PROJECT_NAME_RE
-from .inference import _PYTHON_HINT_RE as _PYTHON_HINT_RE
-from .inference import _REACT_HINT_RE as _REACT_HINT_RE
-from .inference import _TYPE_KEYWORDS as _TYPE_KEYWORDS
-from .inference import _VITE_HINT_RE as _VITE_HINT_RE
-from .inference import _python_package_manifest as _python_package_manifest
-from .inference import _react_manifest as _react_manifest
-from .inference import infer_file_target as infer_file_target
-from .inference import infer_project_manifest as infer_project_manifest
-from .orchestration import _salvage_score as _salvage_score
-from .orchestration import generate_file_content as generate_file_content
-from .prompting import _BUNDLE_HTML_MODULE_RULE as _BUNDLE_HTML_MODULE_RULE
-from .prompting import _BUNDLE_HTML_RULE as _BUNDLE_HTML_RULE
-from .prompting import _FIRST_LINE_HINTS as _FIRST_LINE_HINTS
-from .prompting import _TYPE_RULES as _TYPE_RULES
-from .prompting import _bundle_html_rule as _bundle_html_rule
-from .prompting import build_file_generation_context as build_file_generation_context
-from .repair import _repair_html as _repair_html
-from .repair import repair_file_content as repair_file_content
-from .sanitize import PREVIEWABLE_EXTENSIONS as PREVIEWABLE_EXTENSIONS
-from .sanitize import sanitize_write_content as sanitize_write_content
-from .validation import _BRACED_CODE_EXTENSIONS as _BRACED_CODE_EXTENSIONS
-from .validation import _COMMENTARY_RE as _COMMENTARY_RE
-from .validation import _COMPONENT_EXTENSIONS as _COMPONENT_EXTENSIONS
-from .validation import _REFUSAL_RE as _REFUSAL_RE
-from .validation import _check_balanced_delimiters as _check_balanced_delimiters
-from .validation import _check_component_blocks as _check_component_blocks
-from .validation import _looks_like_commentary as _looks_like_commentary
-from .validation import _strip_code_literals as _strip_code_literals
-from .validation import looks_like_refusal as looks_like_refusal
-from .validation import validate_file_content as validate_file_content
-
-__all__ = [
-    "PREVIEWABLE_EXTENSIONS",
-    "build_file_generation_context",
-    "extract_file_content",
-    "generate_file_content",
-    "infer_file_target",
-    "infer_project_manifest",
-    "looks_like_refusal",
-    "repair_bundle_references",
-    "repair_file_content",
-    "sanitize_write_content",
-    "validate_file_content",
-    "validate_project_bundle",
-]

package/latticeai/core/file_generation/bundles.py

@@ -1,76 +0,0 @@
-"""Bundle-level checks: does this set of files hold together?
-
-A multi-file project is only as valid as its links. These helpers find the
-local references an HTML document makes, repoint the ones a weak model got
-almost right, and report every file that is invalid or dangling.
-"""
-
-from __future__ import annotations
-
-import re
-from typing import Any, Dict, List, Tuple
-
-from .extraction import _ext
-from .validation import validate_file_content
-
-_HTML_LOCAL_REF_RE = re.compile(
-    r"(?:href|src)\s*=\s*[\"']([^\"'#?]+)[\"']", re.IGNORECASE
-)
-_EXTERNAL_REF_PREFIXES = ("http://", "https://", "//", "data:", "mailto:", "tel:", "javascript:")
-
-
-def _local_bundle_refs(html: str) -> List[str]:
-    """File references inside an HTML document that must exist in the bundle."""
-    refs: List[str] = []
-    for ref in _HTML_LOCAL_REF_RE.findall(html or ""):
-        candidate = ref.strip()
-        if not candidate or candidate.startswith(_EXTERNAL_REF_PREFIXES):
-            continue
-        if "." not in candidate.rsplit("/", 1)[-1]:
-            continue  # anchors / routes, not files
-        refs.append(candidate)
-    return refs
-
-
-def repair_bundle_references(files: Dict[str, str]) -> Tuple[Dict[str, str], List[str]]:
-    """Deterministically point dangling HTML refs at real bundle files.
-
-    A weak model asked for ``style.css`` sometimes links ``styles.css``. When
-    a referenced file is missing but the bundle contains exactly one file of
-    the same extension, the reference is rewritten. Returns ``(files, fixes)``.
-    """
-    names = {p.rsplit("/", 1)[-1] for p in files}
-    fixes: List[str] = []
-    repaired = dict(files)
-    for path, content in files.items():
-        if _ext(path) not in (".html", ".htm"):
-            continue
-        updated = content
-        for ref in _local_bundle_refs(content):
-            base = ref.rsplit("/", 1)[-1]
-            if base in names:
-                continue
-            same_ext = [n for n in names if _ext(n) == _ext(base)]
-            if len(same_ext) == 1:
-                updated = updated.replace(ref, same_ext[0])
-                fixes.append(f"{path}: '{ref}' -> '{same_ext[0]}'")
-        if updated != content:
-            repaired[path] = updated
-    return repaired, fixes
-
-
-def validate_project_bundle(files: Dict[str, str]) -> Dict[str, Any]:
-    """Bundle-level verification: every file valid, every HTML ref resolvable."""
-    issues: List[str] = []
-    per_file: Dict[str, Dict[str, Any]] = {}
-    names = {p.rsplit("/", 1)[-1] for p in files}
-    for path, content in files.items():
-        ok, reason = validate_file_content(content, path)
-        per_file[path] = {"valid": ok, "reason": reason}
-        if not ok:
-            issues.append(f"{path}: {reason}")
-        if _ext(path) in (".html", ".htm"):
-            for ref in _local_bundle_refs(content):
-                if ref.rsplit("/", 1)[-1] not in names:
-                    issues.append(f"{path}: references missing file '{ref}'")
-    return {"ok": not issues, "issues": issues, "files": per_file}

package/latticeai/core/file_generation/extraction.py

@@ -1,154 +0,0 @@
-"""Recover the intended file payload from an arbitrary model reply.
-
-Step 2 of the pipeline described in the package docstring: strip reasoning
-blocks and conversational framing, pick the best fenced block, and slice known
-document boundaries. Pure string work — nothing here decides whether the result
-is *good*, only what the model most plausibly meant to hand over.
-"""
-
-from __future__ import annotations
-
-import json
-import re
-from typing import Dict, List, Optional, Tuple
-
-from latticeai.core.quiet import quiet
-
-_THINK_BLOCK_RE = re.compile(
-    r"<(think|thinking|reasoning|reflection)>.*?</\1>",
-    re.DOTALL | re.IGNORECASE,
-)
-# Unclosed think block (model hit the token limit mid-reasoning).
-_THINK_OPEN_RE = re.compile(r"<(think|thinking|reasoning)>.*\Z", re.DOTALL | re.IGNORECASE)
-
-_FENCE_RE = re.compile(r"```([\w.+-]*)[ \t]*\n(.*?)```", re.DOTALL)
-
-# Conversational lines that small models prepend/append around the payload.
-_CHAT_LINE_RE = re.compile(
-    r"^\s*("
-    r"(sure|of course|certainly|okay|ok|alright|great|absolutely)\b[^\n]*"
-    r"|here('s| is| are)\b[^\n]*"
-    r"|i('ve| have) (created|written|generated|made)\b[^\n]*"
-    r"|(below|following) is\b[^\n]*"
-    r"|let me know\b[^\n]*"
-    r"|hope (this|that) helps[^\n]*"
-    r"|feel free\b[^\n]*"
-    r"|물론(입니다|이죠|이에요)?[!., ]*[^\n]*"
-    r"|네[,!. ][^\n]*"
-    r"|알겠습니다[^\n]*"
-    r"|다음은[^\n]*(입니다|합니다)[:.]?[^\n]*"
-    r"|아래는?[^\n]*(입니다|내용)[^\n]*"
-    r"|(요청하신|원하시는)[^\n]*(입니다|만들었습니다|작성했습니다)[^\n]*"
-    r"|(파일|내용|코드)[을를]?\s*(생성|작성|만들)[^\n]*"
-    r"|도움이 (필요하|되)[^\n]*"
-    r"|추가로[^\n]*(말씀|요청)[^\n]*"
-    r")\s*$",
-    re.IGNORECASE,
-)
-
-
-# Language tags that identify a fenced block as the payload for an extension.
-_EXT_FENCE_LANGS: Dict[str, Tuple[str, ...]] = {
-    ".html": ("html", "htm", "xhtml"),
-    ".htm": ("html", "htm", "xhtml"),
-    ".css": ("css",),
-    ".js": ("js", "javascript"),
-    ".jsx": ("jsx", "javascript"),
-    ".ts": ("ts", "typescript"),
-    ".tsx": ("tsx", "typescript"),
-    ".py": ("py", "python"),
-    ".json": ("json",),
-    ".yaml": ("yaml", "yml"),
-    ".yml": ("yaml", "yml"),
-    ".toml": ("toml",),
-    ".md": ("md", "markdown"),
-    ".markdown": ("md", "markdown"),
-    ".sql": ("sql",),
-    ".sh": ("sh", "bash", "shell", "zsh"),
-    ".xml": ("xml", "svg"),
-    ".csv": ("csv",),
-    ".txt": ("txt", "text", "plaintext"),
-    ".vue": ("vue", "html"),
-    ".svelte": ("svelte", "html"),
-}
-
-
-def _ext(path: str) -> str:
-    dot = path.rfind(".")
-    return path[dot:].lower() if dot >= 0 else ""
-
-
-def _strip_chat_lines(text: str) -> str:
-    """Drop leading/trailing conversational lines around the payload."""
-    lines = text.split("\n")
-    start, end = 0, len(lines)
-    while start < end and (not lines[start].strip() or _CHAT_LINE_RE.match(lines[start])):
-        start += 1
-    while end > start and (not lines[end - 1].strip() or _CHAT_LINE_RE.match(lines[end - 1])):
-        end -= 1
-    stripped = "\n".join(lines[start:end]).strip()
-    return stripped if stripped else text.strip()
-
-
-def extract_file_content(raw: str, target_path: str) -> str:
-    """Recover the intended file payload from an arbitrary model reply."""
-    text = (raw or "").strip()
-    if not text:
-        return ""
-    text = _THINK_BLOCK_RE.sub("", text)
-    text = _THINK_OPEN_RE.sub("", text).strip()
-
-    ext = _ext(target_path)
-    fences = _FENCE_RE.findall(text + ("\n```" if text.count("```") % 2 else ""))
-    if fences:
-        wanted = _EXT_FENCE_LANGS.get(ext, ())
-        matching = [body for lang, body in fences if lang.lower() in wanted]
-        candidates = matching if matching else [body for _, body in fences]
-        # The payload is the largest block; short blocks are usually usage
-        # snippets ("run it with: python app.py").
-        content = max(candidates, key=len).strip()
-    else:
-        content = _strip_chat_lines(text)
-
-    if ext in (".html", ".htm"):
-        content = _slice_html_document(content)
-    elif ext == ".json":
-        sliced = _slice_json_document(content)
-        if sliced is not None:
-            content = sliced
-    return content.strip()
-
-
-def _slice_html_document(content: str) -> str:
-    """Cut a complete HTML document out of surrounding prose when present."""
-    lower = content.lower()
-    start = lower.find("<!doctype")
-    if start < 0:
-        start = lower.find("<html")
-    if start > 0:
-        content = content[start:]
-        lower = lower[start:]
-    end = lower.rfind("</html>")
-    if end >= 0:
-        content = content[: end + len("</html>")]
-    return content
-
-
-def _slice_json_document(content: str) -> Optional[str]:
-    """Return the largest parseable JSON value inside ``content``, if any."""
-    candidates: List[str] = [content]
-    for opener, closer in (("{", "}"), ("[", "]")):
-        start = content.find(opener)
-        end = content.rfind(closer)
-        if start >= 0 and end > start:
-            candidates.append(content[start : end + 1])
-    best: Optional[str] = None
-    for candidate in candidates:
-        try:
-            json.loads(candidate)
-        except (ValueError, TypeError):
-            quiet()
-            continue
-        if best is None or len(candidate) > len(best):
-            best = candidate
-    return best