ltcai 9.9.2 → 9.9.4

package/latticeai/services/funnel_metrics.py

@@ -0,0 +1,210 @@
+"""UX funnel metrics — lightweight runtime counters (backlog #16, review §4.3).
+
+Tracks the product's core value funnel with honest, local-only counters:
+
+* ``file_requests``          — chat requests recognized as file intents;
+* ``real_file_delivered``    — file intents that produced actual artifacts;
+* ``code_only_responses``    — file intents that finished with only a
+  code/prose answer (the failure mode the >95% real-file goal watches);
+* ``agent_runs``             — completed chat agent runs (rate denominator);
+* ``needs_review_runs``      — agent runs that ended in ``NEEDS_REVIEW``;
+* ``ingest_completions``     — successful ingestion pipeline completions;
+* ``recall_successes``       — chat turns whose context was grounded in at
+  least one Brain node;
+* ``approval_pauses``        — agent/workflow runs paused for human approval;
+* ``approval_resumes``       — paused runs a human explicitly resumed
+  (approved), the ``approval_resume_rate`` numerator.
+
+TTFV ("time to first value") derives from two first-occurrence timestamps:
+the first successful ingest and the first grounded recall/answer.
+
+Design constraints (deliberate):
+
+* **Cheap.** One JSON file under the data dir, atomic replace on write, a
+  single ``threading.Lock`` — no background threads, no new dependencies.
+* **Never breaks the product.** Every public method swallows I/O errors;
+  metrics are advisory observability, not a gate.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import tempfile
+import threading
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+LOGGER = logging.getLogger(__name__)
+
+COUNTER_NAMES = (
+    "file_requests",
+    "real_file_delivered",
+    "code_only_responses",
+    "agent_runs",
+    "needs_review_runs",
+    "ingest_completions",
+    "recall_successes",
+    "approval_pauses",
+    "approval_resumes",
+)
+
+_FIRST_NAMES = ("first_ingest_at", "first_value_at")
+
+
+def _utc_now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+
+def _parse_iso(value: Any) -> Optional[datetime]:
+    if not value:
+        return None
+    try:
+        parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    if parsed.tzinfo is None:
+        parsed = parsed.replace(tzinfo=timezone.utc)
+    return parsed
+
+
+def _rate(numerator: int, denominator: int) -> Optional[float]:
+    """Honest rate: ``None`` (not 0.0) when there is no denominator yet."""
+    if denominator <= 0:
+        return None
+    return round(numerator / denominator, 4)
+
+
+class FunnelMetricsService:
+    """Thread-safe, JSON-persisted funnel counters."""
+
+    def __init__(self, path: Any) -> None:
+        self._path = Path(path)
+        self._lock = threading.Lock()
+        self._state: Dict[str, Any] = self._load()
+
+    # ── persistence ──────────────────────────────────────────────────────
+
+    def _load(self) -> Dict[str, Any]:
+        state: Dict[str, Any] = {name: 0 for name in COUNTER_NAMES}
+        for name in _FIRST_NAMES:
+            state[name] = None
+        try:
+            raw = json.loads(self._path.read_text(encoding="utf-8"))
+        except FileNotFoundError:
+            return state
+        except Exception as exc:  # noqa: BLE001 — corrupt metrics never break startup
+            LOGGER.warning("funnel metrics load failed (%s); starting fresh", exc)
+            return state
+        if isinstance(raw, dict):
+            for name in COUNTER_NAMES:
+                try:
+                    state[name] = max(0, int(raw.get(name) or 0))
+                except (TypeError, ValueError):
+                    state[name] = 0
+            for name in _FIRST_NAMES:
+                value = raw.get(name)
+                state[name] = str(value) if value else None
+        return state
+
+    def _save_locked(self) -> None:
+        try:
+            self._path.parent.mkdir(parents=True, exist_ok=True)
+            fd, tmp_name = tempfile.mkstemp(
+                prefix=self._path.name, dir=str(self._path.parent)
+            )
+            try:
+                with os.fdopen(fd, "w", encoding="utf-8") as handle:
+                    json.dump(self._state, handle, ensure_ascii=False, indent=2)
+                os.replace(tmp_name, self._path)
+            finally:
+                if os.path.exists(tmp_name):
+                    try:
+                        os.unlink(tmp_name)
+                    except OSError:
+                        pass
+        except Exception as exc:  # noqa: BLE001 — metrics persistence is best-effort
+            LOGGER.warning("funnel metrics save failed: %s", exc)
+
+    # ── mutation ─────────────────────────────────────────────────────────
+
+    def increment(self, name: str, by: int = 1) -> None:
+        """Increment a known counter; unknown names are ignored (logged)."""
+        if name not in COUNTER_NAMES:
+            LOGGER.warning("funnel metrics: unknown counter %r ignored", name)
+            return
+        try:
+            step = int(by)
+        except (TypeError, ValueError):
+            step = 1
+        if step <= 0:
+            return
+        with self._lock:
+            self._state[name] = int(self._state.get(name) or 0) + step
+            self._save_locked()
+
+    def _mark_first_locked(self, name: str) -> None:
+        if not self._state.get(name):
+            self._state[name] = _utc_now_iso()
+
+    def record_ingest(self, *, duplicate: bool = False) -> None:
+        """A successful ingestion completed; the first one starts the TTFV clock."""
+        with self._lock:
+            self._state["ingest_completions"] = (
+                int(self._state.get("ingest_completions") or 0) + 1
+            )
+            self._mark_first_locked("first_ingest_at")
+            self._save_locked()
+
+    def record_recall_success(self) -> None:
+        """A chat answer was grounded in the Brain; the first one ends TTFV."""
+        with self._lock:
+            self._state["recall_successes"] = (
+                int(self._state.get("recall_successes") or 0) + 1
+            )
+            # First value only counts after knowledge actually entered the Brain.
+            if self._state.get("first_ingest_at"):
+                self._mark_first_locked("first_value_at")
+            self._save_locked()
+
+    # ── reads ────────────────────────────────────────────────────────────
+
+    def ttfv_seconds(self) -> Optional[float]:
+        with self._lock:
+            first_ingest = _parse_iso(self._state.get("first_ingest_at"))
+            first_value = _parse_iso(self._state.get("first_value_at"))
+        if first_ingest is None or first_value is None:
+            return None
+        delta = (first_value - first_ingest).total_seconds()
+        return round(delta, 1) if delta >= 0 else None
+
+    def snapshot(self) -> Dict[str, Any]:
+        """Counters + derived rates for the admin surface."""
+        with self._lock:
+            counters = {name: int(self._state.get(name) or 0) for name in COUNTER_NAMES}
+            firsts = {name: self._state.get(name) for name in _FIRST_NAMES}
+        return {
+            "counters": counters,
+            "firsts": firsts,
+            "rates": {
+                "real_file_rate": _rate(
+                    counters["real_file_delivered"], counters["file_requests"]
+                ),
+                "code_only_rate": _rate(
+                    counters["code_only_responses"], counters["file_requests"]
+                ),
+                "needs_review_rate": _rate(
+                    counters["needs_review_runs"], counters["agent_runs"]
+                ),
+                "approval_resume_rate": _rate(
+                    counters["approval_resumes"], counters["approval_pauses"]
+                ),
+            },
+            "ttfv_seconds": self.ttfv_seconds(),
+            "generated_at": _utc_now_iso(),
+        }
+
+
+__all__ = ["FunnelMetricsService", "COUNTER_NAMES"]

package/latticeai/services/product_readiness.py

@@ -18,7 +18,7 @@ from typing import Any, Dict, List
 
 from latticeai.services.architecture_readiness import architecture_readiness
 
-PRODUCT_VERSION_TARGET = "9.9.2"
+PRODUCT_VERSION_TARGET = "9.9.4"
 
 
 @dataclass(frozen=True)

package/latticeai/services/search_service.py

@@ -7,7 +7,11 @@ keyword search into UI-ready contracts without tying routers to store internals.
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import Any, Dict, Mapping, Optional
+from datetime import datetime
+from typing import Any, Dict, List, Mapping, Optional
+
+from lattice_brain.graph._kg_fsutil import _parse_iso, _recency_score
+from lattice_brain.graph.retrieval_policy import resolve_policy
 
 
 DEFAULT_HYBRID_WEIGHTS = {
@@ -402,24 +406,44 @@ class SearchService:
         allowed_workspaces=None,
         include_legacy_global: bool = False,
     ) -> Dict[str, Any]:
-        weights = {**DEFAULT_HYBRID_WEIGHTS, **dict(weights or {})}
+        # Single retrieval policy (review Wave 0.2): when the caller does not
+        # pin explicit weights, resolve the query class + per-class channel
+        # weights + deterministic rewrite from lattice_brain.graph.
+        # retrieval_policy (the same policy the graph-layer hybrid consults).
+        # Explicit weights still win (and disable rewrite/decay), and the
+        # "fact" class equals DEFAULT_HYBRID_WEIGHTS, so pinned-weight callers
+        # and fact-class queries behave exactly as before.
+        query_class: Optional[str] = None
+        search_query = query
+        rewrite_rules: List[str] = []
+        recency_half_life_days: Optional[float] = None
+        if weights is None:
+            policy = resolve_policy(query)
+            query_class = policy["query_class"]
+            weights = dict(policy["weights"])
+            rewrite_rules = list(policy["rewrite_rules"])
+            recency_half_life_days = policy["recency_half_life_days"]
+            if policy["search_query"] and policy["search_query"] != policy["original_query"]:
+                search_query = policy["search_query"]
+        else:
+            weights = {**DEFAULT_HYBRID_WEIGHTS, **dict(weights)}
         # Scope each channel at the source so out-of-scope rows never enter the
         # fusion set (defense-in-depth — the fused result is re-scoped below too).
         channels = {
             "keyword": self.keyword_search(
-                query,
+                search_query,
                 limit=keyword_limit,
                 allowed_workspaces=allowed_workspaces,
                 include_legacy_global=include_legacy_global,
             ),
             "vector": self.vector_search(
-                query,
+                search_query,
                 limit=vector_limit,
                 allowed_workspaces=allowed_workspaces,
                 include_legacy_global=include_legacy_global,
             ),
             "graph": self.graph_search(
-                query,
+                search_query,
                 limit=graph_limit,
                 allowed_workspaces=allowed_workspaces,
                 include_legacy_global=include_legacy_global,
@@ -452,6 +476,24 @@ class SearchService:
                     current.setdefault("graph_context", [])
                     current["graph_context"].extend(result.get("graph_context") or [])
 
+        # Recency-class age decay (retrieval_policy): dampen each fused score
+        # into the [0.5, 1.0] band so old-but-relevant items sink without ever
+        # being zeroed. Missing/unparseable updated_at keeps multiplier 1.0 —
+        # unknown age is not evidence of staleness. Other classes skip this
+        # block byte-identically.
+        if recency_half_life_days is not None:
+            decay_now = datetime.now()
+            for item in fused.values():
+                stamp = item.get("updated_at")
+                if _parse_iso(stamp):
+                    multiplier = 0.5 + 0.5 * _recency_score(
+                        stamp, now=decay_now, half_life_days=recency_half_life_days
+                    )
+                else:
+                    multiplier = 1.0
+                item["source_scores"]["age_decay"] = round(multiplier, 6)
+                item["score"] = float(item["score"]) * multiplier
+
         matches = self._scope(
             sorted(fused.values(), key=lambda item: item["score"], reverse=True),
             allowed_workspaces,
@@ -464,10 +506,14 @@ class SearchService:
                 "weights": weights,
                 "sources": match.get("sources", []),
             }
+            if query_class is not None:
+                match["fusion"]["query_class"] = query_class
         return {
             "query": query,
             "mode": "hybrid",
+            "query_class": query_class,
             "weights": weights,
+            "policy": {"search_query": search_query, "rewrite_rules": rewrite_rules},
             "channels": {
                 name: {
                     key: value

package/latticeai/setup/demo_corpus.py

@@ -0,0 +1,98 @@
+"""Built-in First Value Loop demo corpus (backlog #3, review §3.3 P0).
+
+Three small Korean-friendly documents shipped in-repo so a brand-new user can
+one-click seed the Brain and immediately experience a successful recall with
+source highlighting — no folder connection, no model download, fully offline.
+
+Contract:
+
+* Documents are ingested through the normal :class:`IngestionPipeline`
+  (``source_type="note"``) with ``source_uri = demo://<id>`` and metadata
+  ``{"demo_corpus": true}`` so they are identifiable and removable later.
+* Ingestion is idempotent — the graph layer dedupes by content hash, so
+  re-POSTing never duplicates documents.
+* :data:`SUGGESTED_QUESTIONS` are pre-filled questions whose answers exist in
+  the corpus (the UI renders them as "ask this" chips).
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List
+
+DEMO_URI_PREFIX = "demo://"
+DEMO_METADATA_FLAG = "demo_corpus"
+
+DEMO_DOCUMENTS: List[Dict[str, str]] = [
+    {
+        "id": "meeting-note",
+        "title": "주간 회의록 — 사이드 프로젝트 킥오프",
+        "text": (
+            "2026-07-20 주간 회의록.\n"
+            "참석: 나, 김민준(백엔드), 박서연(디자인).\n"
+            "핵심 결정: 사이드 프로젝트 '새싹 가든'의 첫 공개 버전을 8월 15일에 "
+            "출시하기로 결정했다. 범위는 식물 기록과 물주기 알림 두 가지로 줄인다.\n"
+            "김민준이 알림 백엔드를 맡고, 박서연이 온보딩 화면을 맡는다.\n"
+            "다음 회의 전까지 각자 프로토타입을 준비하기로 했다."
+        ),
+    },
+    {
+        "id": "project-doc",
+        "title": "프로젝트 개요 — 새싹 가든",
+        "text": (
+            "새싹 가든은 집에서 키우는 식물을 기록하는 작은 앱이다.\n"
+            "기술 스택: 프론트엔드는 React, 백엔드는 FastAPI, 데이터는 SQLite에 "
+            "로컬로 저장한다. 사진은 기기 밖으로 나가지 않는다.\n"
+            "첫 버전 목표: 식물 등록, 물주기 알림, 한 줄 관찰 일기.\n"
+            "수익화는 생각하지 않고, 주말에 만드는 것을 원칙으로 한다."
+        ),
+    },
+    {
+        "id": "personal-note",
+        "title": "개인 노트 — 독서 메모: 아주 작은 습관의 힘",
+        "text": (
+            "『아주 작은 습관의 힘』을 읽고 남긴 메모.\n"
+            "가장 기억에 남는 문장: 습관은 목표가 아니라 시스템으로 만들어진다.\n"
+            "적용해 볼 것: 매일 아침 10분 스트레칭을 양치 직후에 붙여서 시작한다.\n"
+            "핵심은 2분 규칙 — 새 습관은 2분 안에 끝나는 크기로 시작하는 것이다."
+        ),
+    },
+]
+
+# Pre-filled questions whose answers exist in the corpus above. ``source_uri``
+# lets the UI verify the recall cited the right demo document.
+SUGGESTED_QUESTIONS: List[Dict[str, str]] = [
+    {
+        "question": "회의에서 결정한 출시일이 언제야?",
+        "expected_source_uri": DEMO_URI_PREFIX + "meeting-note",
+        "expected_title": "주간 회의록 — 사이드 프로젝트 킥오프",
+    },
+    {
+        "question": "새싹 가든의 기술 스택이 뭐야?",
+        "expected_source_uri": DEMO_URI_PREFIX + "project-doc",
+        "expected_title": "프로젝트 개요 — 새싹 가든",
+    },
+    {
+        "question": "새 습관을 시작할 때 쓰는 2분 규칙이 뭐였지?",
+        "expected_source_uri": DEMO_URI_PREFIX + "personal-note",
+        "expected_title": "개인 노트 — 독서 메모: 아주 작은 습관의 힘",
+    },
+]
+
+
+def demo_source_uri(doc_id: str) -> str:
+    return DEMO_URI_PREFIX + str(doc_id)
+
+
+def suggested_questions() -> List[Dict[str, Any]]:
+    """Copy of the suggestion chips (callers may annotate freely)."""
+    return [dict(item) for item in SUGGESTED_QUESTIONS]
+
+
+__all__ = [
+    "DEMO_DOCUMENTS",
+    "DEMO_METADATA_FLAG",
+    "DEMO_URI_PREFIX",
+    "SUGGESTED_QUESTIONS",
+    "demo_source_uri",
+    "suggested_questions",
+]

package/latticeai/tools/filesystem.py

@@ -6,11 +6,13 @@ Path resolution reads ``latticeai.tools.AGENT_ROOT`` so tests can redirect the s
 
 from __future__ import annotations
 
+import io
 import json
 import re
+import zipfile
 from html.parser import HTMLParser
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
 
 import latticeai.tools as tools
 from latticeai.tools import (
@@ -554,3 +556,38 @@ button { padding: 10px 14px; border-radius: 10px; border: 1px solid #d6d6d6; bac
         "file_count": len(created),
         "bytes": total_bytes,
     }
+
+
+# Generous cap for a generated-project archive — the workspace itself caps
+# individual files at MAX_FILE_BYTES, this only bounds pathological trees.
+MAX_ZIP_TOTAL_BYTES = 50_000_000
+
+
+def zip_workspace_dir(path: str, max_total_bytes: int = MAX_ZIP_TOTAL_BYTES) -> Tuple[bytes, str]:
+    """Zip one workspace directory for download; confinement-safe by design.
+
+    Path resolution goes through the sandboxed ``_resolve_path`` (traversal
+    outside the agent workspace raises ``ToolError``), symlinks are skipped so
+    an in-workspace link can never leak files from elsewhere, and the archive
+    root is the directory name itself (``todo-app/index.html``).
+    """
+    target = _resolve_path(path)
+    if target == tools.AGENT_ROOT:
+        raise ToolError("Cannot zip the entire workspace root — pick a project directory.")
+    if not target.exists() or not target.is_dir():
+        raise ToolError("Path is not a directory in the workspace.")
+
+    buffer = io.BytesIO()
+    total = 0
+    with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive:
+        for child in sorted(target.rglob("*")):
+            if child.is_symlink() or not child.is_file():
+                continue
+            resolved = child.resolve()
+            if target not in resolved.parents:
+                continue  # defense in depth — never archive escaped paths
+            total += child.stat().st_size
+            if total > max_total_bytes:
+                raise ToolError("Directory is too large to zip.")
+            archive.write(child, arcname=str(child.relative_to(target.parent)))
+    return buffer.getvalue(), f"{target.name}.zip"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ltcai",
-  "version": "9.9.2",
+  "version": "9.9.4",
   "description": "Lattice AI — local-first Digital Brain that keeps your knowledge durable across any AI model.",
   "homepage": "https://github.com/TaeSooPark-PTS/LatticeAI#readme",
   "repository": {