ltcai 9.9.1 → 9.9.3

package/latticeai/api/chat_intents.py

@@ -3,8 +3,10 @@
 from __future__ import annotations
 
 import logging
+import os
+import re
 from pathlib import Path
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
 
 from fastapi import HTTPException, Request
 from fastapi.responses import JSONResponse, StreamingResponse
@@ -17,7 +19,46 @@ from latticeai.api.chat_helpers import (
 )
 from latticeai.api.chat_stream import agent_payload_stream, single_answer_response
 from latticeai.core.agent import AgentState
-from latticeai.core.file_generation import generate_file_content, infer_file_target
+from latticeai.core.file_generation import (
+    PREVIEWABLE_EXTENSIONS,
+    generate_file_content,
+    infer_file_target,
+    infer_project_manifest,
+    repair_bundle_references,
+    validate_project_bundle,
+)
+
+
+def next_available_path(root: Path, target: str) -> str:
+    """Return ``target`` unchanged, or a ``name_2.ext``-style variant when the
+    file already exists in the workspace.
+
+    The direct chat write path must never silently overwrite an existing file:
+    overwrites belong to the reviewable-proposal flow. Deterministic suffixing
+    keeps "make me an html page" repeatable without a 409 dead end.
+    """
+    candidate = (root / target)
+    if not candidate.exists():
+        return target
+    stem, suffix = candidate.stem, candidate.suffix
+    # An earlier auto-suffix run may have produced name_2 already; restart the
+    # numbering from the base name so we fill the first free slot.
+    base = re.sub(r"_\d+$", "", stem) or stem
+    for index in range(2, 100):
+        variant = candidate.with_name(f"{base}_{index}{suffix}")
+        if not variant.exists():
+            return str(Path(target).with_name(variant.name))
+    raise HTTPException(
+        status_code=409,
+        detail=f"'{target}' 이름의 파일이 너무 많습니다. 다른 이름을 지정해 주세요.",
+    )
+
+
+def ingest_generated_enabled() -> bool:
+    """Feature toggle for auto-indexing generated files into the Brain."""
+    return os.environ.get("LATTICEAI_INGEST_GENERATED", "1").strip().lower() not in {
+        "0", "false", "off", "no",
+    }
 
 
 class ChatIntentController:
@@ -41,6 +82,8 @@ class ChatIntentController:
         execute_tool: Any,
         agent_controller: Any,
         agent_root: Path,
+        ingestion_pipeline: Any = None,
+        funnel_metrics: Any = None,
     ) -> None:
         self.router = model_router
         self.config = config
@@ -59,6 +102,17 @@ class ChatIntentController:
         self.execute_tool = execute_tool
         self.agent_controller = agent_controller
         self.agent_root = Path(agent_root)
+        self.ingestion_pipeline = ingestion_pipeline
+        self.funnel_metrics = funnel_metrics
+
+    def _funnel_increment(self, name: str) -> None:
+        """UX funnel counter (backlog #16) — advisory, never raises."""
+        if self.funnel_metrics is None:
+            return
+        try:
+            self.funnel_metrics.increment(name)
+        except Exception as exc:  # noqa: BLE001 — metrics must never break chat
+            logging.warning("funnel metrics increment failed: %s", exc)
 
     def no_model_response(self) -> JSONResponse:
         detail = "No model loaded. Call /models/load first."
@@ -184,13 +238,39 @@ class ChatIntentController:
         )
         return single_answer_response(req, answer, model="client_url")
 
-    async def direct_file_action(self, req: Any, *, model_id: Optional[str]):
+    async def direct_file_action(
+        self,
+        req: Any,
+        *,
+        model_id: Optional[str],
+        effective_email: Optional[str] = None,
+        workspace_id: Optional[str] = None,
+    ):
         # An explicit path ("report.txt") wins; otherwise infer one from an
         # explicit type keyword ("html 파일 만들어줘") so weak models never
         # have to drive the JSON tool loop just to create a file.
-        target_path = file_action_target(req.message) or infer_file_target(req.message)
+        explicit_target = file_action_target(req.message)
+        if explicit_target is None:
+            # Multi-file intent ("todo 앱 html+css+js") becomes a real linked
+            # project bundle instead of one inlined page. Single-type requests
+            # return None here and keep the unchanged single-file flow.
+            manifest = infer_project_manifest(req.message)
+            if manifest is not None:
+                return await self.direct_project_action(
+                    req,
+                    manifest,
+                    model_id=model_id,
+                    effective_email=effective_email,
+                    workspace_id=workspace_id,
+                )
+        target_path = explicit_target or infer_file_target(req.message)
         if not target_path:
             return None
+        # Never silently overwrite: an existing target gets a _2/_3 suffix so
+        # regeneration is safe and repeatable without the proposal flow.
+        deduped_path = next_available_path(self.agent_root, target_path)
+        renamed = deduped_path != target_path
+        target_path = deduped_path
         content = inline_file_action_content(req.message)
         generation_meta: Optional[Dict[str, Any]] = None
         if content is None and not model_id:
@@ -227,19 +307,44 @@ class ChatIntentController:
             )
         except self.tool_error as exc:
             raise HTTPException(status_code=400, detail=str(exc)) from exc
-        answer = f"{result.get('path') or target_path} 파일을 만들었습니다."
-        if generation_meta and generation_meta.get("repaired"):
+        final_path = result.get("path") or target_path
+        answer = f"{final_path} 파일을 만들었습니다."
+        if renamed:
+            answer += " (같은 이름의 파일이 있어 새 이름으로 저장했습니다.)"
+        repaired = bool(generation_meta and generation_meta.get("repaired"))
+        if repaired:
             answer += (
                 " (모델 출력이 불완전해 자동 보정을 거쳐 유효한 파일로 저장했습니다.)"
             )
         created_files = [
             {
-                "path": result.get("path") or target_path,
-                "filename": Path(result.get("path") or target_path).name,
+                "path": final_path,
+                "filename": Path(final_path).name,
                 "bytes": result.get("bytes", 0),
                 "action": "write_file",
             }
         ]
+        # Artifact-first contract: every file the request produced, with the
+        # honesty flags the UI needs (valid vs repaired scaffold, previewable).
+        artifacts: List[Dict[str, Any]] = [
+            {
+                "kind": "file",
+                "path": final_path,
+                "filename": Path(final_path).name,
+                "bytes": result.get("bytes", 0),
+                "previewable": Path(final_path).suffix.lower() in PREVIEWABLE_EXTENSIONS,
+                "valid": True,
+                "repaired": repaired,
+            }
+        ]
+        brain_ingest = self._ingest_generated_file(
+            final_path,
+            content,
+            effective_email=effective_email,
+            workspace_id=workspace_id,
+            conversation_id=getattr(req, "conversation_id", None),
+        )
+        self._funnel_increment("real_file_delivered")
         self.notify("user", req.message, req.source)
         self.notify("assistant", answer, req.source)
         payload = {
@@ -257,11 +362,14 @@ class ChatIntentController:
             "state_history": [AgentState.EXECUTING.value, AgentState.DONE.value],
             "final_state": AgentState.DONE.value,
             "created_files": created_files,
+            "artifacts": artifacts,
             "routed_to_agent": True,
             "action_route": "direct_write_file",
         }
         if generation_meta is not None:
             payload["generation"] = generation_meta
+        if brain_ingest is not None:
+            payload["brain_ingest"] = brain_ingest
         if req.stream:
             return StreamingResponse(
                 agent_payload_stream(
@@ -275,6 +383,200 @@ class ChatIntentController:
             )
         return JSONResponse(content=payload)
 
+    async def direct_project_action(
+        self,
+        req: Any,
+        manifest: Dict[str, Any],
+        *,
+        model_id: Optional[str],
+        effective_email: Optional[str] = None,
+        workspace_id: Optional[str] = None,
+    ):
+        """Artifact Loop: manifest → per-file generate/validate → bundle write.
+
+        Every file goes through the same model-agnostic pipeline as the
+        single-file path; after generation the *bundle* is verified as a whole
+        (dangling href/src references are deterministically repaired first),
+        then written together under one project directory, exposed through the
+        standard ``artifacts[]`` contract, and downloadable as a zip.
+        """
+        if not model_id:
+            return self.no_model_response()
+        project_dir = next_available_path(self.agent_root, manifest["name"])
+        bundle_files = [str(f["path"]) for f in manifest["files"]]
+
+        async def _generate(context: str) -> str:
+            return str(
+                await self.router.generate_as(
+                    model_id,
+                    message="Return only the requested file content.",
+                    context=context,
+                    max_tokens=max(int(req.max_tokens or 0), 4096),
+                    temperature=min(req.temperature, 0.3),
+                )
+            )
+
+        contents: Dict[str, str] = {}
+        file_meta: Dict[str, Dict[str, Any]] = {}
+        for spec in manifest["files"]:
+            name = str(spec["path"])
+            request_text = f"{req.message}\n\nThis file's role: {spec.get('brief', '')}"
+            content, meta = await generate_file_content(
+                _generate,
+                target_path=name,
+                user_request=request_text,
+                bundle_files=bundle_files,
+            )
+            contents[name] = content
+            file_meta[name] = meta
+
+        contents, reference_fixes = repair_bundle_references(contents)
+        bundle_validation = validate_project_bundle(contents)
+
+        steps: List[Dict[str, Any]] = []
+        created_files: List[Dict[str, Any]] = []
+        artifacts: List[Dict[str, Any]] = []
+        brain_ingests: List[Dict[str, Any]] = []
+        for name, content in contents.items():
+            rel_path = f"{project_dir}/{name}"
+            try:
+                result = self.execute_tool(
+                    "write_file",
+                    {"path": rel_path, "content": content},
+                )
+            except self.tool_error as exc:
+                raise HTTPException(status_code=400, detail=str(exc)) from exc
+            final_path = result.get("path") or rel_path
+            steps.append({
+                "state": AgentState.EXECUTING.value,
+                "action": "write_file",
+                "args": {"path": rel_path},
+                "result": result,
+            })
+            created_files.append({
+                "path": final_path,
+                "filename": Path(final_path).name,
+                "bytes": result.get("bytes", 0),
+                "action": "write_file",
+            })
+            verdict = bundle_validation["files"].get(name, {})
+            artifacts.append({
+                "kind": "file",
+                "path": final_path,
+                "filename": Path(final_path).name,
+                "bytes": result.get("bytes", 0),
+                "previewable": Path(final_path).suffix.lower() in PREVIEWABLE_EXTENSIONS,
+                "valid": bool(verdict.get("valid", True)),
+                "repaired": bool(file_meta.get(name, {}).get("repaired")),
+            })
+            ingest = self._ingest_generated_file(
+                final_path,
+                content,
+                effective_email=effective_email,
+                workspace_id=workspace_id,
+                conversation_id=getattr(req, "conversation_id", None),
+            )
+            if ingest is not None:
+                brain_ingests.append({"path": final_path, **ingest})
+
+        file_list = ", ".join(Path(a["path"]).name for a in artifacts)
+        answer = (
+            f"{project_dir} 프로젝트를 만들었습니다 "
+            f"({len(artifacts)}개 파일: {file_list})."
+        )
+        if not bundle_validation["ok"]:
+            answer += " 일부 파일은 검증 경고가 있어 결과를 확인해 주세요."
+        repaired_any = any(a["repaired"] for a in artifacts)
+        if repaired_any:
+            answer += " (모델 출력이 불완전한 파일은 자동 보정을 거쳤습니다.)"
+
+        from urllib.parse import quote
+
+        payload: Dict[str, Any] = {
+            "status": "ok",
+            "response": answer,
+            "workspace": str(self.agent_root),
+            "steps": steps,
+            "state_history": [AgentState.EXECUTING.value, AgentState.DONE.value],
+            "final_state": AgentState.DONE.value,
+            "created_files": created_files,
+            "artifacts": artifacts,
+            "routed_to_agent": True,
+            "action_route": "direct_project_bundle",
+            "project": {
+                "dir": project_dir,
+                "files": bundle_files,
+                "zip_url": f"/tools/download_zip?path={quote(project_dir)}",
+                "bundle_validation": bundle_validation,
+                "reference_fixes": reference_fixes,
+            },
+            "generation": {
+                "files": file_meta,
+                "repaired": repaired_any,
+            },
+        }
+        if brain_ingests:
+            payload["brain_ingest"] = brain_ingests
+        self._funnel_increment("real_file_delivered")
+        self.notify("user", req.message, req.source)
+        self.notify("assistant", answer, req.source)
+        if req.stream:
+            return StreamingResponse(
+                agent_payload_stream(
+                    answer,
+                    payload,
+                    router=self.router,
+                    model_id=model_id,
+                ),
+                media_type="text/event-stream",
+                headers={"X-Model": model_id or "tool", "X-Routed-To": "agent"},
+            )
+        return JSONResponse(content=payload)
+
+    def _ingest_generated_file(
+        self,
+        rel_path: str,
+        content: Optional[str],
+        *,
+        effective_email: Optional[str],
+        workspace_id: Optional[str],
+        conversation_id: Optional[str],
+    ) -> Optional[Dict[str, Any]]:
+        """Optionally index a just-generated file into the Brain.
+
+        The knowledge promise is "what Lattice makes, Lattice remembers":
+        without this, generated files are invisible to recall until the user
+        re-uploads them. Best-effort and additive — ingestion problems never
+        fail the file creation, and LATTICEAI_INGEST_GENERATED=0 disables it.
+        """
+        if self.ingestion_pipeline is None or not ingest_generated_enabled():
+            return None
+        try:
+            from lattice_brain.ingestion import IngestionItem
+
+            item = IngestionItem(
+                source_type="file",
+                title=Path(rel_path).name,
+                text=content,
+                path=str(self.agent_root / rel_path),
+                source_uri=f"workspace://{rel_path}",
+                owner=effective_email,
+                workspace_id=workspace_id,
+                conversation_id=conversation_id,
+                metadata={"origin": "generated_file", "route": "direct_write_file"},
+            )
+            result = self.ingestion_pipeline.ingest(item, user_email=effective_email)
+            payload = result.as_dict() if hasattr(result, "as_dict") else dict(result or {})
+            return {
+                "status": payload.get("status"),
+                "node_id": payload.get("node_id"),
+                "chunk_count": payload.get("chunk_count", 0),
+                "duplicate": payload.get("duplicate", False),
+            }
+        except Exception as exc:  # noqa: BLE001 — indexing must never break creation
+            logging.warning("generated-file ingest failed: %s", exc)
+            return {"status": "failed", "detail": str(exc)[:200]}
+
     async def route_file_to_agent(
         self,
         req: Any,
@@ -296,6 +598,15 @@ class ChatIntentController:
         )
         result = await self.agent_controller.agent(agent_request, request)
         answer = str(result.get("response") or "파일 작업을 처리했습니다.")
+        # UX funnel (backlog #16): a file-intent request that the agent loop
+        # finished without producing a single artifact is the "code-only"
+        # failure mode the >95% real-file goal watches. Paused/failed runs
+        # count neither way — they did not answer with code instead of files.
+        delivered = bool(result.get("created_files") or result.get("artifacts"))
+        if delivered:
+            self._funnel_increment("real_file_delivered")
+        elif str(result.get("status") or "") == "ok":
+            self._funnel_increment("code_only_responses")
         self.notify("user", req.message, req.source)
         self.notify("assistant", answer, req.source)
         result["routed_to_agent"] = True

package/latticeai/api/chat_stream.py

@@ -8,7 +8,7 @@ from typing import Any, AsyncIterator, Dict, Optional
 
 from fastapi.responses import JSONResponse, StreamingResponse
 
-from latticeai.api.chat_helpers import single_text_stream
+from latticeai.api.chat_helpers import assess_answer_grounding, single_text_stream
 
 
 def single_answer_response(req: Any, answer: str, *, model: str):
@@ -87,6 +87,7 @@ async def stream_chat(
         persisted_response = f"{persisted_response}\n\n[stream_error] {stream_error}"
 
     trace_record = None
+    grounding = None
     try:
         answer_trace = trace_seed or chat_service.build_graph_trace(
             req.message,
@@ -94,6 +95,19 @@ async def stream_chat(
             context,
             allowed_workspaces={workspace_id} if workspace_id else None,
         )
+        # Answer-citation binding (backlog #11): same honest verdict as the
+        # non-streaming path, recorded on the persisted trace + trailer.
+        try:
+            grounding = assess_answer_grounding(
+                full_response,
+                trace=answer_trace if isinstance(answer_trace, dict) else None,
+                context_quality=context_quality,
+            )
+            if isinstance(answer_trace, dict):
+                answer_trace["grounding"] = grounding
+        except Exception as exc:  # noqa: BLE001 — annotation must never break streaming
+            logging.warning("answer grounding assessment failed: %s", exc)
+            grounding = None
         trace_record = await chat_service.persist_answer(
             question=req.message,
             response=persisted_response,
@@ -114,6 +128,8 @@ async def stream_chat(
         trailer.update({"trace_id": trace_record["id"], "trace": trace_record})
     if context_quality is not None:
         trailer["context_quality"] = context_quality
+    if grounding is not None:
+        trailer["grounding"] = grounding
     if stream_error:
         trailer["error"] = stream_error
     yield f"data: {json.dumps(trailer, ensure_ascii=False)}\n\n"

package/latticeai/api/funnel_metrics.py

@@ -0,0 +1,31 @@
+"""Admin-only UX funnel metrics endpoint (backlog #16).
+
+Exposes :class:`~latticeai.services.funnel_metrics.FunnelMetricsService`
+snapshots at ``GET /api/admin/funnel-metrics``. Follows the admin router
+convention: gate through the injected ``require_admin`` callable, read-only,
+no side effects.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable
+
+from fastapi import APIRouter, Request
+
+
+def create_funnel_metrics_router(
+    *,
+    service: Any,
+    require_admin: Callable[[Request], Any],
+) -> APIRouter:
+    router = APIRouter()
+
+    @router.get("/api/admin/funnel-metrics")
+    async def funnel_metrics(request: Request):
+        require_admin(request)
+        return service.snapshot()
+
+    return router
+
+
+__all__ = ["create_funnel_metrics_router"]

package/latticeai/api/knowledge_graph.py

@@ -28,6 +28,16 @@ class KnowledgeGraphIngestRequest(BaseModel):
     metadata: Optional[Dict[str, Any]] = None
 
 
+class CurateNoiseRequest(BaseModel):
+    # Dry-run by default: the job *reports* removals until explicitly applied.
+    dry_run: bool = True
+    max_df_ratio: float = 0.8
+    min_doc_frequency: int = 1
+    min_corpus_docs: int = 5
+    normalize_verbs: bool = True
+    max_removals: int = 200
+
+
 def _workspace_scope_from_request(request: Request) -> Optional[str]:
     header = request.headers.get("X-Workspace-Id")
     if header and header.strip():
@@ -143,6 +153,26 @@ def create_knowledge_graph_router(
         (require_admin or require_user)(request)
         return graph().curate()
 
+    @router.post("/knowledge-graph/curate/noise")
+    async def knowledge_graph_curate_noise(req: CurateNoiseRequest, request: Request):
+        """Noise-reduction curation job (backlog #10).
+
+        Dry-run by default: reports which heuristic concept nodes would be
+        removed (low IDF / below the frequency floor) and which relation
+        verbs would be normalized, without changing the graph. Set
+        ``dry_run=false`` to apply. User-created nodes are never removed.
+        Administrative like ``/knowledge-graph/curate`` when roles exist.
+        """
+        (require_admin or require_user)(request)
+        return graph().curate_noise(
+            dry_run=req.dry_run,
+            max_df_ratio=req.max_df_ratio,
+            min_doc_frequency=req.min_doc_frequency,
+            min_corpus_docs=req.min_corpus_docs,
+            normalize_verbs=req.normalize_verbs,
+            max_removals=req.max_removals,
+        )
+
     @router.get("/knowledge-graph/provenance/coverage")
     async def knowledge_graph_provenance_coverage(request: Request):
         require_user(request)

package/latticeai/api/local_files.py

@@ -49,6 +49,16 @@ class FolderIngestRequest(BaseModel):
     approval_token: Optional[str] = None
 
 
+class FolderWatchEnableRequest(BaseModel):
+    path: str
+    recursive: bool = True
+    workspace_id: Optional[str] = None
+    # Watching continuously reads local disk → same approval dance as
+    # /api/ingestion/folder. This is the explicit opt-in the review requires.
+    approved: bool = False
+    approval_token: Optional[str] = None
+
+
 def create_local_files_router(
     *,
     require_user,
@@ -64,9 +74,26 @@ def create_local_files_router(
     data_dir: Optional[Path] = None,
     allowed_workspaces_for=None,
     workspace_service=None,
+    folder_watch=None,
 ) -> APIRouter:
     router = APIRouter()
 
+    # ── opt-in folder watch service (backlog #8) ──────────────────────────────
+    # Constructed here (not injected) unless a test provides one; restore()
+    # only resumes watches persisted with the explicit opt-in — a fresh
+    # data_dir never starts a polling thread.
+    if folder_watch is None and ingestion_pipeline is not None and data_dir is not None:
+        try:
+            from latticeai.services.folder_watch import FolderWatchService
+
+            folder_watch = FolderWatchService(
+                pipeline=ingestion_pipeline,
+                config_path=Path(data_dir) / "folder_watch.json",
+            )
+            folder_watch.restore()
+        except Exception:  # noqa: BLE001 — watch mode is optional, never blocks routes
+            folder_watch = None
+
     @router.get("/api/local-agent/status")
     async def local_agent_status(request: Request):
         """Real on-device runtime status — the 'Local Agent' is the Lattice
@@ -307,6 +334,68 @@ def create_local_files_router(
             )
         return summary
 
+    # ── folder watch mode: opt-in, off by default (backlog #8) ────────────────
+    def _require_folder_watch():
+        _require_pipeline()
+        if folder_watch is None:
+            raise HTTPException(status_code=503, detail="Folder watch service is unavailable.")
+
+    @router.get("/api/ingestion/watch")
+    async def folder_watch_status(request: Request):
+        """Watch-mode status: stored opt-ins, poller state, last scan results."""
+        require_user(request)
+        _require_folder_watch()
+        return folder_watch.status()
+
+    @router.post("/api/ingestion/watch")
+    async def folder_watch_enable(req: FolderWatchEnableRequest, request: Request):
+        """Explicitly opt a previously-ingested folder into watch mode.
+
+        Follows the same local-read approval dance as ``/api/ingestion/folder``.
+        Enabling snapshots the folder as the baseline; only *future* new or
+        changed files are re-ingested (through the normal pipeline, with the
+        watch owner's workspace scope).
+        """
+        current_user = permission_gateway.require_local_user(request)
+        _require_folder_watch()
+        workspace_id = _ingestion_write_workspace(request, req.workspace_id, current_user)
+        path = (req.path or "").strip()
+        if not path:
+            raise HTTPException(status_code=400, detail="path is required.")
+        if not req.approved:
+            return permission_gateway.local_permission_response(path, "read", current_user)
+        permission_gateway.require_local_approval(
+            token=req.approval_token,
+            path=path,
+            action="read",
+            user_email=current_user,
+        )
+        result = folder_watch.enable(
+            path,
+            owner=current_user or None,
+            workspace_id=workspace_id,
+            recursive=req.recursive,
+        )
+        if result.get("status") != "ok":
+            raise HTTPException(status_code=400, detail=result.get("detail") or "watch enable failed")
+        return result
+
+    @router.delete("/api/ingestion/watch")
+    async def folder_watch_disable(
+        request: Request,
+        watch_id: Optional[str] = None,
+        path: Optional[str] = None,
+    ):
+        """Opt back out of watch mode (removes the stored consent record)."""
+        require_user(request)
+        _require_folder_watch()
+        if not watch_id and not path:
+            raise HTTPException(status_code=400, detail="watch_id or path is required.")
+        result = folder_watch.disable(watch_id=watch_id, path=path)
+        if result.get("status") == "not_found":
+            raise HTTPException(status_code=404, detail="watch not found")
+        return result
+
     @router.post("/local/write")
     async def local_write_endpoint(req: LocalWriteRequest, request: Request):
         current_user = permission_gateway.require_local_user(request)
@@ -353,6 +442,7 @@ def create_local_files_router(
 
 __all__ = [
     "FolderIngestRequest",
+    "FolderWatchEnableRequest",
     "LocalAccessRequest",
     "LocalWriteRequest",
     "create_local_files_router",