ltcai 9.9.2 → 9.9.4

package/latticeai/api/chat_intents.py

@@ -17,12 +17,19 @@ from latticeai.api.chat_helpers import (
     format_network_status,
     inline_file_action_content,
 )
-from latticeai.api.chat_stream import agent_payload_stream, single_answer_response
+from latticeai.api.chat_stream import (
+    agent_live_stream,
+    agent_payload_stream,
+    single_answer_response,
+)
 from latticeai.core.agent import AgentState
 from latticeai.core.file_generation import (
     PREVIEWABLE_EXTENSIONS,
     generate_file_content,
     infer_file_target,
+    infer_project_manifest,
+    repair_bundle_references,
+    validate_project_bundle,
 )
 
 
@@ -80,6 +87,7 @@ class ChatIntentController:
         agent_controller: Any,
         agent_root: Path,
         ingestion_pipeline: Any = None,
+        funnel_metrics: Any = None,
     ) -> None:
         self.router = model_router
         self.config = config
@@ -99,6 +107,16 @@ class ChatIntentController:
         self.agent_controller = agent_controller
         self.agent_root = Path(agent_root)
         self.ingestion_pipeline = ingestion_pipeline
+        self.funnel_metrics = funnel_metrics
+
+    def _funnel_increment(self, name: str) -> None:
+        """UX funnel counter (backlog #16) — advisory, never raises."""
+        if self.funnel_metrics is None:
+            return
+        try:
+            self.funnel_metrics.increment(name)
+        except Exception as exc:  # noqa: BLE001 — metrics must never break chat
+            logging.warning("funnel metrics increment failed: %s", exc)
 
     def no_model_response(self) -> JSONResponse:
         detail = "No model loaded. Call /models/load first."
@@ -235,7 +253,21 @@ class ChatIntentController:
         # An explicit path ("report.txt") wins; otherwise infer one from an
         # explicit type keyword ("html 파일 만들어줘") so weak models never
         # have to drive the JSON tool loop just to create a file.
-        target_path = file_action_target(req.message) or infer_file_target(req.message)
+        explicit_target = file_action_target(req.message)
+        if explicit_target is None:
+            # Multi-file intent ("todo 앱 html+css+js") becomes a real linked
+            # project bundle instead of one inlined page. Single-type requests
+            # return None here and keep the unchanged single-file flow.
+            manifest = infer_project_manifest(req.message)
+            if manifest is not None:
+                return await self.direct_project_action(
+                    req,
+                    manifest,
+                    model_id=model_id,
+                    effective_email=effective_email,
+                    workspace_id=workspace_id,
+                )
+        target_path = explicit_target or infer_file_target(req.message)
         if not target_path:
             return None
         # Never silently overwrite: an existing target gets a _2/_3 suffix so
@@ -316,6 +348,7 @@ class ChatIntentController:
             workspace_id=workspace_id,
             conversation_id=getattr(req, "conversation_id", None),
         )
+        self._funnel_increment("real_file_delivered")
         self.notify("user", req.message, req.source)
         self.notify("assistant", answer, req.source)
         payload = {
@@ -354,6 +387,156 @@ class ChatIntentController:
             )
         return JSONResponse(content=payload)
 
+    async def direct_project_action(
+        self,
+        req: Any,
+        manifest: Dict[str, Any],
+        *,
+        model_id: Optional[str],
+        effective_email: Optional[str] = None,
+        workspace_id: Optional[str] = None,
+    ):
+        """Artifact Loop: manifest → per-file generate/validate → bundle write.
+
+        Every file goes through the same model-agnostic pipeline as the
+        single-file path; after generation the *bundle* is verified as a whole
+        (dangling href/src references are deterministically repaired first),
+        then written together under one project directory, exposed through the
+        standard ``artifacts[]`` contract, and downloadable as a zip.
+        """
+        if not model_id:
+            return self.no_model_response()
+        project_dir = next_available_path(self.agent_root, manifest["name"])
+        bundle_files = [str(f["path"]) for f in manifest["files"]]
+
+        async def _generate(context: str) -> str:
+            return str(
+                await self.router.generate_as(
+                    model_id,
+                    message="Return only the requested file content.",
+                    context=context,
+                    max_tokens=max(int(req.max_tokens or 0), 4096),
+                    temperature=min(req.temperature, 0.3),
+                )
+            )
+
+        contents: Dict[str, str] = {}
+        file_meta: Dict[str, Dict[str, Any]] = {}
+        for spec in manifest["files"]:
+            name = str(spec["path"])
+            request_text = f"{req.message}\n\nThis file's role: {spec.get('brief', '')}"
+            content, meta = await generate_file_content(
+                _generate,
+                target_path=name,
+                user_request=request_text,
+                bundle_files=bundle_files,
+            )
+            contents[name] = content
+            file_meta[name] = meta
+
+        contents, reference_fixes = repair_bundle_references(contents)
+        bundle_validation = validate_project_bundle(contents)
+
+        steps: List[Dict[str, Any]] = []
+        created_files: List[Dict[str, Any]] = []
+        artifacts: List[Dict[str, Any]] = []
+        brain_ingests: List[Dict[str, Any]] = []
+        for name, content in contents.items():
+            rel_path = f"{project_dir}/{name}"
+            try:
+                result = self.execute_tool(
+                    "write_file",
+                    {"path": rel_path, "content": content},
+                )
+            except self.tool_error as exc:
+                raise HTTPException(status_code=400, detail=str(exc)) from exc
+            final_path = result.get("path") or rel_path
+            steps.append({
+                "state": AgentState.EXECUTING.value,
+                "action": "write_file",
+                "args": {"path": rel_path},
+                "result": result,
+            })
+            created_files.append({
+                "path": final_path,
+                "filename": Path(final_path).name,
+                "bytes": result.get("bytes", 0),
+                "action": "write_file",
+            })
+            verdict = bundle_validation["files"].get(name, {})
+            artifacts.append({
+                "kind": "file",
+                "path": final_path,
+                "filename": Path(final_path).name,
+                "bytes": result.get("bytes", 0),
+                "previewable": Path(final_path).suffix.lower() in PREVIEWABLE_EXTENSIONS,
+                "valid": bool(verdict.get("valid", True)),
+                "repaired": bool(file_meta.get(name, {}).get("repaired")),
+            })
+            ingest = self._ingest_generated_file(
+                final_path,
+                content,
+                effective_email=effective_email,
+                workspace_id=workspace_id,
+                conversation_id=getattr(req, "conversation_id", None),
+            )
+            if ingest is not None:
+                brain_ingests.append({"path": final_path, **ingest})
+
+        file_list = ", ".join(Path(a["path"]).name for a in artifacts)
+        answer = (
+            f"{project_dir} 프로젝트를 만들었습니다 "
+            f"({len(artifacts)}개 파일: {file_list})."
+        )
+        if not bundle_validation["ok"]:
+            answer += " 일부 파일은 검증 경고가 있어 결과를 확인해 주세요."
+        repaired_any = any(a["repaired"] for a in artifacts)
+        if repaired_any:
+            answer += " (모델 출력이 불완전한 파일은 자동 보정을 거쳤습니다.)"
+
+        from urllib.parse import quote
+
+        payload: Dict[str, Any] = {
+            "status": "ok",
+            "response": answer,
+            "workspace": str(self.agent_root),
+            "steps": steps,
+            "state_history": [AgentState.EXECUTING.value, AgentState.DONE.value],
+            "final_state": AgentState.DONE.value,
+            "created_files": created_files,
+            "artifacts": artifacts,
+            "routed_to_agent": True,
+            "action_route": "direct_project_bundle",
+            "project": {
+                "dir": project_dir,
+                "files": bundle_files,
+                "zip_url": f"/tools/download_zip?path={quote(project_dir)}",
+                "bundle_validation": bundle_validation,
+                "reference_fixes": reference_fixes,
+            },
+            "generation": {
+                "files": file_meta,
+                "repaired": repaired_any,
+            },
+        }
+        if brain_ingests:
+            payload["brain_ingest"] = brain_ingests
+        self._funnel_increment("real_file_delivered")
+        self.notify("user", req.message, req.source)
+        self.notify("assistant", answer, req.source)
+        if req.stream:
+            return StreamingResponse(
+                agent_payload_stream(
+                    answer,
+                    payload,
+                    router=self.router,
+                    model_id=model_id,
+                ),
+                media_type="text/event-stream",
+                headers={"X-Model": model_id or "tool", "X-Routed-To": "agent"},
+            )
+        return JSONResponse(content=payload)
+
     def _ingest_generated_file(
         self,
         rel_path: str,
@@ -417,22 +600,46 @@ class ChatIntentController:
             user_nickname=req.user_nickname,
             workspace_id=workspace_id,
         )
-        result = await self.agent_controller.agent(agent_request, request)
-        answer = str(result.get("response") or "파일 작업을 처리했습니다.")
-        self.notify("user", req.message, req.source)
-        self.notify("assistant", answer, req.source)
-        result["routed_to_agent"] = True
+
+        def finalize(result: Dict) -> str:
+            """History/funnel side effects once the run has a terminal payload."""
+            answer = str(result.get("response") or "파일 작업을 처리했습니다.")
+            # UX funnel (backlog #16): a file-intent request that the agent loop
+            # finished without producing a single artifact is the "code-only"
+            # failure mode the >95% real-file goal watches. Paused/failed runs
+            # count neither way — they did not answer with code instead of files.
+            delivered = bool(result.get("created_files") or result.get("artifacts"))
+            if delivered:
+                self._funnel_increment("real_file_delivered")
+            elif str(result.get("status") or "") == "ok":
+                self._funnel_increment("code_only_responses")
+            self.notify("user", req.message, req.source)
+            self.notify("assistant", answer, req.source)
+            result["routed_to_agent"] = True
+            return answer
+
         if req.stream:
+            # Live loop visibility (review Wave 1.1): run the agent inside the
+            # SSE generator so step events stream while EXECUTING; the final
+            # frames keep the exact historical payload shape.
+            async def start(observer):
+                return await self.agent_controller.agent(
+                    agent_request, request, on_step=observer
+                )
+
             return StreamingResponse(
-                agent_payload_stream(
-                    answer,
-                    result,
+                agent_live_stream(
+                    start,
                     router=self.router,
                     model_id=model_id,
+                    finalize=finalize,
                 ),
                 media_type="text/event-stream",
                 headers={"X-Model": model_id or "agent", "X-Routed-To": "agent"},
             )
+
+        result = await self.agent_controller.agent(agent_request, request)
+        finalize(result)
         return JSONResponse(content=result)
 
 

package/latticeai/api/chat_stream.py

@@ -2,13 +2,14 @@
 
 from __future__ import annotations
 
+import asyncio
 import json
 import logging
-from typing import Any, AsyncIterator, Dict, Optional
+from typing import Any, AsyncIterator, Awaitable, Callable, Dict, Optional
 
 from fastapi.responses import JSONResponse, StreamingResponse
 
-from latticeai.api.chat_helpers import single_text_stream
+from latticeai.api.chat_helpers import assess_answer_grounding, single_text_stream
 
 
 def single_answer_response(req: Any, answer: str, *, model: str):
@@ -37,6 +38,74 @@ def agent_payload_stream(
     return _stream()
 
 
+def agent_live_stream(
+    start: Callable[[Callable[[Dict[str, Any]], None]], Awaitable[Dict[str, Any]]],
+    *,
+    router: Any,
+    model_id: Optional[str] = None,
+    finalize: Optional[Callable[[Dict[str, Any]], str]] = None,
+) -> AsyncIterator[str]:
+    """Live SSE for an agent run (review Wave 1.1).
+
+    ``start(observer)`` runs the agent with a per-run step observer; every
+    observed step is emitted immediately as a named ``agent_step`` frame, so
+    the user watches the loop work instead of staring at silence. When the
+    run finishes, ``finalize(result)`` (history/funnel side effects) produces
+    the answer text and the classic final payload frames follow — clients
+    that ignore named events see exactly the historical stream shape.
+    """
+
+    async def _stream() -> AsyncIterator[str]:
+        response_model = model_id or router.current_model_id
+        queue: asyncio.Queue = asyncio.Queue()
+
+        def observer(event: Dict[str, Any]) -> None:
+            # Called synchronously from the agent coroutine on this loop.
+            queue.put_nowait(event)
+
+        task = asyncio.create_task(start(observer))
+        try:
+            while True:
+                getter = asyncio.create_task(queue.get())
+                done, _ = await asyncio.wait(
+                    {getter, task}, return_when=asyncio.FIRST_COMPLETED
+                )
+                if getter in done:
+                    step = getter.result()
+                    yield (
+                        "event: agent_step\n"
+                        f"data: {json.dumps(step, ensure_ascii=False)}\n\n"
+                    )
+                    continue
+                getter.cancel()
+                break
+            while not queue.empty():
+                step = queue.get_nowait()
+                yield (
+                    "event: agent_step\n"
+                    f"data: {json.dumps(step, ensure_ascii=False)}\n\n"
+                )
+            result = task.result()
+        except Exception as exc:  # noqa: BLE001 — already streaming a 200
+            logging.warning("agent live stream failed: %s", exc)
+            detail = getattr(exc, "detail", None) or str(exc)
+            yield f"data: {json.dumps({'error': str(detail), 'model': response_model}, ensure_ascii=False)}\n\n"
+            yield "data: [DONE]\n\n"
+            return
+
+        answer = str(result.get("response") or "작업을 완료했습니다.")
+        if finalize is not None:
+            try:
+                answer = finalize(result)
+            except Exception as exc:  # noqa: BLE001 — side effects must not kill the stream
+                logging.warning("agent live stream finalize failed: %s", exc)
+        yield f"data: {json.dumps({'chunk': answer, 'model': response_model, 'agent': result}, ensure_ascii=False)}\n\n"
+        yield f"data: {json.dumps({'chunk': '', 'model': response_model, 'agent': result}, ensure_ascii=False)}\n\n"
+        yield "data: [DONE]\n\n"
+
+    return _stream()
+
+
 async def stream_chat(
     req: Any,
     context: str,
@@ -87,6 +156,7 @@ async def stream_chat(
         persisted_response = f"{persisted_response}\n\n[stream_error] {stream_error}"
 
     trace_record = None
+    grounding = None
     try:
         answer_trace = trace_seed or chat_service.build_graph_trace(
             req.message,
@@ -94,6 +164,19 @@ async def stream_chat(
             context,
             allowed_workspaces={workspace_id} if workspace_id else None,
         )
+        # Answer-citation binding (backlog #11): same honest verdict as the
+        # non-streaming path, recorded on the persisted trace + trailer.
+        try:
+            grounding = assess_answer_grounding(
+                full_response,
+                trace=answer_trace if isinstance(answer_trace, dict) else None,
+                context_quality=context_quality,
+            )
+            if isinstance(answer_trace, dict):
+                answer_trace["grounding"] = grounding
+        except Exception as exc:  # noqa: BLE001 — annotation must never break streaming
+            logging.warning("answer grounding assessment failed: %s", exc)
+            grounding = None
         trace_record = await chat_service.persist_answer(
             question=req.message,
             response=persisted_response,
@@ -114,10 +197,17 @@ async def stream_chat(
         trailer.update({"trace_id": trace_record["id"], "trace": trace_record})
     if context_quality is not None:
         trailer["context_quality"] = context_quality
+    if grounding is not None:
+        trailer["grounding"] = grounding
     if stream_error:
         trailer["error"] = stream_error
     yield f"data: {json.dumps(trailer, ensure_ascii=False)}\n\n"
     yield "data: [DONE]\n\n"
 
 
-__all__ = ["agent_payload_stream", "single_answer_response", "stream_chat"]
+__all__ = [
+    "agent_live_stream",
+    "agent_payload_stream",
+    "single_answer_response",
+    "stream_chat",
+]

package/latticeai/api/funnel_metrics.py

@@ -0,0 +1,31 @@
+"""Admin-only UX funnel metrics endpoint (backlog #16).
+
+Exposes :class:`~latticeai.services.funnel_metrics.FunnelMetricsService`
+snapshots at ``GET /api/admin/funnel-metrics``. Follows the admin router
+convention: gate through the injected ``require_admin`` callable, read-only,
+no side effects.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable
+
+from fastapi import APIRouter, Request
+
+
+def create_funnel_metrics_router(
+    *,
+    service: Any,
+    require_admin: Callable[[Request], Any],
+) -> APIRouter:
+    router = APIRouter()
+
+    @router.get("/api/admin/funnel-metrics")
+    async def funnel_metrics(request: Request):
+        require_admin(request)
+        return service.snapshot()
+
+    return router
+
+
+__all__ = ["create_funnel_metrics_router"]

package/latticeai/api/knowledge_graph.py

@@ -7,7 +7,7 @@ data endpoints back the v3 SPA (Files, Hybrid Search, graph explorer).
 """
 
 from pathlib import Path
-from typing import Any, Callable, Dict, Optional
+from typing import Any, Callable, Dict, List, Optional
 
 from fastapi import APIRouter, HTTPException, Request
 from pydantic import BaseModel
@@ -28,6 +28,21 @@ class KnowledgeGraphIngestRequest(BaseModel):
     metadata: Optional[Dict[str, Any]] = None
 
 
+class CurateNoiseRequest(BaseModel):
+    # Dry-run by default: the job *reports* removals until explicitly applied.
+    dry_run: bool = True
+    max_df_ratio: float = 0.8
+    min_doc_frequency: int = 1
+    min_corpus_docs: int = 5
+    normalize_verbs: bool = True
+    max_removals: int = 200
+
+
+class PromotionActionRequest(BaseModel):
+    # None applies/rejects every pending promotion; otherwise only these ids.
+    ids: Optional[List[str]] = None
+
+
 def _workspace_scope_from_request(request: Request) -> Optional[str]:
     header = request.headers.get("X-Workspace-Id")
     if header and header.strip():
@@ -143,6 +158,55 @@ def create_knowledge_graph_router(
         (require_admin or require_user)(request)
         return graph().curate()
 
+    @router.post("/knowledge-graph/curate/noise")
+    async def knowledge_graph_curate_noise(req: CurateNoiseRequest, request: Request):
+        """Noise-reduction curation job (backlog #10).
+
+        Dry-run by default: reports which heuristic concept nodes would be
+        removed (low IDF / below the frequency floor) and which relation
+        verbs would be normalized, without changing the graph. Set
+        ``dry_run=false`` to apply. User-created nodes are never removed.
+        Administrative like ``/knowledge-graph/curate`` when roles exist.
+        """
+        (require_admin or require_user)(request)
+        return graph().curate_noise(
+            dry_run=req.dry_run,
+            max_df_ratio=req.max_df_ratio,
+            min_doc_frequency=req.min_doc_frequency,
+            min_corpus_docs=req.min_corpus_docs,
+            normalize_verbs=req.normalize_verbs,
+            max_removals=req.max_removals,
+        )
+
+    @router.get("/knowledge-graph/promotions")
+    async def knowledge_graph_promotions(request: Request):
+        """Pending curator promotions awaiting human review (review Wave 4).
+
+        Populated when ``curate()`` runs in review mode (explicit
+        ``review_mode=True`` or the LATTICEAI_GRAPH_PROMOTION_REVIEW env
+        opt-in). Administrative like ``/knowledge-graph/curate``: the queue
+        governs the shared graph.
+        """
+        (require_admin or require_user)(request)
+        pending = graph().pending_promotions()
+        return {"pending": pending, "total": len(pending)}
+
+    @router.post("/knowledge-graph/promotions/apply")
+    async def knowledge_graph_promotions_apply(
+        req: PromotionActionRequest, request: Request
+    ):
+        """Apply pending promotions (all when ``ids`` is omitted)."""
+        (require_admin or require_user)(request)
+        return graph().apply_pending_promotions(ids=req.ids)
+
+    @router.post("/knowledge-graph/promotions/reject")
+    async def knowledge_graph_promotions_reject(
+        req: PromotionActionRequest, request: Request
+    ):
+        """Reject pending promotions without writing (all when ``ids`` omitted)."""
+        (require_admin or require_user)(request)
+        return graph().reject_pending_promotions(ids=req.ids)
+
     @router.get("/knowledge-graph/provenance/coverage")
     async def knowledge_graph_provenance_coverage(request: Request):
         require_user(request)

package/latticeai/api/local_files.py

@@ -49,6 +49,16 @@ class FolderIngestRequest(BaseModel):
     approval_token: Optional[str] = None
 
 
+class FolderWatchEnableRequest(BaseModel):
+    path: str
+    recursive: bool = True
+    workspace_id: Optional[str] = None
+    # Watching continuously reads local disk → same approval dance as
+    # /api/ingestion/folder. This is the explicit opt-in the review requires.
+    approved: bool = False
+    approval_token: Optional[str] = None
+
+
 def create_local_files_router(
     *,
     require_user,
@@ -64,9 +74,26 @@ def create_local_files_router(
     data_dir: Optional[Path] = None,
     allowed_workspaces_for=None,
     workspace_service=None,
+    folder_watch=None,
 ) -> APIRouter:
     router = APIRouter()
 
+    # ── opt-in folder watch service (backlog #8) ──────────────────────────────
+    # Constructed here (not injected) unless a test provides one; restore()
+    # only resumes watches persisted with the explicit opt-in — a fresh
+    # data_dir never starts a polling thread.
+    if folder_watch is None and ingestion_pipeline is not None and data_dir is not None:
+        try:
+            from latticeai.services.folder_watch import FolderWatchService
+
+            folder_watch = FolderWatchService(
+                pipeline=ingestion_pipeline,
+                config_path=Path(data_dir) / "folder_watch.json",
+            )
+            folder_watch.restore()
+        except Exception:  # noqa: BLE001 — watch mode is optional, never blocks routes
+            folder_watch = None
+
     @router.get("/api/local-agent/status")
     async def local_agent_status(request: Request):
         """Real on-device runtime status — the 'Local Agent' is the Lattice
@@ -307,6 +334,68 @@ def create_local_files_router(
             )
         return summary
 
+    # ── folder watch mode: opt-in, off by default (backlog #8) ────────────────
+    def _require_folder_watch():
+        _require_pipeline()
+        if folder_watch is None:
+            raise HTTPException(status_code=503, detail="Folder watch service is unavailable.")
+
+    @router.get("/api/ingestion/watch")
+    async def folder_watch_status(request: Request):
+        """Watch-mode status: stored opt-ins, poller state, last scan results."""
+        require_user(request)
+        _require_folder_watch()
+        return folder_watch.status()
+
+    @router.post("/api/ingestion/watch")
+    async def folder_watch_enable(req: FolderWatchEnableRequest, request: Request):
+        """Explicitly opt a previously-ingested folder into watch mode.
+
+        Follows the same local-read approval dance as ``/api/ingestion/folder``.
+        Enabling snapshots the folder as the baseline; only *future* new or
+        changed files are re-ingested (through the normal pipeline, with the
+        watch owner's workspace scope).
+        """
+        current_user = permission_gateway.require_local_user(request)
+        _require_folder_watch()
+        workspace_id = _ingestion_write_workspace(request, req.workspace_id, current_user)
+        path = (req.path or "").strip()
+        if not path:
+            raise HTTPException(status_code=400, detail="path is required.")
+        if not req.approved:
+            return permission_gateway.local_permission_response(path, "read", current_user)
+        permission_gateway.require_local_approval(
+            token=req.approval_token,
+            path=path,
+            action="read",
+            user_email=current_user,
+        )
+        result = folder_watch.enable(
+            path,
+            owner=current_user or None,
+            workspace_id=workspace_id,
+            recursive=req.recursive,
+        )
+        if result.get("status") != "ok":
+            raise HTTPException(status_code=400, detail=result.get("detail") or "watch enable failed")
+        return result
+
+    @router.delete("/api/ingestion/watch")
+    async def folder_watch_disable(
+        request: Request,
+        watch_id: Optional[str] = None,
+        path: Optional[str] = None,
+    ):
+        """Opt back out of watch mode (removes the stored consent record)."""
+        require_user(request)
+        _require_folder_watch()
+        if not watch_id and not path:
+            raise HTTPException(status_code=400, detail="watch_id or path is required.")
+        result = folder_watch.disable(watch_id=watch_id, path=path)
+        if result.get("status") == "not_found":
+            raise HTTPException(status_code=404, detail="watch not found")
+        return result
+
     @router.post("/local/write")
     async def local_write_endpoint(req: LocalWriteRequest, request: Request):
         current_user = permission_gateway.require_local_user(request)
@@ -353,6 +442,7 @@ def create_local_files_router(
 
 __all__ = [
     "FolderIngestRequest",
+    "FolderWatchEnableRequest",
     "LocalAccessRequest",
     "LocalWriteRequest",
     "create_local_files_router",