ltcai 1.3.0 → 1.5.0

package/latticeai/api/computer_use.py

@@ -0,0 +1,294 @@
+"""Computer-use routes and the desktop-control agent loop."""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Optional
+
+from fastapi import APIRouter, HTTPException, Request
+from fastapi.responses import StreamingResponse
+from pydantic import BaseModel
+
+from latticeai.core.agent import extract_action as _extract_agent_action
+from tools import (
+    ToolError,
+    computer_click,
+    computer_drag,
+    computer_key,
+    computer_move,
+    computer_open_app,
+    computer_open_url,
+    computer_screenshot,
+    computer_scroll,
+    computer_status,
+    computer_type,
+    desktop_bridge_status,
+    execute_tool,
+)
+
+
+CU_SYSTEM_PROMPT = """You are Lattice AI desktop-control agent. You control the Mac desktop using tools.
+Prefer non-visual direct actions when possible. Use screenshots only when you must inspect visible UI state or choose screen coordinates.
+
+Available actions:
+- computer_screenshot: {"action":"computer_screenshot","args":{}} — capture screen, returns screenshot_b64
+- computer_open_app: {"action":"computer_open_app","args":{"app":"Google Chrome"}} — open or focus a Mac app
+- computer_open_url: {"action":"computer_open_url","args":{"url":"https://example.com","app":"Google Chrome"}} — open URL in app
+- computer_click: {"action":"computer_click","args":{"x":500,"y":300,"button":"left","double":false}}
+- computer_type: {"action":"computer_type","args":{"text":"hello world","interval":0.04}}
+- computer_key: {"action":"computer_key","args":{"key":"return"}} — keys: return, escape, tab, space, command+c, etc.
+- computer_scroll: {"action":"computer_scroll","args":{"x":500,"y":300,"direction":"down","clicks":3}}
+- computer_move: {"action":"computer_move","args":{"x":500,"y":300}}
+- computer_drag: {"action":"computer_drag","args":{"x1":100,"y1":100,"x2":500,"y2":500}}
+- final: {"action":"final","message":"Korean summary of what was accomplished"}
+
+Rules:
+- Respond with exactly ONE JSON object. No markdown, no extra text.
+- Do not take screenshots for simple app launch, URL opening, keyboard shortcuts, or non-visual tasks.
+- Take a screenshot before coordinate-based clicks/drags or when the task explicitly asks you to inspect the screen.
+- After coordinate-based clicking or typing into an unknown focused field, take a screenshot only if verification is necessary.
+- Use coordinates relative to the screen (0,0 is top-left).
+- If a UI element is not visible, scroll or search for it first.
+- macOS Accessibility permission required for mouse/keyboard control.
+"""
+
+
+class CuAgentRequest(BaseModel):
+    task: str
+    conversation_id: Optional[str] = None
+    max_steps: int = 15
+    temperature: float = 0.1
+
+
+class CuClickRequest(BaseModel):
+    x: int
+    y: int
+    button: str = "left"
+    double: bool = False
+
+
+class CuOpenAppRequest(BaseModel):
+    app: str = "Google Chrome"
+
+
+class CuOpenUrlRequest(BaseModel):
+    url: str
+    app: str = "Google Chrome"
+
+
+class CuTypeRequest(BaseModel):
+    text: str
+    interval: float = 0.04
+
+
+class CuKeyRequest(BaseModel):
+    key: str
+
+
+class CuScrollRequest(BaseModel):
+    x: int
+    y: int
+    direction: str = "down"
+    clicks: int = 3
+
+
+class CuMoveRequest(BaseModel):
+    x: int
+    y: int
+
+
+class CuDragRequest(BaseModel):
+    x1: int
+    y1: int
+    x2: int
+    y2: int
+
+
+def create_computer_use_router(*, model_router, require_user, tool_response, save_to_history) -> APIRouter:
+    router = APIRouter()
+
+    @router.get("/tools/chrome_status")
+    async def tools_chrome_status(request: Request):
+        require_user(request)
+        return tool_response(desktop_bridge_status)
+
+    @router.get("/tools/computer_use_status")
+    async def tools_computer_use_status(request: Request):
+        require_user(request)
+        return tool_response(computer_status)
+
+    @router.get("/cu/status")
+    async def cu_status(request: Request):
+        require_user(request)
+        try:
+            return computer_status()
+        except ToolError as exc:
+            raise HTTPException(status_code=400, detail=str(exc))
+
+    @router.get("/cu/screenshot")
+    async def cu_screenshot(request: Request):
+        require_user(request)
+        try:
+            return computer_screenshot()
+        except ToolError as exc:
+            raise HTTPException(status_code=400, detail=str(exc))
+
+    @router.post("/cu/open_app")
+    async def cu_open_app(req: CuOpenAppRequest, request: Request):
+        require_user(request)
+        return tool_response(computer_open_app, req.app)
+
+    @router.post("/cu/open_url")
+    async def cu_open_url(req: CuOpenUrlRequest, request: Request):
+        require_user(request)
+        return tool_response(computer_open_url, req.url, req.app)
+
+    @router.post("/cu/click")
+    async def cu_click(req: CuClickRequest, request: Request):
+        require_user(request)
+        return tool_response(computer_click, req.x, req.y, req.button, req.double)
+
+    @router.post("/cu/type")
+    async def cu_type(req: CuTypeRequest, request: Request):
+        require_user(request)
+        return tool_response(computer_type, req.text, req.interval)
+
+    @router.post("/cu/key")
+    async def cu_key(req: CuKeyRequest, request: Request):
+        require_user(request)
+        return tool_response(computer_key, req.key)
+
+    @router.post("/cu/scroll")
+    async def cu_scroll(req: CuScrollRequest, request: Request):
+        require_user(request)
+        return tool_response(computer_scroll, req.x, req.y, req.direction, req.clicks)
+
+    @router.post("/cu/move")
+    async def cu_move(req: CuMoveRequest, request: Request):
+        require_user(request)
+        return tool_response(computer_move, req.x, req.y)
+
+    @router.post("/cu/drag")
+    async def cu_drag(req: CuDragRequest, request: Request):
+        require_user(request)
+        return tool_response(computer_drag, req.x1, req.y1, req.x2, req.y2)
+
+    @router.post("/cu/agent")
+    async def cu_agent(req: CuAgentRequest, request: Request):
+        require_user(request)
+
+        async def _stream():
+            task_lower = (req.task or "").lower()
+            url_match = re.search(r"(https?://[^\s]+|localhost:\d+[^\s]*|127\.0\.0\.1:\d+[^\s]*)", req.task or "")
+
+            def _send(event: str, data: dict) -> str:
+                return f"event: {event}\ndata: {json.dumps(data, ensure_ascii=False)}\n\n"
+
+            if ("chrome" in task_lower or "크롬" in task_lower) and any(
+                word in task_lower for word in ["open", "열", "켜", "실행", "띄"]
+            ):
+                yield _send("start", {"task": req.task, "max_steps": 1})
+                try:
+                    if url_match:
+                        url = url_match.group(1)
+                        yield _send(
+                            "action",
+                            {"step": 1, "action": "computer_open_url", "args": {"url": url, "app": "Google Chrome"}},
+                        )
+                        result = computer_open_url(url, "Google Chrome")
+                        yield _send("result", {"step": 1, "action": "computer_open_url", "result": result})
+                        message = f"Google Chrome에서 {url}을 열었습니다."
+                        action_name = "computer_open_url"
+                    else:
+                        yield _send(
+                            "action",
+                            {"step": 1, "action": "computer_open_app", "args": {"app": "Google Chrome"}},
+                        )
+                        result = computer_open_app("Google Chrome")
+                        yield _send("result", {"step": 1, "action": "computer_open_app", "result": result})
+                        message = "Google Chrome을 열었습니다."
+                        action_name = "computer_open_app"
+                    save_to_history("user", req.task, source="web", conversation_id=req.conversation_id)
+                    save_to_history("assistant", message, source="web", conversation_id=req.conversation_id)
+                    yield _send("final", {"message": message, "steps": [{"step": 1, "action": action_name, "result": result}]})
+                except ToolError as exc:
+                    yield _send("tool_error", {"step": 1, "action": "computer_open_app", "error": str(exc)})
+                return
+
+            if not model_router.current_model_id:
+                yield _send("error", {"error": "No model loaded."})
+                return
+
+            transcript = []
+            last_screenshot_b64: Optional[str] = None
+            max_steps = max(1, min(req.max_steps, 20))
+            yield _send("start", {"task": req.task, "max_steps": max_steps})
+
+            for step in range(max_steps):
+                context = (
+                    f"{CU_SYSTEM_PROMPT}\n\n"
+                    f"Task: {req.task}\n\n"
+                    f"Steps completed so far:\n{json.dumps(transcript, ensure_ascii=False, indent=2)}"
+                )
+                raw = await model_router.generate(
+                    message="Choose the next computer use action.",
+                    context=context,
+                    image_data=last_screenshot_b64,
+                    max_tokens=1024,
+                    temperature=req.temperature,
+                )
+
+                try:
+                    action = _extract_agent_action(str(raw))
+                except ValueError as exc:
+                    yield _send("error", {"step": step + 1, "error": str(exc), "raw": str(raw)})
+                    break
+
+                name = action.get("action")
+                args = action.get("args") or {}
+                if name == "final":
+                    message = action.get("message", "작업을 완료했습니다.")
+                    save_to_history("user", req.task, source="web", conversation_id=req.conversation_id)
+                    save_to_history("assistant", message, source="web", conversation_id=req.conversation_id)
+                    yield _send("final", {"message": message, "steps": transcript})
+                    return
+
+                yield _send("action", {"step": step + 1, "action": name, "args": args})
+                try:
+                    result = execute_tool(name, args)
+                    if name == "computer_screenshot" and "screenshot_b64" in result:
+                        last_screenshot_b64 = result["screenshot_b64"]
+                        result_summary = {k: v for k, v in result.items() if k != "screenshot_b64"}
+                        result_summary["screenshot_captured"] = True
+                        transcript.append({"step": step + 1, "action": name, "args": args, "result": result_summary})
+                        yield _send(
+                            "screenshot",
+                            {
+                                "step": step + 1,
+                                "screenshot_b64": last_screenshot_b64,
+                                "width": result.get("screen_width"),
+                                "height": result.get("screen_height"),
+                            },
+                        )
+                    else:
+                        last_screenshot_b64 = None
+                        transcript.append({"step": step + 1, "action": name, "args": args, "result": result})
+                        yield _send("result", {"step": step + 1, "action": name, "result": result})
+                except (ToolError, KeyError, TypeError) as exc:
+                    error_str = str(exc)
+                    transcript.append({"step": step + 1, "action": name, "args": args, "error": error_str})
+                    yield _send("tool_error", {"step": step + 1, "action": name, "error": error_str})
+
+            yield _send("done", {"steps": len(transcript), "transcript": transcript})
+
+        return StreamingResponse(
+            _stream(),
+            media_type="text/event-stream",
+            headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
+        )
+
+    return router
+
+
+__all__ = ["create_computer_use_router"]

package/latticeai/api/deps.py

@@ -0,0 +1,15 @@
+"""Shared API dependency type aliases.
+
+Routers receive concrete callables from ``server_app`` at assembly time. Keeping
+the aliases here avoids app imports inside router modules and gives future
+router splits a single dependency vocabulary.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Dict
+
+RequireUser = Callable[[Any], str]
+RequireAdmin = Callable[[Any], tuple[str, Dict]]
+AuditAppender = Callable[..., None]
+

package/latticeai/api/garden.py

@@ -0,0 +1,34 @@
+"""P-Reinforce garden routes."""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from fastapi import APIRouter, Request
+from pydantic import BaseModel
+
+
+class GardenRequest(BaseModel):
+    raw_data: str
+    category: Optional[str] = None
+
+
+def create_garden_router(*, gardener, require_user) -> APIRouter:
+    api_router = APIRouter()
+
+    # ── P-Reinforce Knowledge Gardener ────────────────────────────────────────────
+    
+    @api_router.post("/garden")
+    async def garden(req: GardenRequest, request: Request):
+        """Raw 데이터를 P-Reinforce 구조로 자동 분류·저장"""
+        require_user(request)
+        result = await gardener.process(req.raw_data, req.category)
+        return result
+    
+    
+    @api_router.get("/garden/tree")
+    async def garden_tree(request: Request):
+        """지식 정원 파일트리 반환"""
+        require_user(request)
+        return gardener.get_tree()
+    return api_router

package/latticeai/api/local_files.py

@@ -0,0 +1,125 @@
+"""Local file access and local knowledge graph routes."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+from fastapi import APIRouter, HTTPException, Request
+from fastapi.responses import FileResponse
+from pydantic import BaseModel
+
+from knowledge_graph_api import create_knowledge_graph_router
+from local_knowledge_api import create_local_knowledge_router
+from tools import local_list, local_read, local_write
+
+
+class LocalAccessRequest(BaseModel):
+    path: str
+    approved: bool = False
+    approval_token: Optional[str] = None
+
+
+class LocalWriteRequest(BaseModel):
+    path: str
+    content: str
+    approved: bool = False
+    approval_token: Optional[str] = None
+
+
+def create_local_files_router(
+    *,
+    require_user,
+    tool_response,
+    permission_gateway,
+    knowledge_graph,
+    require_graph,
+    static_dir: Path,
+    local_kg_watcher,
+) -> APIRouter:
+    router = APIRouter()
+
+    @router.post("/local/list")
+    async def local_list_endpoint(req: LocalAccessRequest, request: Request):
+        current_user = permission_gateway.require_local_user(request)
+        if not req.approved:
+            return permission_gateway.local_permission_response(req.path, "list", current_user)
+        permission_gateway.require_local_approval(
+            token=req.approval_token,
+            path=req.path,
+            action="list",
+            user_email=current_user,
+        )
+        return tool_response(local_list, req.path)
+
+    @router.get("/local/list")
+    async def local_list_get_endpoint(path: str, request: Request):
+        current_user = permission_gateway.require_local_user(request)
+        return permission_gateway.local_permission_response(path, "list", current_user)
+
+    @router.post("/local/read")
+    async def local_read_endpoint(req: LocalAccessRequest, request: Request):
+        current_user = permission_gateway.require_local_user(request)
+        if not req.approved:
+            return permission_gateway.local_permission_response(req.path, "read", current_user)
+        permission_gateway.require_local_approval(
+            token=req.approval_token,
+            path=req.path,
+            action="read",
+            user_email=current_user,
+        )
+        return tool_response(local_read, req.path)
+
+    @router.get("/local/serve")
+    async def local_serve_file(path: str, request: Request, approval_token: Optional[str] = None):
+        current_user = permission_gateway.require_local_user(request)
+        permission_gateway.require_local_approval(
+            token=approval_token,
+            path=path,
+            action="read",
+            user_email=current_user,
+        )
+        target = Path(path).expanduser().resolve()
+        if not target.exists() or not target.is_file():
+            raise HTTPException(status_code=404, detail="File not found")
+        return FileResponse(str(target))
+
+    @router.post("/local/write")
+    async def local_write_endpoint(req: LocalWriteRequest, request: Request):
+        current_user = permission_gateway.require_local_user(request)
+        if not req.approved:
+            return permission_gateway.local_permission_response(req.path, "write", current_user, req.content)
+        permission_gateway.require_local_approval(
+            token=req.approval_token,
+            path=req.path,
+            action="write",
+            user_email=current_user,
+            content=req.content,
+        )
+        return tool_response(local_write, req.path, req.content)
+
+    router.include_router(
+        create_knowledge_graph_router(
+            get_graph=lambda: knowledge_graph,
+            require_graph=require_graph,
+            require_user=require_user,
+            static_dir=static_dir,
+        )
+    )
+
+    router.include_router(
+        create_local_knowledge_router(
+            get_graph=lambda: knowledge_graph,
+            require_graph=require_graph,
+            require_user=require_user,
+            require_local_user=permission_gateway.require_local_user,
+            local_permission_response=permission_gateway.local_permission_response,
+            require_local_approval=permission_gateway.require_local_approval,
+            watcher=local_kg_watcher,
+        )
+    )
+
+    return router
+
+
+__all__ = ["LocalAccessRequest", "LocalWriteRequest", "create_local_files_router"]

package/latticeai/api/models.py

@@ -304,4 +304,20 @@ def create_models_router(
         _router.unload_all()
         return {"status": "ok", "unloaded": unloaded}
 
+    @router.get("/models/recommendations")
+    async def model_recommendations(request: Request, engine: str = "local_mlx"):
+        """Hardware-aware tri-state model recommendation for this machine.
+
+        Detects the system profile (OS/RAM/CPU/GPU/disk) and classifies the
+        ``engine`` catalog into recommended / compatible / not_recommended,
+        grouped by family. Used by the onboarding and model-picker UIs.
+        """
+        require_user(request)
+        from auto_setup import probe as auto_setup_probe
+        from latticeai.services.model_recommendation import recommend_catalog
+
+        profile = await asyncio.to_thread(lambda: auto_setup_probe().to_json())
+        catalog = recommend_catalog(profile, engine=engine)
+        return {"profile": profile, "recommendations": catalog}
+
     return router