flowent 0.1.2 → 0.1.4

package/backend/src/flowent/main.py

@@ -4,11 +4,12 @@ import logging
 import os
 from collections.abc import AsyncIterator
 from contextlib import asynccontextmanager
+from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Literal
 from uuid import uuid4
 
-from fastapi import FastAPI, HTTPException
+from fastapi import FastAPI, HTTPException, Query
 from fastapi.responses import FileResponse, StreamingResponse
 from fastapi.staticfiles import StaticFiles
 from pydantic import BaseModel, ConfigDict
@@ -16,18 +17,24 @@ from pydantic import BaseModel, ConfigDict
 from flowent._version import __version__
 from flowent.agent import run_agent_stream
 from flowent.channels import TelegramBotManager, TelegramTransport
+from flowent.compact import (
+    CompactInput,
+    LocalSummaryCompactProvider,
+    transcript_messages_after,
+)
 from flowent.context import runtime_context_messages
 from flowent.llm import (
     ChatMessage,
     CompletionCallable,
     ProviderConnection,
     ProviderFormat,
-    complete_chat,
     list_provider_models,
 )
 from flowent.logging import TRACE_LEVEL, ensure_logging_configured
 from flowent.mcp import McpManager, McpTransport
 from flowent.mcp_import import McpImportDiscovery, discover_imported_mcp_servers
+from flowent.paths import resolve_workdir
+from flowent.permissions import WritablePathDecision, run_tool_with_path_permissions
 from flowent.sandbox import ensure_sandbox_available
 from flowent.skills import (
     discover_skills,
@@ -36,8 +43,10 @@ from flowent.skills import (
 )
 from flowent.storage import (
     StateStore,
+    StoredCompactionCheckpoint,
     StoredMcpServer,
     StoredMessage,
+    StoredPermissionRequest,
     StoredProvider,
     StoredSettings,
     StoredSkill,
@@ -45,14 +54,15 @@ from flowent.storage import (
     StoredTelegramBot,
     StoredTelegramSession,
     StoredToolItem,
+    StoredWritablePath,
 )
+from flowent.tools import ToolContext
 
 logger = logging.getLogger("flowent.main")
 
 
 DEFAULT_STATIC_DIR = Path(__file__).parent / "static"
 COMPACTED_CONTEXT_MARKER = "Context compacted"
-COMPACT_SYSTEM_PROMPT = "You are compacting Flowent workspace context."
 
 
 class ProviderModelsRequest(BaseModel):
@@ -79,6 +89,12 @@ class WorkspaceRespondRequest(BaseModel):
     content: str
 
 
+class WorkspaceRunResponse(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    run_id: str
+
+
 class WorkspaceCompactResponse(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
@@ -116,6 +132,61 @@ class McpImportPreviewRequest(BaseModel):
     source: Literal["claude_code", "codex"]
 
 
+class WritablePathRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    path: str
+
+
+class WritablePathListResponse(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    writable_paths: list[StoredWritablePath]
+
+
+class WorkspacePermissionDecisionRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    decision: Literal["allow_once", "always_allow", "deny"]
+    id: str
+
+
+@dataclass
+class PendingWorkspacePermission:
+    future: asyncio.Future[WritablePathDecision]
+    path: Path
+    reason: str
+    tool_call_id: str | None = None
+
+
+@dataclass
+class WorkspaceRun:
+    condition: asyncio.Condition
+    discard_on_cancel: bool = False
+    events: list[tuple[int, str, dict[str, object]]] = field(default_factory=list)
+    id: str = field(default_factory=lambda: str(uuid4()))
+    is_done: bool = False
+    pending_permissions: dict[str, PendingWorkspacePermission] = field(
+        default_factory=dict
+    )
+    task: asyncio.Task[None] | None = None
+
+    @property
+    def latest_event_index(self) -> int:
+        return self.events[-1][0] if self.events else 0
+
+    def permission_requests(self) -> list[StoredPermissionRequest]:
+        return [
+            StoredPermissionRequest(
+                id=permission_id,
+                path=str(permission.path),
+                reason=permission.reason,
+                tool_call_id=permission.tool_call_id,
+            )
+            for permission_id, permission in self.pending_permissions.items()
+        ]
+
+
 def stream_event(event: str, data: dict[str, object]) -> str:
     return f"event: {event}\ndata: {json.dumps(data)}\n\n"
 
@@ -184,8 +255,34 @@ def latest_compacted_context_index(messages: list[StoredMessage]) -> int | None:
 def workspace_chat_messages(
     messages: list[StoredMessage],
     compacted_context: str = "",
+    checkpoint: StoredCompactionCheckpoint | None = None,
 ) -> list[ChatMessage]:
     chat_messages: list[ChatMessage] = []
+
+    if checkpoint is not None:
+        chat_messages.extend(checkpoint.replacement_history)
+        visible_messages = transcript_messages_after(
+            messages,
+            checkpoint.source_message_id,
+        )
+        for message in visible_messages:
+            if (
+                message.author == "system"
+                and message.content == COMPACTED_CONTEXT_MARKER
+            ):
+                continue
+            if message.author not in ("user", "assistant"):
+                raise HTTPException(
+                    status_code=400, detail="Message history is invalid."
+                )
+            checkpoint_role: Literal["user", "assistant"] = (
+                "user" if message.author == "user" else "assistant"
+            )
+            chat_messages.append(
+                ChatMessage(role=checkpoint_role, content=message.content)
+            )
+        return chat_messages
+
     marker_index = latest_compacted_context_index(messages)
     visible_messages = messages
 
@@ -210,30 +307,11 @@ def workspace_chat_messages(
     return chat_messages
 
 
-def compact_prompt_messages(
-    messages: list[StoredMessage],
-    compacted_context: str,
-    runtime_messages: list[ChatMessage] | None = None,
-) -> list[ChatMessage]:
-    history_messages = [
-        *(runtime_messages or []),
-        *workspace_chat_messages(messages, compacted_context),
-    ]
-    history = "\n\n".join(
-        f"{message.role}: {message.content}" for message in history_messages
-    )
-    return [
-        ChatMessage(role="system", content=COMPACT_SYSTEM_PROMPT),
-        ChatMessage(
-            role="user",
-            content=(
-                "Compact the current Flowent workspace context for the next turn.\n\n"
-                "Keep the details needed to continue accurately, including decisions, "
-                "constraints, pending work, and referenced facts.\n\n"
-                f"Conversation:\n{history}"
-            ),
-        ),
-    ]
+def normalized_request_path(path: str, cwd: Path) -> Path:
+    raw_path = Path(path).expanduser()
+    if not raw_path.is_absolute():
+        raw_path = cwd / raw_path
+    return raw_path.resolve(strict=False)
 
 
 def create_app(
@@ -242,22 +320,27 @@ def create_app(
     chat_completion: CompletionCallable | None = None,
     mcp_transport: McpTransport | None = None,
     telegram_transport: TelegramTransport | None = None,
+    workdir: Path | str | None = None,
 ) -> FastAPI:
     ensure_logging_configured()
     ensure_sandbox_available()
 
+    cwd = resolve_workdir(workdir)
     store = StateStore()
+    compact_provider = LocalSummaryCompactProvider()
     mcp_manager = McpManager(store=store, transport=mcp_transport)
     telegram_bot_manager: TelegramBotManager | None = None
+    workspace_runs: dict[str, WorkspaceRun] = {}
+    active_workspace_run_id: str | None = None
 
     static_dir = frontend_static_directory().resolve(strict=False)
     logger.debug("Flowent app created serve_frontend=%s", serve_frontend)
+    logger.info("Workdir: %s", cwd)
     logger.info("Static directory: %s", static_dir)
 
     async def run_workspace_turn(content: str) -> StoredMessage:
         state = store.read_state()
         connection = selected_connection(state)
-        cwd = Path.cwd()
         user_message = StoredMessage(
             author="user",
             content=content,
@@ -268,6 +351,7 @@ def create_app(
         chat_messages = workspace_chat_messages(
             next_messages,
             store.read_compacted_context(),
+            store.read_active_compaction_checkpoint(),
         )
         skill_messages = explicit_skill_messages(cwd, store, content)
         request_messages = [
@@ -363,9 +447,23 @@ def create_app(
     @app.get("/api/state")
     async def app_state() -> StoredState:
         state = store.read_state()
+        active_run = (
+            workspace_runs.get(active_workspace_run_id)
+            if active_workspace_run_id
+            else None
+        )
         update: dict[str, object] = {
+            "active_run_event_index": active_run.latest_event_index
+            if active_run
+            else 0,
+            "active_run_id": active_run.id
+            if active_run and not active_run.is_done
+            else None,
             "mcp_servers": mcp_manager.servers_with_status(state.mcp_servers),
-            "skills": discover_skills(Path.cwd(), store),
+            "permission_requests": active_run.permission_requests()
+            if active_run and not active_run.is_done
+            else [],
+            "skills": discover_skills(cwd, store),
         }
         if telegram_bot_manager is not None:
             update["telegram_bot"] = telegram_bot_manager.bot_with_status(
@@ -390,12 +488,12 @@ def create_app(
     async def preview_mcp_import(
         request: McpImportPreviewRequest,
     ) -> McpImportDiscovery:
-        return discover_imported_mcp_servers(Path.cwd(), source=request.source)
+        return discover_imported_mcp_servers(cwd, source=request.source)
 
     @app.post("/api/mcp/import")
     async def import_mcp_servers(request: McpImportRequest) -> list[StoredMcpServer]:
         imported_servers = discover_imported_mcp_servers(
-            Path.cwd(),
+            cwd,
             source=request.source,
         ).servers
         existing_servers = {server.id for server in store.read_mcp_servers()}
@@ -426,7 +524,7 @@ def create_app(
 
     @app.post("/api/skills/reload")
     async def reload_skills() -> list[StoredSkill]:
-        return discover_skills(Path.cwd(), store)
+        return discover_skills(cwd, store)
 
     @app.put("/api/skills/{skill_id:path}")
     async def save_skill_settings(
@@ -434,7 +532,7 @@ def create_app(
         request: SkillSettingsRequest,
     ) -> StoredSkill:
         try:
-            return update_skill_enabled(Path.cwd(), store, skill_id, request.enabled)
+            return update_skill_enabled(cwd, store, skill_id, request.enabled)
         except KeyError as error:
             raise HTTPException(status_code=404, detail="Skill not found.") from error
 
@@ -472,67 +570,74 @@ def create_app(
     async def save_settings(settings: StoredSettings) -> StoredSettings:
         return store.save_settings(settings)
 
+    @app.post("/api/permissions/writable-paths")
+    async def save_writable_path(
+        request: WritablePathRequest,
+    ) -> StoredWritablePath:
+        return store.save_writable_path(normalized_request_path(request.path, cwd))
+
+    @app.delete("/api/permissions/writable-paths")
+    async def delete_writable_path(
+        request: WritablePathRequest,
+    ) -> WritablePathListResponse:
+        return WritablePathListResponse(
+            writable_paths=store.delete_writable_path(
+                normalized_request_path(request.path, cwd)
+            )
+        )
+
+    @app.post("/api/workspace/permissions/approve")
+    async def approve_workspace_permission(
+        request: WorkspacePermissionDecisionRequest,
+    ) -> dict[str, bool]:
+        run = active_workspace_run()
+        if run is None:
+            raise HTTPException(status_code=404, detail="Request not found.")
+        pending = run.pending_permissions.pop(request.id, None)
+        if pending is None:
+            raise HTTPException(status_code=404, detail="Request not found.")
+        path = pending.path
+        if request.decision == "always_allow":
+            saved_path = store.save_writable_path(path)
+            path = Path(saved_path.path)
+        pending.future.set_result(
+            WritablePathDecision(decision=request.decision, path=path)
+        )
+        return {"ok": True}
+
     @app.put("/api/workspace/messages")
     async def save_workspace_messages(
         request: WorkspaceMessagesRequest,
     ) -> WorkspaceMessagesRequest:
+        nonlocal active_workspace_run_id
+        if not request.messages:
+            run = active_workspace_run()
+            if run is not None and run.task is not None and not run.task.done():
+                run.discard_on_cancel = True
+                run.task.cancel()
+                active_workspace_run_id = None
         return WorkspaceMessagesRequest(messages=store.save_messages(request.messages))
 
-    @app.post("/api/workspace/compact")
-    async def compact_workspace() -> WorkspaceCompactResponse:
-        logger.info("Workspace compact requested")
-        state = store.read_state()
-        connection = selected_connection(state)
-        compacted_context = store.read_compacted_context()
-        cwd = Path.cwd()
+    async def append_run_event(
+        run: WorkspaceRun, event: str, data: dict[str, object]
+    ) -> None:
+        async with run.condition:
+            run.events.append((run.latest_event_index + 1, event, data))
+            run.condition.notify_all()
 
-        try:
-            summary = await complete_chat(
-                connection,
-                compact_prompt_messages(
-                    state.messages,
-                    compacted_context,
-                    runtime_context_messages(cwd),
-                ),
-                completion=chat_completion,
-            )
-        except HTTPException:
-            raise
-        except Exception as error:
-            logger.exception("Workspace compact failed")
-            raise HTTPException(
-                status_code=500,
-                detail="Context could not be compacted.",
-            ) from error
+    def active_workspace_run() -> WorkspaceRun | None:
+        if active_workspace_run_id is None:
+            return None
+        return workspace_runs.get(active_workspace_run_id)
 
-        marker = StoredMessage(
-            author="system",
-            content=COMPACTED_CONTEXT_MARKER,
-            id=str(uuid4()),
-        )
-        store.save_compacted_context(summary.content)
-        store.save_messages([*state.messages, marker])
-        logger.info(
-            "Workspace compact completed summary_length=%s", len(summary.content)
-        )
-        logger.log(TRACE_LEVEL, "Workspace compact summary=%r", summary.content)
-        return WorkspaceCompactResponse(message=marker)
-
-    @app.post("/api/workspace/respond")
-    async def respond_to_workspace(
-        request: WorkspaceRespondRequest,
-    ) -> StreamingResponse:
-        logger.info(
-            "Workspace response requested content_length=%s", len(request.content)
-        )
-        logger.log(TRACE_LEVEL, "Workspace user content=%r", request.content)
+    def create_workspace_run(content: str) -> WorkspaceRun:
+        nonlocal active_workspace_run_id
         state = store.read_state()
         connection = selected_connection(state)
-        cwd = Path.cwd()
 
         user_message = StoredMessage(
             author="user",
-            content=request.content,
+            content=content,
             id=str(uuid4()),
         )
         next_messages = [*state.messages, user_message]
@@ -540,17 +645,22 @@ def create_app(
         chat_messages = workspace_chat_messages(
             next_messages,
             store.read_compacted_context(),
+            store.read_active_compaction_checkpoint(),
         )
         request_messages = [
             message.model_dump()
             for message in [
                 *runtime_context_messages(cwd),
-                *explicit_skill_messages(cwd, store, request.content),
+                *explicit_skill_messages(cwd, store, content),
                 *chat_messages,
             ]
         ]
+        run = WorkspaceRun(condition=asyncio.Condition())
+        workspace_runs[run.id] = run
+        active_workspace_run_id = run.id
 
-        async def response_stream() -> AsyncIterator[str]:
+        async def run_task() -> None:
+            nonlocal active_workspace_run_id
             assistant_tools: dict[str, StoredToolItem] = {}
             assistant_message = StoredMessage(
                 author="assistant",
@@ -577,6 +687,61 @@ def create_app(
                 store.upsert_message(assistant_message)
 
             try:
+                current_tool_id: str | None = None
+
+                async def request_writable_path(
+                    path: Path, reason: str
+                ) -> WritablePathDecision:
+                    permission_id = str(uuid4())
+                    future = asyncio.get_running_loop().create_future()
+                    run.pending_permissions[permission_id] = PendingWorkspacePermission(
+                        future=future,
+                        path=path,
+                        reason=reason,
+                        tool_call_id=current_tool_id,
+                    )
+                    if current_tool_id and current_tool_id in assistant_tools:
+                        assistant_tools[current_tool_id] = (
+                            StoredToolItem.model_validate(
+                                {
+                                    **assistant_tools[current_tool_id].model_dump(
+                                        exclude_none=True
+                                    ),
+                                    "status": "waiting",
+                                }
+                            )
+                        )
+                        persist_assistant()
+                    await append_run_event(
+                        run,
+                        "permission_request",
+                        {
+                            "id": permission_id,
+                            "path": str(path),
+                            "reason": reason,
+                            "tool_call_id": current_tool_id,
+                        },
+                    )
+                    try:
+                        return await future
+                    finally:
+                        run.pending_permissions.pop(permission_id, None)
+
+                async def tool_runner(
+                    name: str,
+                    arguments: dict[str, object],
+                    context: ToolContext,
+                ):
+                    return await run_tool_with_path_permissions(
+                        name,
+                        arguments,
+                        context,
+                        request_writable_path=request_writable_path,
+                        writable_paths=[
+                            Path(path.path) for path in store.read_writable_paths()
+                        ],
+                    )
+
                 async for event in run_agent_stream(
                     completion=chat_completion,
                     connection=connection,
@@ -585,6 +750,7 @@ def create_app(
                     extra_tool_specs=mcp_manager.tool_specs(),
                     extra_tool_title=mcp_manager.tool_title,
                     messages=request_messages,
+                    tool_runner=tool_runner,
                 ):
                     if event.event == "start":
                         event_id = event.data.get("id")
@@ -596,6 +762,7 @@ def create_app(
                     if event.event == "tool_start":
                         tool = event.data.get("tool")
                         if isinstance(tool, dict) and isinstance(tool.get("id"), str):
+                            current_tool_id = tool["id"]
                             assistant_tools[tool["id"]] = StoredToolItem.model_validate(
                                 tool
                             )
@@ -603,6 +770,9 @@ def create_app(
                     if event.event in {"tool_done", "tool_error"}:
                         tool_id = event.data.get("id")
                         if isinstance(tool_id, str) and tool_id in assistant_tools:
+                            current_tool_id = (
+                                None if current_tool_id == tool_id else current_tool_id
+                            )
                             assistant_tools[tool_id] = StoredToolItem.model_validate(
                                 {
                                     **assistant_tools[tool_id].model_dump(
@@ -634,22 +804,169 @@ def create_app(
                                 message.get("thinking") or assistant_thinking
                             )
                             persist_assistant("completed")
-                    yield stream_event(event.event, event.data)
+                    await append_run_event(run, event.event, event.data)
             except asyncio.CancelledError:
-                logger.info("Workspace response interrupted")
-                persist_assistant("interrupted")
+                logger.info("Workspace run stopped")
+                if not run.discard_on_cancel:
+                    persist_assistant("interrupted")
+                    await append_run_event(
+                        run,
+                        "error",
+                        {"message": "Response stopped."},
+                    )
                 raise
             except Exception as error:
                 logger.exception("Workspace response failed")
                 persist_assistant("failed")
-                yield stream_event(
+                await append_run_event(
+                    run,
                     "error",
                     {"message": str(error) or "Message could not be sent."},
                 )
+            finally:
+                run.is_done = True
+                async with run.condition:
+                    run.condition.notify_all()
+                if active_workspace_run_id == run.id:
+                    active_workspace_run_id = None
+
+        run.task = asyncio.create_task(run_task())
+        return run
+
+    async def workspace_run_stream(
+        run: WorkspaceRun, after: int = 0
+    ) -> AsyncIterator[str]:
+        next_event_index = after + 1
+        while True:
+            async with run.condition:
+
+                def has_next_event(index: int = next_event_index) -> bool:
+                    return run.is_done or any(
+                        event_index >= index for event_index, _, _ in run.events
+                    )
+
+                await run.condition.wait_for(has_next_event)
+                events = [event for event in run.events if event[0] >= next_event_index]
+
+            for index, event, data in events:
+                next_event_index = index + 1
+                yield stream_event(event, data)
+                if event in {"done", "error"}:
+                    return
+
+            if run.is_done and not events:
                 return
 
+    @app.post("/api/workspace/runs")
+    async def start_workspace_run(
+        request: WorkspaceRespondRequest,
+    ) -> WorkspaceRunResponse:
+        logger.info("Workspace run requested content_length=%s", len(request.content))
+        logger.log(TRACE_LEVEL, "Workspace user content=%r", request.content)
+        run = create_workspace_run(request.content)
+        return WorkspaceRunResponse(run_id=run.id)
+
+    @app.get("/api/workspace/runs/{run_id}/stream")
+    async def stream_workspace_run(
+        run_id: str,
+        after: int = Query(default=0, ge=0),
+    ) -> StreamingResponse:
+        run = workspace_runs.get(run_id)
+        if run is None:
+            raise HTTPException(status_code=404, detail="Run not found.")
+        return StreamingResponse(
+            workspace_run_stream(run, after),
+            media_type="text/event-stream",
+        )
+
+    @app.post("/api/workspace/runs/{run_id}/stop")
+    async def stop_workspace_run(run_id: str) -> dict[str, bool]:
+        run = workspace_runs.get(run_id)
+        if run is None:
+            raise HTTPException(status_code=404, detail="Run not found.")
+        if run.task is not None and not run.task.done():
+            run.task.cancel()
+        return {"ok": True}
+
+    @app.post("/api/workspace/compact")
+    async def compact_workspace() -> WorkspaceCompactResponse:
+        if active_workspace_run() is not None:
+            raise HTTPException(
+                status_code=409,
+                detail="Compact is unavailable while Flowent is responding.",
+            )
+        logger.info("Workspace compact requested")
+        state = store.read_state()
+        connection = selected_connection(state)
+        checkpoint = store.read_active_compaction_checkpoint()
+        model_history = [
+            *runtime_context_messages(cwd),
+            *workspace_chat_messages(
+                state.messages,
+                store.read_compacted_context(),
+                checkpoint,
+            ),
+        ]
+
+        try:
+            compact_result = await compact_provider.compact(
+                connection,
+                CompactInput(
+                    messages=state.messages,
+                    model_history=model_history,
+                    trigger="manual",
+                ),
+                completion=chat_completion,
+            )
+        except HTTPException:
+            raise
+        except Exception as error:
+            logger.exception("Workspace compact failed")
+            raise HTTPException(
+                status_code=500,
+                detail="Context could not be compacted.",
+            ) from error
+
+        marker = StoredMessage(
+            author="system",
+            content=COMPACTED_CONTEXT_MARKER,
+            id=str(uuid4()),
+        )
+        source_message_id = state.messages[-1].id if state.messages else None
+        store.save_compaction_checkpoint(
+            StoredCompactionCheckpoint(
+                id=str(uuid4()),
+                method=compact_result.method,
+                replacement_history=compact_result.replacement_history,
+                source_message_id=source_message_id,
+                summary=compact_result.summary,
+                token_after=compact_result.token_after,
+                token_before=compact_result.token_before,
+                trigger="manual",
+            )
+        )
+        store.save_messages([*state.messages, marker])
+        logger.info(
+            "Workspace compact completed method=%s summary_length=%s token_before=%s token_after=%s",
+            compact_result.method,
+            len(compact_result.summary),
+            compact_result.token_before,
+            compact_result.token_after,
+        )
+        logger.log(TRACE_LEVEL, "Workspace compact summary=%r", compact_result.summary)
+        return WorkspaceCompactResponse(message=marker)
+
+    @app.post("/api/workspace/respond")
+    async def respond_to_workspace(
+        request: WorkspaceRespondRequest,
+    ) -> StreamingResponse:
+        logger.info(
+            "Workspace response requested content_length=%s", len(request.content)
+        )
+        logger.log(TRACE_LEVEL, "Workspace user content=%r", request.content)
+        run = create_workspace_run(request.content)
         return StreamingResponse(
-            response_stream(),
+            workspace_run_stream(run),
             media_type="text/event-stream",
         )