flowent 0.2.4 → 0.3.0

package/backend/src/flowent/mcp.py

@@ -5,17 +5,23 @@ import json
 import logging
 import os
 import re
+import sys
 from collections.abc import Callable
-from contextlib import AsyncExitStack
+from contextlib import AsyncExitStack, suppress
 from dataclasses import dataclass
 from importlib import import_module
-from typing import Any, Protocol
+from typing import Any, Protocol, TextIO, cast
 
 from flowent.storage import StateStore, StoredMcpServer, StoredMcpTool
 from flowent.tools import ToolResult
 
 logger = logging.getLogger("flowent.mcp")
 MCP_CONNECT_TIMEOUT_SECONDS = 10
+PYTHON_TRACEBACK_START = "Traceback (most recent call last):"
+PYTHON_TRACEBACK_TERMINAL_PATTERN = re.compile(
+    r"^(?:[A-Za-z_][A-Za-z0-9_.]*(?:Error|Exception|Interrupt|Warning)|"
+    r"BaseExceptionGroup|ExceptionGroup)(?::|$)"
+)
 
 
 class McpTransport(Protocol):
@@ -128,6 +134,96 @@ class _McpConnection:
     session: Any = None
 
 
+class _McpStdioErrorFilter:
+    def __init__(self, target: TextIO) -> None:
+        self.target = target
+        self.line_buffer = ""
+        self.traceback_lines: list[str] | None = None
+
+    def feed(self, text: str) -> None:
+        self.line_buffer += text
+        while "\n" in self.line_buffer:
+            line, self.line_buffer = self.line_buffer.split("\n", 1)
+            self.feed_line(f"{line}\n")
+
+    def finish(self) -> None:
+        if self.line_buffer:
+            self.feed_line(self.line_buffer)
+            self.line_buffer = ""
+        if self.traceback_lines is not None:
+            self.write("".join(self.traceback_lines))
+            self.traceback_lines = None
+
+    def feed_line(self, line: str) -> None:
+        stripped_line = line.rstrip("\r\n")
+        if self.traceback_lines is not None:
+            self.traceback_lines.append(line)
+            if stripped_line == "KeyboardInterrupt":
+                self.traceback_lines = None
+                return
+            if PYTHON_TRACEBACK_TERMINAL_PATTERN.match(stripped_line):
+                self.write("".join(self.traceback_lines))
+                self.traceback_lines = None
+            return
+        if stripped_line == PYTHON_TRACEBACK_START:
+            self.traceback_lines = [line]
+            return
+        self.write(line)
+
+    def write(self, text: str) -> None:
+        self.target.write(text)
+        self.target.flush()
+
+
+class _McpStdioErrorLog:
+    def __init__(self, target: TextIO | None = None) -> None:
+        self.target = target or sys.stderr
+        self.filter = _McpStdioErrorFilter(self.target)
+        self.read_fd, write_fd = os.pipe()
+        self.write_file = os.fdopen(write_fd, "wb", buffering=0)
+        self.drain_task: asyncio.Task[None] | None = None
+
+    async def __aenter__(self) -> _McpStdioErrorLog:
+        self.drain_task = asyncio.create_task(self.drain())
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        traceback: object,
+    ) -> None:
+        self.close_write_file()
+        if self.drain_task is not None:
+            await asyncio.gather(self.drain_task, return_exceptions=True)
+        else:
+            self.close_read_fd()
+
+    def fileno(self) -> int:
+        return self.write_file.fileno()
+
+    async def drain(self) -> None:
+        try:
+            while True:
+                chunk = await asyncio.to_thread(os.read, self.read_fd, 4096)
+                if not chunk:
+                    break
+                self.filter.feed(chunk.decode("utf-8", errors="replace"))
+        except OSError:
+            pass
+        finally:
+            self.filter.finish()
+            self.close_read_fd()
+
+    def close_write_file(self) -> None:
+        with suppress(OSError, ValueError):
+            self.write_file.close()
+
+    def close_read_fd(self) -> None:
+        with suppress(OSError):
+            os.close(self.read_fd)
+
+
 class DefaultMcpTransport:
     def __init__(self) -> None:
         self._connections: dict[str, _McpConnection] = {}
@@ -197,9 +293,11 @@ class DefaultMcpTransport:
                             )
                         )
                 else:
+                    stdio_errlog = await stack.enter_async_context(_McpStdioErrorLog())
                     read_stream, write_stream = await stack.enter_async_context(
                         stdio_client(
                             self._stdio_parameters(server, config),
+                            errlog=cast(TextIO, stdio_errlog),
                         )
                     )
                 session = await stack.enter_async_context(

package/backend/src/flowent/network.py

@@ -0,0 +1,5 @@
+from flowent._version import __version__
+
+
+def flowent_user_agent() -> str:
+    return f"Flowent/{__version__}"

package/backend/src/flowent/provider_connections.py

@@ -0,0 +1,42 @@
+import logging
+
+from fastapi import HTTPException
+
+from flowent.llm import ProviderConnection
+from flowent.storage import StoredState
+
+logger = logging.getLogger("flowent.provider_connections")
+
+
+def selected_connection(state: StoredState) -> ProviderConnection:
+    provider = next(
+        (
+            stored_provider
+            for stored_provider in state.providers
+            if stored_provider.id == state.settings.selected_provider_id
+        ),
+        None,
+    )
+    if provider is None or not state.settings.selected_model:
+        logger.warning("Workspace request blocked because provider or model is missing")
+        raise HTTPException(
+            status_code=400,
+            detail="Choose a provider and model before sending.",
+        )
+    if not provider.api_key:
+        logger.warning("Workspace request blocked because selected provider has no key")
+        raise HTTPException(status_code=400, detail="Add a key before sending.")
+
+    logger.debug(
+        "Workspace request using provider=%s model=%s",
+        provider.name,
+        state.settings.selected_model,
+    )
+    return ProviderConnection(
+        base_url=provider.base_url or None,
+        model=state.settings.selected_model,
+        name=provider.name,
+        provider=provider.type,
+        reasoning_effort=state.settings.reasoning_effort,
+        secret_reference=provider.api_key,
+    )

package/backend/src/flowent/routes/integrations.py

@@ -0,0 +1,105 @@
+from pathlib import Path
+
+from fastapi import FastAPI, HTTPException
+
+from flowent.api_models import (
+    McpImportPreviewRequest,
+    McpImportRequest,
+    SkillSettingsRequest,
+    TelegramSessionApproveRequest,
+)
+from flowent.channels import TelegramBotManager
+from flowent.mcp import McpManager
+from flowent.mcp_import import McpImportDiscovery, discover_imported_mcp_servers
+from flowent.skills import discover_skills, update_skill_enabled
+from flowent.storage import (
+    StateStore,
+    StoredMcpServer,
+    StoredSkill,
+    StoredTelegramBot,
+    StoredTelegramSession,
+)
+
+
+def register_integration_routes(
+    app: FastAPI,
+    *,
+    cwd: Path,
+    mcp_manager: McpManager,
+    store: StateStore,
+    telegram_bot_manager: TelegramBotManager,
+) -> None:
+    @app.put("/api/mcp/servers")
+    async def save_mcp_server(server: StoredMcpServer) -> StoredMcpServer:
+        saved_server = store.save_mcp_server(server)
+        return await mcp_manager.sync_server(saved_server)
+
+    @app.post("/api/mcp/import/preview")
+    async def preview_mcp_import(
+        request: McpImportPreviewRequest,
+    ) -> McpImportDiscovery:
+        return discover_imported_mcp_servers(cwd, source=request.source)
+
+    @app.post("/api/mcp/import")
+    async def import_mcp_servers(request: McpImportRequest) -> list[StoredMcpServer]:
+        imported_servers = discover_imported_mcp_servers(
+            cwd,
+            source=request.source,
+        ).servers
+        existing_servers = {server.id for server in store.read_mcp_servers()}
+        for server in imported_servers:
+            if server.id != request.server_id:
+                continue
+            if server.id in existing_servers:
+                continue
+            store.save_mcp_server(server)
+            existing_servers.add(server.id)
+        return mcp_manager.servers_with_status(store.read_mcp_servers())
+
+    @app.delete("/api/mcp/servers/{server_id}")
+    async def delete_mcp_server(server_id: str) -> dict[str, bool]:
+        await mcp_manager.delete_server(server_id)
+        return {"ok": True}
+
+    @app.post("/api/mcp/servers/{server_id}/reconnect")
+    async def reconnect_mcp_server(server_id: str) -> StoredMcpServer:
+        try:
+            return await mcp_manager.reconnect_server(server_id)
+        except KeyError as error:
+            raise HTTPException(status_code=404, detail="Server not found.") from error
+
+    @app.post("/api/mcp/reload")
+    async def reload_mcp_servers() -> list[StoredMcpServer]:
+        return await mcp_manager.reload()
+
+    @app.post("/api/skills/reload")
+    async def reload_skills() -> list[StoredSkill]:
+        return discover_skills(cwd, store)
+
+    @app.put("/api/skills/{skill_id:path}")
+    async def save_skill_settings(
+        skill_id: str,
+        request: SkillSettingsRequest,
+    ) -> StoredSkill:
+        try:
+            return update_skill_enabled(cwd, store, skill_id, request.enabled)
+        except KeyError as error:
+            raise HTTPException(status_code=404, detail="Skill not found.") from error
+
+    @app.put("/api/telegram-bot")
+    async def save_telegram_bot(telegram_bot: StoredTelegramBot) -> StoredTelegramBot:
+        saved_bot = store.save_telegram_bot(telegram_bot)
+        await telegram_bot_manager.sync_bot(saved_bot)
+        return telegram_bot_manager.bot_with_status(saved_bot)
+
+    @app.post("/api/telegram-bot/approve")
+    async def approve_telegram_session(
+        request: TelegramSessionApproveRequest,
+    ) -> StoredTelegramSession:
+        try:
+            return store.approve_telegram_session(request.chat_id)
+        except KeyError as error:
+            raise HTTPException(
+                status_code=404,
+                detail="Conversation not found.",
+            ) from error

package/backend/src/flowent/routes/permissions.py

@@ -0,0 +1,36 @@
+from pathlib import Path
+
+from fastapi import FastAPI
+
+from flowent.api_models import WritablePathListResponse, WritablePathRequest
+from flowent.storage import StateStore, StoredWritablePath
+
+
+def normalized_request_path(path: str, cwd: Path) -> Path:
+    raw_path = Path(path).expanduser()
+    if not raw_path.is_absolute():
+        raw_path = cwd / raw_path
+    return raw_path.resolve(strict=False)
+
+
+def register_permission_routes(
+    app: FastAPI,
+    *,
+    cwd: Path,
+    store: StateStore,
+) -> None:
+    @app.post("/api/permissions/writable-paths")
+    async def save_writable_path(
+        request: WritablePathRequest,
+    ) -> StoredWritablePath:
+        return store.save_writable_path(normalized_request_path(request.path, cwd))
+
+    @app.delete("/api/permissions/writable-paths")
+    async def delete_writable_path(
+        request: WritablePathRequest,
+    ) -> WritablePathListResponse:
+        return WritablePathListResponse(
+            writable_paths=store.delete_writable_path(
+                normalized_request_path(request.path, cwd)
+            )
+        )

package/backend/src/flowent/routes/providers.py

@@ -0,0 +1,30 @@
+from fastapi import FastAPI
+
+from flowent.api_models import ProviderModelsRequest, ProviderModelsResponse
+from flowent.llm import list_provider_models
+from flowent.storage import StateStore, StoredProvider, StoredSettings
+
+
+def register_provider_routes(app: FastAPI, *, store: StateStore) -> None:
+    @app.post("/api/providers")
+    async def save_provider(provider: StoredProvider) -> StoredProvider:
+        return store.save_provider(provider)
+
+    @app.delete("/api/providers/{provider_id}")
+    async def delete_provider(provider_id: str) -> dict[str, bool]:
+        store.delete_provider(provider_id)
+        return {"ok": True}
+
+    @app.post("/api/providers/models")
+    async def provider_models(request: ProviderModelsRequest) -> ProviderModelsResponse:
+        return ProviderModelsResponse(
+            models=list_provider_models(
+                base_url=request.base_url,
+                provider=request.provider,
+                secret_reference=request.secret_reference,
+            ),
+        )
+
+    @app.put("/api/settings")
+    async def save_settings(settings: StoredSettings) -> StoredSettings:
+        return store.save_settings(settings)

package/backend/src/flowent/routes/system.py

@@ -0,0 +1,49 @@
+from pathlib import Path
+
+from fastapi import FastAPI
+
+from flowent._version import __version__
+from flowent.api_models import AboutResponse
+from flowent.channels import TelegramBotManager
+from flowent.mcp import McpManager
+from flowent.skills import discover_skills
+from flowent.storage import StateStore, StoredState
+from flowent.workspace.context import state_with_current_model_context_window
+from flowent.workspace.runtime import WorkspaceRuntime
+
+
+def register_system_routes(
+    app: FastAPI,
+    *,
+    cwd: Path,
+    mcp_manager: McpManager,
+    runtime: WorkspaceRuntime,
+    store: StateStore,
+    telegram_bot_manager: TelegramBotManager,
+) -> None:
+    @app.get("/api/health")
+    async def health() -> dict[str, str]:
+        return {"status": "ok"}
+
+    @app.get("/api/state")
+    async def app_state() -> StoredState:
+        state = state_with_current_model_context_window(store.read_state())
+        active_run = runtime.active_run()
+        update: dict[str, object] = {
+            "active_run_event_index": active_run.latest_event_index
+            if active_run
+            else 0,
+            "active_run_id": active_run.id
+            if active_run and not active_run.is_done
+            else None,
+            "mcp_servers": mcp_manager.servers_with_status(state.mcp_servers),
+            "skills": discover_skills(cwd, store),
+        }
+        update["telegram_bot"] = telegram_bot_manager.bot_with_status(
+            state.telegram_bot
+        )
+        return state.model_copy(update=update)
+
+    @app.get("/api/about")
+    async def about() -> AboutResponse:
+        return AboutResponse(version=__version__)

package/backend/src/flowent/routes/workflow_routes.py

@@ -0,0 +1,63 @@
+from fastapi import FastAPI, HTTPException
+
+from flowent.llm import CompletionCallable
+from flowent.provider_connections import selected_connection
+from flowent.storage import StateStore, StoredWorkflow
+from flowent.workflows import (
+    WorkflowRunResponse,
+    run_workflow_definition,
+    validate_workflow_draft,
+    workflow_requires_connection,
+)
+
+
+def register_workflow_routes(
+    app: FastAPI,
+    *,
+    chat_completion: CompletionCallable | None,
+    store: StateStore,
+) -> None:
+    @app.put("/api/workflows")
+    async def save_workflow(workflow: StoredWorkflow) -> StoredWorkflow:
+        try:
+            return store.save_workflow(
+                validate_workflow_draft(
+                    workflow.model_copy(
+                        update={"name": workflow.name.strip() or "Untitled Workflow"}
+                    )
+                )
+            )
+        except ValueError as error:
+            raise HTTPException(status_code=400, detail=str(error)) from error
+
+    @app.delete("/api/workflows/{workflow_id}")
+    async def delete_workflow(workflow_id: str) -> dict[str, bool]:
+        store.delete_workflow(workflow_id)
+        return {"ok": True}
+
+    @app.post("/api/workflows/{workflow_id}/run")
+    async def run_workflow(workflow_id: str) -> WorkflowRunResponse:
+        workflow = next(
+            (
+                current_workflow
+                for current_workflow in store.read_workflows()
+                if current_workflow.id == workflow_id
+            ),
+            None,
+        )
+        if workflow is None:
+            raise HTTPException(status_code=404, detail="Workflow not found.")
+        try:
+            connection = (
+                selected_connection(store.read_state())
+                if workflow_requires_connection(workflow.definition)
+                else None
+            )
+            return await run_workflow_definition(
+                completion=chat_completion,
+                connection=connection,
+                definition=workflow.definition,
+                workflow_id=workflow.id,
+            )
+        except ValueError as error:
+            raise HTTPException(status_code=400, detail=str(error)) from error

package/backend/src/flowent/routes/workspace.py

@@ -0,0 +1,105 @@
+import logging
+
+from fastapi import FastAPI, Query
+from fastapi.responses import StreamingResponse
+
+from flowent.api_models import (
+    WorkspaceClearResponse,
+    WorkspaceMessageEditRequest,
+    WorkspaceMessageEditResponse,
+    WorkspaceMessagesRequest,
+    WorkspaceRespondRequest,
+    WorkspaceRunResponse,
+)
+from flowent.logging import TRACE_LEVEL
+from flowent.storage import StateStore
+from flowent.workspace.runtime import WorkspaceRuntime
+
+logger = logging.getLogger("flowent.routes.workspace")
+
+
+def register_workspace_routes(
+    app: FastAPI,
+    *,
+    runtime: WorkspaceRuntime,
+    store: StateStore,
+) -> None:
+    @app.put("/api/workspace/messages")
+    async def save_workspace_messages(
+        request: WorkspaceMessagesRequest,
+    ) -> WorkspaceMessagesRequest:
+        return WorkspaceMessagesRequest(messages=store.save_messages(request.messages))
+
+    @app.post("/api/workspace/messages/{message_id}/edit")
+    async def edit_workspace_message(
+        message_id: str,
+        request: WorkspaceMessageEditRequest,
+    ) -> WorkspaceMessageEditResponse:
+        logger.info(
+            "Workspace message edit requested action=%s message_id=%s content_length=%s",
+            request.action,
+            message_id,
+            len(request.content),
+        )
+        logger.log(TRACE_LEVEL, "Workspace edited user content=%r", request.content)
+        messages, run = runtime.edit_message(
+            message_id,
+            action=request.action,
+            content=request.content,
+        )
+        return WorkspaceMessageEditResponse(
+            messages=messages,
+            run_id=run.id if run else None,
+        )
+
+    @app.post("/api/workspace/clear")
+    async def clear_workspace() -> WorkspaceClearResponse:
+        messages = runtime.clear()
+        await runtime.notify_cleared_runs()
+        return WorkspaceClearResponse(messages=messages)
+
+    @app.post("/api/workspace/runs")
+    async def start_workspace_run(
+        request: WorkspaceRespondRequest,
+    ) -> WorkspaceRunResponse:
+        logger.info("Workspace run requested content_length=%s", len(request.content))
+        logger.log(TRACE_LEVEL, "Workspace user content=%r", request.content)
+        run = runtime.create_run(request.content, message_id=request.message_id)
+        return WorkspaceRunResponse(run_id=run.id)
+
+    @app.get("/api/workspace/runs/{run_id}/stream")
+    async def stream_workspace_run(
+        run_id: str,
+        after: int = Query(default=0, ge=0),
+    ) -> StreamingResponse:
+        run = runtime.run_by_id(run_id)
+        return StreamingResponse(
+            runtime.run_stream(run, after),
+            media_type="text/event-stream",
+        )
+
+    @app.post("/api/workspace/runs/{run_id}/stop")
+    async def stop_workspace_run(run_id: str) -> dict[str, bool]:
+        runtime.stop_run(run_id)
+        return {"ok": True}
+
+    @app.post("/api/workspace/compact", response_class=StreamingResponse)
+    async def compact_workspace() -> StreamingResponse:
+        return StreamingResponse(
+            runtime.compact_stream(),
+            media_type="text/event-stream",
+        )
+
+    @app.post("/api/workspace/respond")
+    async def respond_to_workspace(
+        request: WorkspaceRespondRequest,
+    ) -> StreamingResponse:
+        logger.info(
+            "Workspace response requested content_length=%s", len(request.content)
+        )
+        logger.log(TRACE_LEVEL, "Workspace user content=%r", request.content)
+        run = runtime.create_run(request.content, message_id=request.message_id)
+        return StreamingResponse(
+            runtime.run_stream(run, include_snapshots=False),
+            media_type="text/event-stream",
+        )

package/backend/src/flowent/state/__init__.py

@@ -0,0 +1,53 @@
+from flowent.state.models import (
+    StoredAssistantOutputGroup,
+    StoredCompactionCheckpoint,
+    StoredErrorOutputItem,
+    StoredMcpServer,
+    StoredMcpTool,
+    StoredMessage,
+    StoredOutputItem,
+    StoredProvider,
+    StoredSettings,
+    StoredSkill,
+    StoredState,
+    StoredTelegramBot,
+    StoredTelegramSession,
+    StoredTextOutputItem,
+    StoredThinkingOutputItem,
+    StoredToolItem,
+    StoredToolOutputItem,
+    StoredWorkflow,
+    StoredWorkflowDefinition,
+    StoredWorkflowEdge,
+    StoredWorkflowNode,
+    StoredWorkflowNodePosition,
+    StoredWritablePath,
+)
+from flowent.state.store import StateStore
+
+__all__ = [
+    "StateStore",
+    "StoredAssistantOutputGroup",
+    "StoredCompactionCheckpoint",
+    "StoredErrorOutputItem",
+    "StoredMcpServer",
+    "StoredMcpTool",
+    "StoredMessage",
+    "StoredOutputItem",
+    "StoredProvider",
+    "StoredSettings",
+    "StoredSkill",
+    "StoredState",
+    "StoredTelegramBot",
+    "StoredTelegramSession",
+    "StoredTextOutputItem",
+    "StoredThinkingOutputItem",
+    "StoredToolItem",
+    "StoredToolOutputItem",
+    "StoredWorkflow",
+    "StoredWorkflowDefinition",
+    "StoredWorkflowEdge",
+    "StoredWorkflowNode",
+    "StoredWorkflowNodePosition",
+    "StoredWritablePath",
+]