flowent 0.2.3 → 0.3.0

package/backend/src/flowent/mcp.py

@@ -5,16 +5,23 @@ import json
 import logging
 import os
 import re
+import sys
 from collections.abc import Callable
-from contextlib import AsyncExitStack
+from contextlib import AsyncExitStack, suppress
+from dataclasses import dataclass
 from importlib import import_module
-from typing import Any, Protocol
+from typing import Any, Protocol, TextIO, cast
 
 from flowent.storage import StateStore, StoredMcpServer, StoredMcpTool
 from flowent.tools import ToolResult
 
 logger = logging.getLogger("flowent.mcp")
 MCP_CONNECT_TIMEOUT_SECONDS = 10
+PYTHON_TRACEBACK_START = "Traceback (most recent call last):"
+PYTHON_TRACEBACK_TERMINAL_PATTERN = re.compile(
+    r"^(?:[A-Za-z_][A-Za-z0-9_.]*(?:Error|Exception|Interrupt|Warning)|"
+    r"BaseExceptionGroup|ExceptionGroup)(?::|$)"
+)
 
 
 class McpTransport(Protocol):
@@ -119,54 +126,230 @@ def expand_mcp_config(config: dict[str, object]) -> dict[str, object]:
     return expanded if isinstance(expanded, dict) else {}
 
 
+@dataclass
+class _McpConnection:
+    close_event: asyncio.Event
+    ready: asyncio.Future[list[dict[str, object]]]
+    owner_task: asyncio.Task[None] | None = None
+    session: Any = None
+
+
+class _McpStdioErrorFilter:
+    def __init__(self, target: TextIO) -> None:
+        self.target = target
+        self.line_buffer = ""
+        self.traceback_lines: list[str] | None = None
+
+    def feed(self, text: str) -> None:
+        self.line_buffer += text
+        while "\n" in self.line_buffer:
+            line, self.line_buffer = self.line_buffer.split("\n", 1)
+            self.feed_line(f"{line}\n")
+
+    def finish(self) -> None:
+        if self.line_buffer:
+            self.feed_line(self.line_buffer)
+            self.line_buffer = ""
+        if self.traceback_lines is not None:
+            self.write("".join(self.traceback_lines))
+            self.traceback_lines = None
+
+    def feed_line(self, line: str) -> None:
+        stripped_line = line.rstrip("\r\n")
+        if self.traceback_lines is not None:
+            self.traceback_lines.append(line)
+            if stripped_line == "KeyboardInterrupt":
+                self.traceback_lines = None
+                return
+            if PYTHON_TRACEBACK_TERMINAL_PATTERN.match(stripped_line):
+                self.write("".join(self.traceback_lines))
+                self.traceback_lines = None
+            return
+        if stripped_line == PYTHON_TRACEBACK_START:
+            self.traceback_lines = [line]
+            return
+        self.write(line)
+
+    def write(self, text: str) -> None:
+        self.target.write(text)
+        self.target.flush()
+
+
+class _McpStdioErrorLog:
+    def __init__(self, target: TextIO | None = None) -> None:
+        self.target = target or sys.stderr
+        self.filter = _McpStdioErrorFilter(self.target)
+        self.read_fd, write_fd = os.pipe()
+        self.write_file = os.fdopen(write_fd, "wb", buffering=0)
+        self.drain_task: asyncio.Task[None] | None = None
+
+    async def __aenter__(self) -> _McpStdioErrorLog:
+        self.drain_task = asyncio.create_task(self.drain())
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        traceback: object,
+    ) -> None:
+        self.close_write_file()
+        if self.drain_task is not None:
+            await asyncio.gather(self.drain_task, return_exceptions=True)
+        else:
+            self.close_read_fd()
+
+    def fileno(self) -> int:
+        return self.write_file.fileno()
+
+    async def drain(self) -> None:
+        try:
+            while True:
+                chunk = await asyncio.to_thread(os.read, self.read_fd, 4096)
+                if not chunk:
+                    break
+                self.filter.feed(chunk.decode("utf-8", errors="replace"))
+        except OSError:
+            pass
+        finally:
+            self.filter.finish()
+            self.close_read_fd()
+
+    def close_write_file(self) -> None:
+        with suppress(OSError, ValueError):
+            self.write_file.close()
+
+    def close_read_fd(self) -> None:
+        with suppress(OSError):
+            os.close(self.read_fd)
+
+
 class DefaultMcpTransport:
     def __init__(self) -> None:
-        self._sessions: dict[str, Any] = {}
-        self._stacks: dict[str, AsyncExitStack] = {}
+        self._connections: dict[str, _McpConnection] = {}
 
     async def connect(self, server: StoredMcpServer) -> list[dict[str, object]]:
+        await self.disconnect(server.id)
+        loop = asyncio.get_running_loop()
+        connection = _McpConnection(
+            close_event=asyncio.Event(),
+            ready=loop.create_future(),
+        )
+        connection.owner_task = asyncio.create_task(
+            self._run_connection(server, connection)
+        )
+        self._connections[server.id] = connection
+        try:
+            return await asyncio.shield(connection.ready)
+        except asyncio.CancelledError:
+            await self._close_connection(
+                server.id,
+                connection,
+                cancel_owner=True,
+                suppress_errors=True,
+            )
+            raise
+        except Exception:
+            await self._close_connection(
+                server.id,
+                connection,
+                cancel_owner=False,
+                suppress_errors=True,
+            )
+            raise
+
+    async def _run_connection(
+        self,
+        server: StoredMcpServer,
+        connection: _McpConnection,
+    ) -> None:
         from mcp import ClientSession
         from mcp.client.stdio import stdio_client
 
-        await self.disconnect(server.id)
         stack = AsyncExitStack()
-        config = expand_mcp_config(server.config)
-        if server.type == "url":
-            http_module = import_module("mcp.client.streamable_http")
-            http_headers = self._streamable_http_headers(config) or None
-            if hasattr(http_module, "streamablehttp_client"):
-                read_stream, write_stream, _ = await stack.enter_async_context(
-                    http_module.streamablehttp_client(
-                        server.url or str(config.get("url") or ""),
-                        headers=http_headers,
+        try:
+            async with stack:
+                config = expand_mcp_config(server.config)
+                if server.type == "url":
+                    http_module = import_module("mcp.client.streamable_http")
+                    http_headers = self._streamable_http_headers(config) or None
+                    if hasattr(http_module, "streamablehttp_client"):
+                        read_stream, write_stream, _ = await stack.enter_async_context(
+                            http_module.streamablehttp_client(
+                                server.url or str(config.get("url") or ""),
+                                headers=http_headers,
+                            )
+                        )
+                    else:
+                        import httpx
+
+                        http_client = await stack.enter_async_context(
+                            httpx.AsyncClient(headers=http_headers)
+                        )
+                        read_stream, write_stream, _ = await stack.enter_async_context(
+                            http_module.streamable_http_client(
+                                server.url or str(config.get("url") or ""),
+                                http_client=http_client,
+                            )
+                        )
+                else:
+                    stdio_errlog = await stack.enter_async_context(_McpStdioErrorLog())
+                    read_stream, write_stream = await stack.enter_async_context(
+                        stdio_client(
+                            self._stdio_parameters(server, config),
+                            errlog=cast(TextIO, stdio_errlog),
+                        )
                     )
+                session = await stack.enter_async_context(
+                    ClientSession(read_stream, write_stream)
                 )
-            else:
-                import httpx
-
-                http_client = await stack.enter_async_context(
-                    httpx.AsyncClient(headers=http_headers)
-                )
-                read_stream, write_stream, _ = await stack.enter_async_context(
-                    http_module.streamable_http_client(
-                        server.url or str(config.get("url") or ""),
-                        http_client=http_client,
+                await session.initialize()
+                result = await session.list_tools()
+                connection.session = session
+                if not connection.ready.done():
+                    connection.ready.set_result(
+                        [self._model_dump(tool) for tool in result.tools]
                     )
-                )
-        else:
-            read_stream, write_stream = await stack.enter_async_context(
-                stdio_client(
-                    self._stdio_parameters(server, config),
-                )
-            )
-        session = await stack.enter_async_context(
-            ClientSession(read_stream, write_stream)
-        )
-        await session.initialize()
-        result = await session.list_tools()
-        self._sessions[server.id] = session
-        self._stacks[server.id] = stack
-        return [self._model_dump(tool) for tool in result.tools]
+                await connection.close_event.wait()
+        except asyncio.CancelledError:
+            if not connection.ready.done():
+                connection.ready.cancel()
+            raise
+        except Exception as error:
+            if not connection.ready.done():
+                connection.ready.set_exception(error)
+            raise
+        finally:
+            connection.session = None
+
+    async def _close_connection(
+        self,
+        server_id: str,
+        connection: _McpConnection,
+        *,
+        cancel_owner: bool,
+        suppress_errors: bool = False,
+    ) -> None:
+        if self._connections.get(server_id) is connection:
+            self._connections.pop(server_id, None)
+        connection.session = None
+        owner_task = connection.owner_task
+        if owner_task is None:
+            return
+        if not owner_task.done():
+            if cancel_owner:
+                owner_task.cancel()
+            else:
+                connection.close_event.set()
+        try:
+            await asyncio.shield(owner_task)
+        except asyncio.CancelledError:
+            if cancel_owner or suppress_errors:
+                return
+            raise
+        except Exception:
+            if not suppress_errors:
+                raise
 
     def _streamable_http_headers(self, config: dict[str, object]) -> dict[str, str]:
         headers: dict[str, str] = {}
@@ -238,10 +421,13 @@ class DefaultMcpTransport:
         )
 
     async def disconnect(self, server_id: str) -> None:
-        stack = self._stacks.pop(server_id, None)
-        self._sessions.pop(server_id, None)
-        if stack is not None:
-            await stack.aclose()
+        connection = self._connections.pop(server_id, None)
+        if connection is not None:
+            await self._close_connection(
+                server_id,
+                connection,
+                cancel_owner=not connection.ready.done(),
+            )
 
     async def call_tool(
         self,
@@ -249,7 +435,8 @@ class DefaultMcpTransport:
         tool_name: str,
         arguments: dict[str, object],
     ) -> dict[str, object]:
-        session = self._sessions.get(server_id)
+        connection = self._connections.get(server_id)
+        session = connection.session if connection is not None else None
         if session is None:
             raise RuntimeError("Server is not connected.")
         result = await session.call_tool(tool_name, arguments=arguments)

package/backend/src/flowent/network.py

@@ -0,0 +1,5 @@
+from flowent._version import __version__
+
+
+def flowent_user_agent() -> str:
+    return f"Flowent/{__version__}"

package/backend/src/flowent/permissions.py

@@ -348,7 +348,11 @@ def tool_failure_text(result: ToolResult) -> str:
     stderr = str(result.data.get("stderr", "") or "").strip()
     stdout = str(result.data.get("stdout", "") or "").strip()
     content = result.content.strip()
-    return "\n".join(part for part in [stderr, stdout, content] if part)
+    parts: list[str] = []
+    for part in [stderr, stdout, content]:
+        if part and part not in parts:
+            parts.append(part)
+    return "\n".join(parts)
 
 
 async def shell_command_without_sandbox(

package/backend/src/flowent/provider_connections.py

@@ -0,0 +1,42 @@
+import logging
+
+from fastapi import HTTPException
+
+from flowent.llm import ProviderConnection
+from flowent.storage import StoredState
+
+logger = logging.getLogger("flowent.provider_connections")
+
+
+def selected_connection(state: StoredState) -> ProviderConnection:
+    provider = next(
+        (
+            stored_provider
+            for stored_provider in state.providers
+            if stored_provider.id == state.settings.selected_provider_id
+        ),
+        None,
+    )
+    if provider is None or not state.settings.selected_model:
+        logger.warning("Workspace request blocked because provider or model is missing")
+        raise HTTPException(
+            status_code=400,
+            detail="Choose a provider and model before sending.",
+        )
+    if not provider.api_key:
+        logger.warning("Workspace request blocked because selected provider has no key")
+        raise HTTPException(status_code=400, detail="Add a key before sending.")
+
+    logger.debug(
+        "Workspace request using provider=%s model=%s",
+        provider.name,
+        state.settings.selected_model,
+    )
+    return ProviderConnection(
+        base_url=provider.base_url or None,
+        model=state.settings.selected_model,
+        name=provider.name,
+        provider=provider.type,
+        reasoning_effort=state.settings.reasoning_effort,
+        secret_reference=provider.api_key,
+    )

package/backend/src/flowent/routes/integrations.py

@@ -0,0 +1,105 @@
+from pathlib import Path
+
+from fastapi import FastAPI, HTTPException
+
+from flowent.api_models import (
+    McpImportPreviewRequest,
+    McpImportRequest,
+    SkillSettingsRequest,
+    TelegramSessionApproveRequest,
+)
+from flowent.channels import TelegramBotManager
+from flowent.mcp import McpManager
+from flowent.mcp_import import McpImportDiscovery, discover_imported_mcp_servers
+from flowent.skills import discover_skills, update_skill_enabled
+from flowent.storage import (
+    StateStore,
+    StoredMcpServer,
+    StoredSkill,
+    StoredTelegramBot,
+    StoredTelegramSession,
+)
+
+
+def register_integration_routes(
+    app: FastAPI,
+    *,
+    cwd: Path,
+    mcp_manager: McpManager,
+    store: StateStore,
+    telegram_bot_manager: TelegramBotManager,
+) -> None:
+    @app.put("/api/mcp/servers")
+    async def save_mcp_server(server: StoredMcpServer) -> StoredMcpServer:
+        saved_server = store.save_mcp_server(server)
+        return await mcp_manager.sync_server(saved_server)
+
+    @app.post("/api/mcp/import/preview")
+    async def preview_mcp_import(
+        request: McpImportPreviewRequest,
+    ) -> McpImportDiscovery:
+        return discover_imported_mcp_servers(cwd, source=request.source)
+
+    @app.post("/api/mcp/import")
+    async def import_mcp_servers(request: McpImportRequest) -> list[StoredMcpServer]:
+        imported_servers = discover_imported_mcp_servers(
+            cwd,
+            source=request.source,
+        ).servers
+        existing_servers = {server.id for server in store.read_mcp_servers()}
+        for server in imported_servers:
+            if server.id != request.server_id:
+                continue
+            if server.id in existing_servers:
+                continue
+            store.save_mcp_server(server)
+            existing_servers.add(server.id)
+        return mcp_manager.servers_with_status(store.read_mcp_servers())
+
+    @app.delete("/api/mcp/servers/{server_id}")
+    async def delete_mcp_server(server_id: str) -> dict[str, bool]:
+        await mcp_manager.delete_server(server_id)
+        return {"ok": True}
+
+    @app.post("/api/mcp/servers/{server_id}/reconnect")
+    async def reconnect_mcp_server(server_id: str) -> StoredMcpServer:
+        try:
+            return await mcp_manager.reconnect_server(server_id)
+        except KeyError as error:
+            raise HTTPException(status_code=404, detail="Server not found.") from error
+
+    @app.post("/api/mcp/reload")
+    async def reload_mcp_servers() -> list[StoredMcpServer]:
+        return await mcp_manager.reload()
+
+    @app.post("/api/skills/reload")
+    async def reload_skills() -> list[StoredSkill]:
+        return discover_skills(cwd, store)
+
+    @app.put("/api/skills/{skill_id:path}")
+    async def save_skill_settings(
+        skill_id: str,
+        request: SkillSettingsRequest,
+    ) -> StoredSkill:
+        try:
+            return update_skill_enabled(cwd, store, skill_id, request.enabled)
+        except KeyError as error:
+            raise HTTPException(status_code=404, detail="Skill not found.") from error
+
+    @app.put("/api/telegram-bot")
+    async def save_telegram_bot(telegram_bot: StoredTelegramBot) -> StoredTelegramBot:
+        saved_bot = store.save_telegram_bot(telegram_bot)
+        await telegram_bot_manager.sync_bot(saved_bot)
+        return telegram_bot_manager.bot_with_status(saved_bot)
+
+    @app.post("/api/telegram-bot/approve")
+    async def approve_telegram_session(
+        request: TelegramSessionApproveRequest,
+    ) -> StoredTelegramSession:
+        try:
+            return store.approve_telegram_session(request.chat_id)
+        except KeyError as error:
+            raise HTTPException(
+                status_code=404,
+                detail="Conversation not found.",
+            ) from error

package/backend/src/flowent/routes/permissions.py

@@ -0,0 +1,36 @@
+from pathlib import Path
+
+from fastapi import FastAPI
+
+from flowent.api_models import WritablePathListResponse, WritablePathRequest
+from flowent.storage import StateStore, StoredWritablePath
+
+
+def normalized_request_path(path: str, cwd: Path) -> Path:
+    raw_path = Path(path).expanduser()
+    if not raw_path.is_absolute():
+        raw_path = cwd / raw_path
+    return raw_path.resolve(strict=False)
+
+
+def register_permission_routes(
+    app: FastAPI,
+    *,
+    cwd: Path,
+    store: StateStore,
+) -> None:
+    @app.post("/api/permissions/writable-paths")
+    async def save_writable_path(
+        request: WritablePathRequest,
+    ) -> StoredWritablePath:
+        return store.save_writable_path(normalized_request_path(request.path, cwd))
+
+    @app.delete("/api/permissions/writable-paths")
+    async def delete_writable_path(
+        request: WritablePathRequest,
+    ) -> WritablePathListResponse:
+        return WritablePathListResponse(
+            writable_paths=store.delete_writable_path(
+                normalized_request_path(request.path, cwd)
+            )
+        )

package/backend/src/flowent/routes/providers.py

@@ -0,0 +1,30 @@
+from fastapi import FastAPI
+
+from flowent.api_models import ProviderModelsRequest, ProviderModelsResponse
+from flowent.llm import list_provider_models
+from flowent.storage import StateStore, StoredProvider, StoredSettings
+
+
+def register_provider_routes(app: FastAPI, *, store: StateStore) -> None:
+    @app.post("/api/providers")
+    async def save_provider(provider: StoredProvider) -> StoredProvider:
+        return store.save_provider(provider)
+
+    @app.delete("/api/providers/{provider_id}")
+    async def delete_provider(provider_id: str) -> dict[str, bool]:
+        store.delete_provider(provider_id)
+        return {"ok": True}
+
+    @app.post("/api/providers/models")
+    async def provider_models(request: ProviderModelsRequest) -> ProviderModelsResponse:
+        return ProviderModelsResponse(
+            models=list_provider_models(
+                base_url=request.base_url,
+                provider=request.provider,
+                secret_reference=request.secret_reference,
+            ),
+        )
+
+    @app.put("/api/settings")
+    async def save_settings(settings: StoredSettings) -> StoredSettings:
+        return store.save_settings(settings)

package/backend/src/flowent/routes/system.py

@@ -0,0 +1,49 @@
+from pathlib import Path
+
+from fastapi import FastAPI
+
+from flowent._version import __version__
+from flowent.api_models import AboutResponse
+from flowent.channels import TelegramBotManager
+from flowent.mcp import McpManager
+from flowent.skills import discover_skills
+from flowent.storage import StateStore, StoredState
+from flowent.workspace.context import state_with_current_model_context_window
+from flowent.workspace.runtime import WorkspaceRuntime
+
+
+def register_system_routes(
+    app: FastAPI,
+    *,
+    cwd: Path,
+    mcp_manager: McpManager,
+    runtime: WorkspaceRuntime,
+    store: StateStore,
+    telegram_bot_manager: TelegramBotManager,
+) -> None:
+    @app.get("/api/health")
+    async def health() -> dict[str, str]:
+        return {"status": "ok"}
+
+    @app.get("/api/state")
+    async def app_state() -> StoredState:
+        state = state_with_current_model_context_window(store.read_state())
+        active_run = runtime.active_run()
+        update: dict[str, object] = {
+            "active_run_event_index": active_run.latest_event_index
+            if active_run
+            else 0,
+            "active_run_id": active_run.id
+            if active_run and not active_run.is_done
+            else None,
+            "mcp_servers": mcp_manager.servers_with_status(state.mcp_servers),
+            "skills": discover_skills(cwd, store),
+        }
+        update["telegram_bot"] = telegram_bot_manager.bot_with_status(
+            state.telegram_bot
+        )
+        return state.model_copy(update=update)
+
+    @app.get("/api/about")
+    async def about() -> AboutResponse:
+        return AboutResponse(version=__version__)

package/backend/src/flowent/routes/workflow_routes.py

@@ -0,0 +1,63 @@
+from fastapi import FastAPI, HTTPException
+
+from flowent.llm import CompletionCallable
+from flowent.provider_connections import selected_connection
+from flowent.storage import StateStore, StoredWorkflow
+from flowent.workflows import (
+    WorkflowRunResponse,
+    run_workflow_definition,
+    validate_workflow_draft,
+    workflow_requires_connection,
+)
+
+
+def register_workflow_routes(
+    app: FastAPI,
+    *,
+    chat_completion: CompletionCallable | None,
+    store: StateStore,
+) -> None:
+    @app.put("/api/workflows")
+    async def save_workflow(workflow: StoredWorkflow) -> StoredWorkflow:
+        try:
+            return store.save_workflow(
+                validate_workflow_draft(
+                    workflow.model_copy(
+                        update={"name": workflow.name.strip() or "Untitled Workflow"}
+                    )
+                )
+            )
+        except ValueError as error:
+            raise HTTPException(status_code=400, detail=str(error)) from error
+
+    @app.delete("/api/workflows/{workflow_id}")
+    async def delete_workflow(workflow_id: str) -> dict[str, bool]:
+        store.delete_workflow(workflow_id)
+        return {"ok": True}
+
+    @app.post("/api/workflows/{workflow_id}/run")
+    async def run_workflow(workflow_id: str) -> WorkflowRunResponse:
+        workflow = next(
+            (
+                current_workflow
+                for current_workflow in store.read_workflows()
+                if current_workflow.id == workflow_id
+            ),
+            None,
+        )
+        if workflow is None:
+            raise HTTPException(status_code=404, detail="Workflow not found.")
+        try:
+            connection = (
+                selected_connection(store.read_state())
+                if workflow_requires_connection(workflow.definition)
+                else None
+            )
+            return await run_workflow_definition(
+                completion=chat_completion,
+                connection=connection,
+                definition=workflow.definition,
+                workflow_id=workflow.id,
+            )
+        except ValueError as error:
+            raise HTTPException(status_code=400, detail=str(error)) from error