flowent 0.2.3 → 0.3.0

package/backend/src/flowent/system_tools.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+import shutil
+
+
+class SystemToolError(RuntimeError):
+    pass
+
+
+RIPGREP_INSTALL_HINT = (
+    "Install ripgrep and try again. Debian/Ubuntu: "
+    "sudo apt-get install ripgrep. Fedora: sudo dnf install ripgrep. "
+    "Arch: sudo pacman -S ripgrep."
+)
+
+
+def ripgrep_binary() -> str | None:
+    return shutil.which("rg")
+
+
+def ensure_ripgrep_available() -> str:
+    rg = ripgrep_binary()
+    if not rg:
+        raise SystemToolError(f"Search is not available. {RIPGREP_INSTALL_HINT}")
+    return rg

package/backend/src/flowent/tools.py

@@ -13,9 +13,11 @@ from uuid import uuid4
 
 from pydantic import BaseModel, ConfigDict
 
+from flowent.network import flowent_user_agent
 from flowent.patch import affected_paths
 from flowent.sandbox import SandboxError, SandboxRunner
 from flowent.shell import shell_invocation
+from flowent.system_tools import ensure_ripgrep_available
 
 
 class ToolResult(BaseModel):
@@ -285,7 +287,7 @@ def grep_files(arguments: dict[str, object], context: ToolContext) -> ToolResult
     pattern = str(arguments["pattern"])
     path = resolve_tool_path(str(arguments.get("path", ".") or "."), context.cwd)
     limit = integer_argument(arguments, "limit", 100)
-    command = ["rg", "--line-number", "--max-count", str(limit)]
+    command = [ensure_ripgrep_available(), "--line-number", "--max-count", str(limit)]
     include = arguments.get("include")
     if include:
         command.extend(["--glob", str(include)])
@@ -438,7 +440,7 @@ def default_web_search(query: str) -> list[dict[str, str]]:
     encoded_query = urllib.parse.urlencode({"q": query})
     request = urllib.request.Request(
         f"https://duckduckgo.com/html/?{encoded_query}",
-        headers={"User-Agent": "Flowent/0.1"},
+        headers={"User-Agent": flowent_user_agent()},
     )
     with urllib.request.urlopen(request, timeout=10) as response:
         body = response.read().decode(errors="replace")

package/backend/src/flowent/usage.py

@@ -5,6 +5,7 @@ from typing import Any
 from pydantic import BaseModel, ConfigDict, Field
 
 DEFAULT_MODEL_CONTEXT_WINDOW = 120_000
+APPROX_BYTES_PER_TOKEN = 4
 
 MODEL_CONTEXT_WINDOWS: dict[str, int] = {
     "claude-3-7-sonnet-20250219": 200_000,
@@ -251,12 +252,12 @@ def estimated_token_usage_for_messages(
     *,
     output_content: str = "",
 ) -> TokenUsage:
-    total_tokens = sum(estimate_mapping_message_tokens(message) for message in messages)
+    input_tokens = sum(estimate_mapping_message_tokens(message) for message in messages)
     output_tokens = approximate_token_count(output_content)
     return TokenUsage(
-        input_tokens=max(total_tokens - output_tokens, 0),
+        input_tokens=input_tokens,
         output_tokens=output_tokens,
-        total_tokens=total_tokens,
+        total_tokens=input_tokens + output_tokens,
     )
 
 
@@ -273,7 +274,11 @@ def estimate_mapping_message_tokens(message: Mapping[str, object]) -> int:
 def approximate_token_count(content: str) -> int:
     if not content:
         return 0
-    return max(1, (len(content) + 3) // 4)
+    return max(
+        1,
+        (len(content.encode("utf-8")) + APPROX_BYTES_PER_TOKEN - 1)
+        // APPROX_BYTES_PER_TOKEN,
+    )
 
 
 def string_content(value: object) -> str:

package/backend/src/flowent/workflows.py

@@ -0,0 +1,282 @@
+import json
+import re
+from collections import defaultdict, deque
+from collections.abc import Mapping
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from flowent.llm import (
+    ChatMessage,
+    CompletionCallable,
+    ProviderConnection,
+    complete_chat,
+)
+from flowent.storage import (
+    StoredWorkflow,
+    StoredWorkflowDefinition,
+    StoredWorkflowEdge,
+    StoredWorkflowNode,
+)
+
+
+class WorkflowNodeRunResult(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    error: str = ""
+    id: str
+    output: str = ""
+    status: str
+
+
+class WorkflowRunResponse(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    node_results: list[WorkflowNodeRunResult] = Field(default_factory=list)
+    outputs: dict[str, str] = Field(default_factory=dict)
+    status: str
+    workflow_id: str
+
+
+PLACEHOLDER_PATTERN = re.compile(r"\{\{\s*([A-Za-z0-9_.-]+)\.output\s*\}\}")
+
+
+def validate_workflow(workflow: StoredWorkflow) -> StoredWorkflow:
+    validate_workflow_definition(workflow.definition)
+    return workflow
+
+
+def validate_workflow_draft(workflow: StoredWorkflow) -> StoredWorkflow:
+    validate_workflow_draft_definition(workflow.definition)
+    return workflow
+
+
+def validate_workflow_draft_definition(definition: StoredWorkflowDefinition) -> None:
+    node_ids = [node.id for node in definition.nodes]
+    if any(not node_id.strip() for node_id in node_ids):
+        raise ValueError("Workflow node ids must not be empty.")
+    if len(set(node_ids)) != len(node_ids):
+        raise ValueError("Workflow node ids must be unique.")
+
+    edge_ids = [edge.id for edge in definition.edges]
+    if any(not edge_id.strip() for edge_id in edge_ids):
+        raise ValueError("Workflow edge ids must not be empty.")
+    if len(set(edge_ids)) != len(edge_ids):
+        raise ValueError("Workflow edge ids must be unique.")
+
+    node_id_set = set(node_ids)
+    for edge in definition.edges:
+        if edge.source not in node_id_set or edge.target not in node_id_set:
+            raise ValueError("Workflow edges must connect existing nodes.")
+
+
+def validate_workflow_definition(definition: StoredWorkflowDefinition) -> list[str]:
+    node_ids = [node.id for node in definition.nodes]
+    if not node_ids:
+        raise ValueError("Workflow needs at least one node.")
+    if any(not node_id.strip() for node_id in node_ids):
+        raise ValueError("Workflow node ids must not be empty.")
+    if len(set(node_ids)) != len(node_ids):
+        raise ValueError("Workflow node ids must be unique.")
+    if not any(node.type == "input" for node in definition.nodes):
+        raise ValueError("Workflow needs an input node.")
+    if not any(node.type == "output" for node in definition.nodes):
+        raise ValueError("Workflow needs an output node.")
+
+    edge_ids = [edge.id for edge in definition.edges]
+    if any(not edge_id.strip() for edge_id in edge_ids):
+        raise ValueError("Workflow edge ids must not be empty.")
+    if len(set(edge_ids)) != len(edge_ids):
+        raise ValueError("Workflow edge ids must be unique.")
+
+    node_id_set = set(node_ids)
+    for edge in definition.edges:
+        if edge.source not in node_id_set or edge.target not in node_id_set:
+            raise ValueError("Workflow edges must connect existing nodes.")
+
+    return topological_node_ids(definition)
+
+
+def workflow_requires_connection(definition: StoredWorkflowDefinition) -> bool:
+    return any(node.type == "agent" for node in definition.nodes)
+
+
+def topological_node_ids(definition: StoredWorkflowDefinition) -> list[str]:
+    node_ids = [node.id for node in definition.nodes]
+    outgoing: dict[str, list[str]] = defaultdict(list)
+    indegree = {node_id: 0 for node_id in node_ids}
+    for edge in definition.edges:
+        outgoing[edge.source].append(edge.target)
+        indegree[edge.target] += 1
+
+    node_order = {node_id: index for index, node_id in enumerate(node_ids)}
+    ready = deque(
+        sorted(
+            [node_id for node_id, degree in indegree.items() if degree == 0],
+            key=lambda node_id: node_order[node_id],
+        )
+    )
+    ordered: list[str] = []
+    while ready:
+        node_id = ready.popleft()
+        ordered.append(node_id)
+        for target in sorted(
+            outgoing[node_id], key=lambda node_id: node_order[node_id]
+        ):
+            indegree[target] -= 1
+            if indegree[target] == 0:
+                ready.append(target)
+
+    if len(ordered) != len(node_ids):
+        raise ValueError("Workflow cannot contain cycles.")
+    return ordered
+
+
+async def run_workflow_definition(
+    *,
+    completion: CompletionCallable | None,
+    connection: ProviderConnection | None,
+    definition: StoredWorkflowDefinition,
+    workflow_id: str,
+) -> WorkflowRunResponse:
+    ordered_ids = validate_workflow_definition(definition)
+    if workflow_requires_connection(definition) and connection is None:
+        raise ValueError("Choose a provider and model before running.")
+
+    nodes = {node.id: node for node in definition.nodes}
+    incoming_edges = edges_by_target(definition.edges)
+    results: dict[str, WorkflowNodeRunResult] = {
+        node.id: WorkflowNodeRunResult(id=node.id, status="pending")
+        for node in definition.nodes
+    }
+    outputs: dict[str, str] = {}
+    named_outputs: dict[str, str] = {}
+
+    for node_id in ordered_ids:
+        node = nodes[node_id]
+        results[node.id] = WorkflowNodeRunResult(id=node.id, status="running")
+        try:
+            output = await run_node(
+                completion=completion,
+                connection=connection,
+                incoming_edges=incoming_edges[node.id],
+                node=node,
+                outputs=outputs,
+            )
+        except Exception as error:
+            results[node.id] = WorkflowNodeRunResult(
+                error=str(error) or "Node could not be completed.",
+                id=node.id,
+                status="failed",
+            )
+            return WorkflowRunResponse(
+                node_results=list(results.values()),
+                outputs=named_outputs,
+                status="failed",
+                workflow_id=workflow_id,
+            )
+        outputs[node.id] = output
+        if node.type == "output":
+            named_outputs[node_output_key(node)] = output
+        results[node.id] = WorkflowNodeRunResult(
+            id=node.id,
+            output=output,
+            status="success",
+        )
+
+    return WorkflowRunResponse(
+        node_results=list(results.values()),
+        outputs=named_outputs,
+        status="success",
+        workflow_id=workflow_id,
+    )
+
+
+async def run_node(
+    *,
+    completion: CompletionCallable | None,
+    connection: ProviderConnection | None,
+    incoming_edges: list[StoredWorkflowEdge],
+    node: StoredWorkflowNode,
+    outputs: Mapping[str, str],
+) -> str:
+    if node.type == "input":
+        return node_data_text(node, "default_value")
+    if node.type == "agent":
+        if connection is None:
+            raise ValueError("Choose a provider and model before running.")
+        prompt = render_template(
+            node_data_text(node, "prompt")
+            or joined_upstream_outputs(incoming_edges, outputs),
+            outputs,
+        )
+        response = await complete_chat(
+            connection,
+            [ChatMessage(role="user", content=prompt)],
+            completion=completion,
+        )
+        return response.content
+    if node.type == "merge":
+        upstream = upstream_outputs(incoming_edges, outputs)
+        if node_data_text(node, "merge_strategy") == "json":
+            return merge_json_outputs(upstream)
+        return "\n".join(output for output in upstream if output)
+    if node.type == "output":
+        return joined_upstream_outputs(incoming_edges, outputs)
+    raise ValueError("Node type is not supported.")
+
+
+def edges_by_target(
+    edges: list[StoredWorkflowEdge],
+) -> dict[str, list[StoredWorkflowEdge]]:
+    grouped: dict[str, list[StoredWorkflowEdge]] = defaultdict(list)
+    for edge in edges:
+        grouped[edge.target].append(edge)
+    return grouped
+
+
+def node_data_text(node: StoredWorkflowNode, key: str) -> str:
+    value = node.data.get(key, "")
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    return str(value)
+
+
+def node_output_key(node: StoredWorkflowNode) -> str:
+    return node_data_text(node, "output_key") or node.id
+
+
+def upstream_outputs(
+    incoming_edges: list[StoredWorkflowEdge],
+    outputs: Mapping[str, str],
+) -> list[str]:
+    return [outputs[edge.source] for edge in incoming_edges if edge.source in outputs]
+
+
+def joined_upstream_outputs(
+    incoming_edges: list[StoredWorkflowEdge],
+    outputs: Mapping[str, str],
+) -> str:
+    return "\n".join(
+        output for output in upstream_outputs(incoming_edges, outputs) if output
+    )
+
+
+def render_template(template: str, outputs: Mapping[str, str]) -> str:
+    return PLACEHOLDER_PATTERN.sub(
+        lambda match: outputs.get(match.group(1), ""),
+        template,
+    )
+
+
+def merge_json_outputs(upstream: list[str]) -> str:
+    merged: dict[str, object] = {}
+    for output in upstream:
+        try:
+            parsed = json.loads(output)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(parsed, dict):
+            merged.update(parsed)
+    return json.dumps(merged, ensure_ascii=False)

package/backend/src/flowent/workspace/context.py

@@ -0,0 +1,249 @@
+import json
+import os
+from collections.abc import Mapping, Sequence
+from typing import Literal
+
+from fastapi import HTTPException
+
+from flowent.compact import transcript_messages_after
+from flowent.llm import ChatMessage
+from flowent.storage import (
+    StoredCompactionCheckpoint,
+    StoredMessage,
+    StoredSettings,
+    StoredState,
+)
+from flowent.usage import (
+    TokenUsageInfo,
+    current_model_context_window,
+    estimated_token_usage_for_messages,
+    recompute_context_usage,
+)
+from flowent.workspace.output import error_context_summary, message_error_items
+
+COMPACTED_CONTEXT_MARKER = "Context compacted"
+OPTIMIZED_CONTEXT_MARKER = "Context optimized"
+DEFAULT_AUTO_COMPACT_CONTEXT_WINDOW_RATIO = 0.95
+
+
+def latest_compacted_context_index(messages: list[StoredMessage]) -> int | None:
+    for index in range(len(messages) - 1, -1, -1):
+        message = messages[index]
+        if message.author == "system" and is_context_marker(message):
+            return index
+    return None
+
+
+def is_context_marker(message: StoredMessage) -> bool:
+    return message.content in {COMPACTED_CONTEXT_MARKER, OPTIMIZED_CONTEXT_MARKER}
+
+
+def auto_compact_token_limit(context_window: int) -> int:
+    raw_limit = os.environ.get("FLOWENT_AUTO_COMPACT_TOKEN_LIMIT", "")
+    if not raw_limit:
+        return max(0, int(context_window * DEFAULT_AUTO_COMPACT_CONTEXT_WINDOW_RATIO))
+    try:
+        return max(0, int(raw_limit))
+    except ValueError:
+        return max(0, int(context_window * DEFAULT_AUTO_COMPACT_CONTEXT_WINDOW_RATIO))
+
+
+def should_auto_compact(
+    messages: list[ChatMessage],
+    *,
+    context_window: int,
+) -> bool:
+    token_limit = auto_compact_token_limit(context_window)
+    if token_limit <= 0:
+        return False
+    return (
+        estimated_token_usage_for_messages(
+            [message.model_dump() for message in messages]
+        ).total_tokens
+        >= token_limit
+    )
+
+
+def model_visible_messages_for_usage(
+    messages: Sequence[Mapping[str, object]],
+) -> list[dict[str, object]]:
+    return [
+        dict(message)
+        for message in messages
+        if message.get("role") in {"system", "user", "assistant", "tool"}
+    ]
+
+
+def usage_event_data(usage_info: TokenUsageInfo) -> dict[str, object]:
+    return {"usage_info": usage_info.model_dump()}
+
+
+def update_context_usage_for_response(
+    usage_info: TokenUsageInfo | None,
+    *,
+    messages: Sequence[Mapping[str, object]],
+    output_content: str,
+    output_tools: Sequence[Mapping[str, object]] = (),
+    model_context_window: int,
+) -> TokenUsageInfo:
+    return recompute_context_usage(
+        usage_info,
+        estimated_token_usage_for_messages(
+            [
+                *model_visible_messages_for_usage(messages),
+                *model_visible_response_messages_for_usage(
+                    output_content, output_tools
+                ),
+            ],
+        ).total_tokens,
+        model_context_window=model_context_window,
+    )
+
+
+def model_visible_response_messages_for_usage(
+    output_content: str,
+    output_tools: Sequence[Mapping[str, object]],
+) -> list[dict[str, object]]:
+    visible_messages: list[dict[str, object]] = []
+    for index, tool in enumerate(output_tools):
+        tool_id = str(tool.get("id") or f"call_{index}")
+        arguments = tool.get("arguments")
+        visible_messages.append(
+            {
+                "role": "assistant",
+                "content": "",
+                "tool_calls": [
+                    {
+                        "id": tool_id,
+                        "type": "function",
+                        "function": {
+                            "name": str(tool.get("name") or ""),
+                            "arguments": json.dumps(
+                                arguments if arguments is not None else {},
+                                ensure_ascii=False,
+                            ),
+                        },
+                    }
+                ],
+            }
+        )
+        visible_messages.append(
+            {
+                "role": "tool",
+                "tool_call_id": tool_id,
+                "content": str(tool.get("content") or ""),
+            }
+        )
+    if output_content:
+        visible_messages.append({"role": "assistant", "content": output_content})
+    return visible_messages
+
+
+def usage_info_for_model(
+    usage_info: TokenUsageInfo | None,
+    model_context_window: int,
+) -> TokenUsageInfo | None:
+    if usage_info is None:
+        return None
+    return usage_info.model_copy(update={"model_context_window": model_context_window})
+
+
+def context_window_for_settings(settings: StoredSettings) -> int:
+    if settings.context_window_limit is not None:
+        return settings.context_window_limit
+    return current_model_context_window(settings.selected_model)
+
+
+def state_with_current_model_context_window(state: StoredState) -> StoredState:
+    model_context_window = context_window_for_settings(state.settings)
+    return state.model_copy(
+        update={
+            "messages": [
+                message.model_copy(
+                    update={
+                        "usage_info": usage_info_for_model(
+                            message.usage_info,
+                            model_context_window,
+                        )
+                    }
+                )
+                if message.usage_info is not None
+                else message
+                for message in state.messages
+            ],
+            "usage_info": usage_info_for_model(
+                state.usage_info,
+                model_context_window,
+            ),
+        }
+    )
+
+
+def workspace_chat_messages(
+    messages: list[StoredMessage],
+    compacted_context: str = "",
+    checkpoint: StoredCompactionCheckpoint | None = None,
+) -> list[ChatMessage]:
+    chat_messages: list[ChatMessage] = []
+
+    if checkpoint is not None:
+        chat_messages.extend(checkpoint.replacement_history)
+        visible_messages = transcript_messages_after(
+            messages,
+            checkpoint.source_message_id,
+        )
+        for message in visible_messages:
+            if message.author == "system" and is_context_marker(message):
+                continue
+            if message.author not in ("user", "assistant"):
+                raise HTTPException(
+                    status_code=400, detail="Message history is invalid."
+                )
+            if message.author == "assistant":
+                errors = message_error_items(message)
+                if errors:
+                    chat_messages.extend(
+                        ChatMessage(
+                            role="assistant", content=error_context_summary(error)
+                        )
+                        for error in errors
+                    )
+                    continue
+            checkpoint_role: Literal["user", "assistant"] = (
+                "user" if message.author == "user" else "assistant"
+            )
+            chat_messages.append(
+                ChatMessage(role=checkpoint_role, content=message.content)
+            )
+        return chat_messages
+
+    marker_index = latest_compacted_context_index(messages)
+    visible_messages = messages
+
+    if compacted_context and marker_index is not None:
+        chat_messages.extend(
+            [
+                ChatMessage(role="user", content=COMPACTED_CONTEXT_MARKER),
+                ChatMessage(role="assistant", content=compacted_context),
+            ]
+        )
+        visible_messages = messages[marker_index + 1 :]
+
+    for message in visible_messages:
+        if message.author == "system" and is_context_marker(message):
+            continue
+        if message.author not in ("user", "assistant"):
+            raise HTTPException(status_code=400, detail="Message history is invalid.")
+        if message.author == "assistant":
+            errors = message_error_items(message)
+            if errors:
+                chat_messages.extend(
+                    ChatMessage(role="assistant", content=error_context_summary(error))
+                    for error in errors
+                )
+                continue
+        role: Literal["user", "assistant"] = (
+            "user" if message.author == "user" else "assistant"
+        )
+        chat_messages.append(ChatMessage(role=role, content=message.content))
+    return chat_messages