flowent 0.2.4 → 0.3.1

package/backend/src/flowent/system_tools.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+import shutil
+
+
+class SystemToolError(RuntimeError):
+    pass
+
+
+RIPGREP_INSTALL_HINT = (
+    "Install ripgrep and try again. Debian/Ubuntu: "
+    "sudo apt-get install ripgrep. Fedora: sudo dnf install ripgrep. "
+    "Arch: sudo pacman -S ripgrep."
+)
+
+
+def ripgrep_binary() -> str | None:
+    return shutil.which("rg")
+
+
+def ensure_ripgrep_available() -> str:
+    rg = ripgrep_binary()
+    if not rg:
+        raise SystemToolError(f"Search is not available. {RIPGREP_INSTALL_HINT}")
+    return rg

package/backend/src/flowent/tools.py

@@ -13,9 +13,11 @@ from uuid import uuid4
 
 from pydantic import BaseModel, ConfigDict
 
+from flowent.network import flowent_user_agent
 from flowent.patch import affected_paths
 from flowent.sandbox import SandboxError, SandboxRunner
 from flowent.shell import shell_invocation
+from flowent.system_tools import ensure_ripgrep_available
 
 
 class ToolResult(BaseModel):
@@ -285,7 +287,7 @@ def grep_files(arguments: dict[str, object], context: ToolContext) -> ToolResult
     pattern = str(arguments["pattern"])
     path = resolve_tool_path(str(arguments.get("path", ".") or "."), context.cwd)
     limit = integer_argument(arguments, "limit", 100)
-    command = ["rg", "--line-number", "--max-count", str(limit)]
+    command = [ensure_ripgrep_available(), "--line-number", "--max-count", str(limit)]
     include = arguments.get("include")
     if include:
         command.extend(["--glob", str(include)])
@@ -438,7 +440,7 @@ def default_web_search(query: str) -> list[dict[str, str]]:
     encoded_query = urllib.parse.urlencode({"q": query})
     request = urllib.request.Request(
         f"https://duckduckgo.com/html/?{encoded_query}",
-        headers={"User-Agent": "Flowent/0.1"},
+        headers={"User-Agent": flowent_user_agent()},
     )
     with urllib.request.urlopen(request, timeout=10) as response:
         body = response.read().decode(errors="replace")

package/backend/src/flowent/usage.py

@@ -5,6 +5,7 @@ from typing import Any
 from pydantic import BaseModel, ConfigDict, Field
 
 DEFAULT_MODEL_CONTEXT_WINDOW = 120_000
+APPROX_BYTES_PER_TOKEN = 4
 
 MODEL_CONTEXT_WINDOWS: dict[str, int] = {
     "claude-3-7-sonnet-20250219": 200_000,
@@ -251,12 +252,12 @@ def estimated_token_usage_for_messages(
     *,
     output_content: str = "",
 ) -> TokenUsage:
-    total_tokens = sum(estimate_mapping_message_tokens(message) for message in messages)
+    input_tokens = sum(estimate_mapping_message_tokens(message) for message in messages)
     output_tokens = approximate_token_count(output_content)
     return TokenUsage(
-        input_tokens=max(total_tokens - output_tokens, 0),
+        input_tokens=input_tokens,
         output_tokens=output_tokens,
-        total_tokens=total_tokens,
+        total_tokens=input_tokens + output_tokens,
     )
 
 
@@ -273,7 +274,11 @@ def estimate_mapping_message_tokens(message: Mapping[str, object]) -> int:
 def approximate_token_count(content: str) -> int:
     if not content:
         return 0
-    return max(1, (len(content) + 3) // 4)
+    return max(
+        1,
+        (len(content.encode("utf-8")) + APPROX_BYTES_PER_TOKEN - 1)
+        // APPROX_BYTES_PER_TOKEN,
+    )
 
 
 def string_content(value: object) -> str:

package/backend/src/flowent/workflows.py

@@ -0,0 +1,282 @@
+import json
+import re
+from collections import defaultdict, deque
+from collections.abc import Mapping
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from flowent.llm import (
+    ChatMessage,
+    CompletionCallable,
+    ProviderConnection,
+    complete_chat,
+)
+from flowent.storage import (
+    StoredWorkflow,
+    StoredWorkflowDefinition,
+    StoredWorkflowEdge,
+    StoredWorkflowNode,
+)
+
+
+class WorkflowNodeRunResult(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    error: str = ""
+    id: str
+    output: str = ""
+    status: str
+
+
+class WorkflowRunResponse(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    node_results: list[WorkflowNodeRunResult] = Field(default_factory=list)
+    outputs: dict[str, str] = Field(default_factory=dict)
+    status: str
+    workflow_id: str
+
+
+PLACEHOLDER_PATTERN = re.compile(r"\{\{\s*([A-Za-z0-9_.-]+)\.output\s*\}\}")
+
+
+def validate_workflow(workflow: StoredWorkflow) -> StoredWorkflow:
+    validate_workflow_definition(workflow.definition)
+    return workflow
+
+
+def validate_workflow_draft(workflow: StoredWorkflow) -> StoredWorkflow:
+    validate_workflow_draft_definition(workflow.definition)
+    return workflow
+
+
+def validate_workflow_draft_definition(definition: StoredWorkflowDefinition) -> None:
+    node_ids = [node.id for node in definition.nodes]
+    if any(not node_id.strip() for node_id in node_ids):
+        raise ValueError("Workflow node ids must not be empty.")
+    if len(set(node_ids)) != len(node_ids):
+        raise ValueError("Workflow node ids must be unique.")
+
+    edge_ids = [edge.id for edge in definition.edges]
+    if any(not edge_id.strip() for edge_id in edge_ids):
+        raise ValueError("Workflow edge ids must not be empty.")
+    if len(set(edge_ids)) != len(edge_ids):
+        raise ValueError("Workflow edge ids must be unique.")
+
+    node_id_set = set(node_ids)
+    for edge in definition.edges:
+        if edge.source not in node_id_set or edge.target not in node_id_set:
+            raise ValueError("Workflow edges must connect existing nodes.")
+
+
+def validate_workflow_definition(definition: StoredWorkflowDefinition) -> list[str]:
+    node_ids = [node.id for node in definition.nodes]
+    if not node_ids:
+        raise ValueError("Workflow needs at least one node.")
+    if any(not node_id.strip() for node_id in node_ids):
+        raise ValueError("Workflow node ids must not be empty.")
+    if len(set(node_ids)) != len(node_ids):
+        raise ValueError("Workflow node ids must be unique.")
+    if not any(node.type == "input" for node in definition.nodes):
+        raise ValueError("Workflow needs an input node.")
+    if not any(node.type == "output" for node in definition.nodes):
+        raise ValueError("Workflow needs an output node.")
+
+    edge_ids = [edge.id for edge in definition.edges]
+    if any(not edge_id.strip() for edge_id in edge_ids):
+        raise ValueError("Workflow edge ids must not be empty.")
+    if len(set(edge_ids)) != len(edge_ids):
+        raise ValueError("Workflow edge ids must be unique.")
+
+    node_id_set = set(node_ids)
+    for edge in definition.edges:
+        if edge.source not in node_id_set or edge.target not in node_id_set:
+            raise ValueError("Workflow edges must connect existing nodes.")
+
+    return topological_node_ids(definition)
+
+
+def workflow_requires_connection(definition: StoredWorkflowDefinition) -> bool:
+    return any(node.type == "agent" for node in definition.nodes)
+
+
+def topological_node_ids(definition: StoredWorkflowDefinition) -> list[str]:
+    node_ids = [node.id for node in definition.nodes]
+    outgoing: dict[str, list[str]] = defaultdict(list)
+    indegree = {node_id: 0 for node_id in node_ids}
+    for edge in definition.edges:
+        outgoing[edge.source].append(edge.target)
+        indegree[edge.target] += 1
+
+    node_order = {node_id: index for index, node_id in enumerate(node_ids)}
+    ready = deque(
+        sorted(
+            [node_id for node_id, degree in indegree.items() if degree == 0],
+            key=lambda node_id: node_order[node_id],
+        )
+    )
+    ordered: list[str] = []
+    while ready:
+        node_id = ready.popleft()
+        ordered.append(node_id)
+        for target in sorted(
+            outgoing[node_id], key=lambda node_id: node_order[node_id]
+        ):
+            indegree[target] -= 1
+            if indegree[target] == 0:
+                ready.append(target)
+
+    if len(ordered) != len(node_ids):
+        raise ValueError("Workflow cannot contain cycles.")
+    return ordered
+
+
+async def run_workflow_definition(
+    *,
+    completion: CompletionCallable | None,
+    connection: ProviderConnection | None,
+    definition: StoredWorkflowDefinition,
+    workflow_id: str,
+) -> WorkflowRunResponse:
+    ordered_ids = validate_workflow_definition(definition)
+    if workflow_requires_connection(definition) and connection is None:
+        raise ValueError("Choose a provider and model before running.")
+
+    nodes = {node.id: node for node in definition.nodes}
+    incoming_edges = edges_by_target(definition.edges)
+    results: dict[str, WorkflowNodeRunResult] = {
+        node.id: WorkflowNodeRunResult(id=node.id, status="pending")
+        for node in definition.nodes
+    }
+    outputs: dict[str, str] = {}
+    named_outputs: dict[str, str] = {}
+
+    for node_id in ordered_ids:
+        node = nodes[node_id]
+        results[node.id] = WorkflowNodeRunResult(id=node.id, status="running")
+        try:
+            output = await run_node(
+                completion=completion,
+                connection=connection,
+                incoming_edges=incoming_edges[node.id],
+                node=node,
+                outputs=outputs,
+            )
+        except Exception as error:
+            results[node.id] = WorkflowNodeRunResult(
+                error=str(error) or "Node could not be completed.",
+                id=node.id,
+                status="failed",
+            )
+            return WorkflowRunResponse(
+                node_results=list(results.values()),
+                outputs=named_outputs,
+                status="failed",
+                workflow_id=workflow_id,
+            )
+        outputs[node.id] = output
+        if node.type == "output":
+            named_outputs[node_output_key(node)] = output
+        results[node.id] = WorkflowNodeRunResult(
+            id=node.id,
+            output=output,
+            status="success",
+        )
+
+    return WorkflowRunResponse(
+        node_results=list(results.values()),
+        outputs=named_outputs,
+        status="success",
+        workflow_id=workflow_id,
+    )
+
+
+async def run_node(
+    *,
+    completion: CompletionCallable | None,
+    connection: ProviderConnection | None,
+    incoming_edges: list[StoredWorkflowEdge],
+    node: StoredWorkflowNode,
+    outputs: Mapping[str, str],
+) -> str:
+    if node.type == "input":
+        return node_data_text(node, "default_value")
+    if node.type == "agent":
+        if connection is None:
+            raise ValueError("Choose a provider and model before running.")
+        prompt = render_template(
+            node_data_text(node, "prompt")
+            or joined_upstream_outputs(incoming_edges, outputs),
+            outputs,
+        )
+        response = await complete_chat(
+            connection,
+            [ChatMessage(role="user", content=prompt)],
+            completion=completion,
+        )
+        return response.content
+    if node.type == "merge":
+        upstream = upstream_outputs(incoming_edges, outputs)
+        if node_data_text(node, "merge_strategy") == "json":
+            return merge_json_outputs(upstream)
+        return "\n".join(output for output in upstream if output)
+    if node.type == "output":
+        return joined_upstream_outputs(incoming_edges, outputs)
+    raise ValueError("Node type is not supported.")
+
+
+def edges_by_target(
+    edges: list[StoredWorkflowEdge],
+) -> dict[str, list[StoredWorkflowEdge]]:
+    grouped: dict[str, list[StoredWorkflowEdge]] = defaultdict(list)
+    for edge in edges:
+        grouped[edge.target].append(edge)
+    return grouped
+
+
+def node_data_text(node: StoredWorkflowNode, key: str) -> str:
+    value = node.data.get(key, "")
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    return str(value)
+
+
+def node_output_key(node: StoredWorkflowNode) -> str:
+    return node_data_text(node, "output_key") or node.id
+
+
+def upstream_outputs(
+    incoming_edges: list[StoredWorkflowEdge],
+    outputs: Mapping[str, str],
+) -> list[str]:
+    return [outputs[edge.source] for edge in incoming_edges if edge.source in outputs]
+
+
+def joined_upstream_outputs(
+    incoming_edges: list[StoredWorkflowEdge],
+    outputs: Mapping[str, str],
+) -> str:
+    return "\n".join(
+        output for output in upstream_outputs(incoming_edges, outputs) if output
+    )
+
+
+def render_template(template: str, outputs: Mapping[str, str]) -> str:
+    return PLACEHOLDER_PATTERN.sub(
+        lambda match: outputs.get(match.group(1), ""),
+        template,
+    )
+
+
+def merge_json_outputs(upstream: list[str]) -> str:
+    merged: dict[str, object] = {}
+    for output in upstream:
+        try:
+            parsed = json.loads(output)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(parsed, dict):
+            merged.update(parsed)
+    return json.dumps(merged, ensure_ascii=False)