flowent 0.1.4 → 0.2.0

package/backend/tests/test_agent_tools.py

@@ -1,5 +1,6 @@
 import asyncio
 import json
+import logging
 import subprocess
 import time
 from pathlib import Path
@@ -11,7 +12,7 @@ from flowent.agent import FLOWENT_AGENT_SYSTEM_PROMPT, run_agent_stream
 from flowent.llm import ProviderConnection, ProviderFormat
 from flowent.main import create_app
 from flowent.sandbox import SandboxCommand, SandboxRunner
-from flowent.tools import ToolContext, run_tool
+from flowent.tools import ToolContext, ToolResult, run_tool
 
 
 def stream_events(content: str) -> list[dict[str, object]]:
@@ -249,6 +250,19 @@ def test_sandbox_command_omits_proc_mount_when_preflight_reports_permission_erro
     assert "--proc" not in command.args
 
 
+def test_sandbox_command_binds_writable_socket_path(tmp_path, monkeypatch) -> None:
+    socket_path = tmp_path / "docker.sock"
+    socket_path.touch()
+    runner = SandboxRunner(cwd=tmp_path, writable_roots=[socket_path])
+    monkeypatch.setattr("flowent.sandbox.sandbox_supports_proc_mount", lambda: False)
+
+    command = runner.build_command(["/bin/true"])
+
+    bind_index = command.args.index(str(socket_path))
+    assert command.args[bind_index - 1] == "--bind"
+    assert command.args[bind_index + 1] == str(socket_path)
+
+
 def test_sandbox_proc_preflight_does_not_hide_non_proc_errors(
     tmp_path, monkeypatch
 ) -> None:
@@ -791,6 +805,141 @@ def test_agent_continues_until_final_text_after_multiple_tool_rounds(
     assert events[-1]["data"]["message"]["content"] == "The notes are ready."
 
 
+@pytest.mark.anyio
+async def test_agent_logs_model_call_decisions_after_tool_rounds(
+    tmp_path, caplog
+) -> None:
+    (tmp_path / "notes.txt").write_text("Launch notes")
+    captured_requests: list[dict[str, object]] = []
+    caplog.set_level(logging.INFO, logger="flowent.agent")
+
+    async def fake_completion(**request: object) -> object:
+        captured_requests.append(request)
+
+        async def chunks() -> object:
+            if len(captured_requests) == 1:
+                yield tool_call_chunk("read_file", {"path": "notes.txt"})
+            else:
+                yield text_chunk("The notes are ready.")
+
+        return chunks()
+
+    events = [
+        event
+        async for event in run_agent_stream(
+            completion=fake_completion,
+            connection=ProviderConnection(
+                model="gpt-5.1",
+                name="Provider",
+                provider=ProviderFormat.OPENAI,
+                secret_reference="secret",
+            ),
+            cwd=tmp_path,
+            messages=[{"role": "user", "content": "Inspect notes."}],
+        )
+    ]
+    rendered_logs = "\n".join(record.getMessage() for record in caplog.records)
+
+    assert events[-1].data["message"]["content"] == "The notes are ready."
+    assert "Agent model call started" in rendered_logs
+    assert "round=1" in rendered_logs
+    assert "round=2" in rendered_logs
+    assert "decision=run_tools" in rendered_logs
+    assert "decision=final_response" in rendered_logs
+    assert "Agent continuing after tools" in rendered_logs
+
+
+@pytest.mark.anyio
+async def test_agent_logs_model_call_failure_after_tool_result(
+    tmp_path, caplog
+) -> None:
+    (tmp_path / "notes.txt").write_text("Launch notes")
+    captured_requests: list[dict[str, object]] = []
+    caplog.set_level(logging.INFO, logger="flowent.agent")
+
+    async def fake_completion(**request: object) -> object:
+        captured_requests.append(request)
+
+        async def chunks() -> object:
+            if len(captured_requests) == 1:
+                yield tool_call_chunk("read_file", {"path": "notes.txt"})
+                return
+            raise RuntimeError("stream request failed")
+
+        return chunks()
+
+    with pytest.raises(RuntimeError, match="stream request failed"):
+        [
+            event
+            async for event in run_agent_stream(
+                completion=fake_completion,
+                connection=ProviderConnection(
+                    model="gpt-5.1",
+                    name="Provider",
+                    provider=ProviderFormat.OPENAI,
+                    secret_reference="secret",
+                ),
+                cwd=tmp_path,
+                messages=[{"role": "user", "content": "Inspect notes."}],
+            )
+        ]
+    rendered_logs = "\n".join(record.getMessage() for record in caplog.records)
+
+    assert len(captured_requests) == 2
+    assert "Agent model call failed" in rendered_logs
+    assert "round=2" in rendered_logs
+    assert "chunk_count=0" in rendered_logs
+
+
+@pytest.mark.anyio
+async def test_agent_does_not_log_final_response_when_responses_stream_fails(
+    tmp_path, caplog, fake_litellm_responses_transformer
+) -> None:
+    caplog.set_level(logging.INFO, logger="flowent.agent")
+
+    async def fake_completion(**request: object) -> object:
+        async def chunks() -> object:
+            from litellm.completion_extras.litellm_responses_transformation.transformation import (
+                OpenAiResponsesToChatCompletionStreamIterator,
+            )
+
+            yield text_chunk("Partial answer.")
+            yield OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(
+                {
+                    "response": {
+                        "error": {
+                            "code": "upstream_error",
+                            "message": "Upstream request failed",
+                        },
+                        "status": "failed",
+                    },
+                    "type": "response.failed",
+                }
+            )
+
+        return chunks()
+
+    with pytest.raises(RuntimeError, match="Upstream request failed"):
+        [
+            event
+            async for event in run_agent_stream(
+                completion=fake_completion,
+                connection=ProviderConnection(
+                    model="gpt-5.1",
+                    name="Provider",
+                    provider=ProviderFormat.OPENAI,
+                    secret_reference="secret",
+                ),
+                cwd=tmp_path,
+                messages=[{"role": "user", "content": "Inspect notes."}],
+            )
+        ]
+    rendered_logs = "\n".join(record.getMessage() for record in caplog.records)
+
+    assert "Agent model call failed" in rendered_logs
+    assert "decision=final_response" not in rendered_logs
+
+
 def test_agent_finishes_without_tools(tmp_path, monkeypatch) -> None:
     monkeypatch.setenv("FLOWENT_DATA_DIR", str(tmp_path / "data"))
     monkeypatch.chdir(tmp_path)
@@ -901,6 +1050,69 @@ def test_tool_failure_is_reported_and_agent_continues(tmp_path, monkeypatch) ->
     assert events[-1]["data"]["message"]["content"] == "I could not read it."
 
 
+@pytest.mark.anyio
+async def test_approval_denial_result_is_sent_to_agent(tmp_path) -> None:
+    captured_requests: list[dict[str, object]] = []
+
+    async def fake_completion(**request: object) -> object:
+        captured_requests.append(request)
+
+        async def chunks() -> object:
+            if len(captured_requests) == 1:
+                yield tool_call_chunk(
+                    "shell_command",
+                    {"command": "rm -rf /important"},
+                )
+            else:
+                yield text_chunk("I need explicit approval for that risk.")
+
+        return chunks()
+
+    async def denying_tool_runner(
+        name: str,
+        arguments: dict[str, object],
+        context: ToolContext,
+    ) -> ToolResult:
+        return ToolResult(
+            content=(
+                "Automatic approval review denied this action as high risk: "
+                "The command can delete broad data. The agent must not work around "
+                "this denial."
+            ),
+            ok=False,
+            title="Denied by reviewer",
+        )
+
+    events = [
+        event
+        async for event in run_agent_stream(
+            completion=fake_completion,
+            connection=ProviderConnection(
+                model="gpt-5.1",
+                name="Provider",
+                provider=ProviderFormat.OPENAI,
+                secret_reference="secret",
+            ),
+            cwd=tmp_path,
+            messages=[{"role": "user", "content": "Delete the important directory."}],
+            tool_runner=denying_tool_runner,
+        )
+    ]
+
+    assert len(captured_requests) == 2
+    assert captured_requests[1]["messages"][-1]["role"] == "tool"
+    assert "Automatic approval review denied this action" in str(
+        captured_requests[1]["messages"][-1]["content"]
+    )
+    assert "must not work around" in str(
+        captured_requests[1]["messages"][-1]["content"]
+    )
+    assert events[-2].data["content"] == "I need explicit approval for that risk."
+    assert events[-1].data["message"]["content"] == (
+        "I need explicit approval for that risk."
+    )
+
+
 def test_update_plan_outputs_plan_state(tmp_path) -> None:
     result = run_tool(
         "update_plan",

package/backend/tests/test_approval.py

@@ -0,0 +1,283 @@
+import json
+
+import pytest
+
+from flowent.approval import (
+    ApprovalReviewRequest,
+    ApprovalTranscriptEntry,
+    review_approval_request,
+)
+from flowent.llm import ProviderConnection, ProviderFormat
+
+
+def provider_connection() -> ProviderConnection:
+    return ProviderConnection(
+        model="model",
+        name="Provider",
+        provider=ProviderFormat.OPENAI,
+        secret_reference="secret",
+    )
+
+
+@pytest.mark.anyio
+async def test_review_payload_includes_current_user_request_and_transcript(
+    tmp_path,
+) -> None:
+    captured_messages: list[dict[str, object]] = []
+
+    async def fake_completion(**request: object) -> object:
+        captured_messages.extend(request["messages"])
+        return {
+            "choices": [
+                {
+                    "message": {
+                        "content": json.dumps(
+                            {
+                                "risk_level": "low",
+                                "risk_score": 25,
+                                "rationale": "User approved after concrete risk context.",
+                                "evidence": [
+                                    {
+                                        "message": "Assistant explained Docker socket impact.",
+                                        "why": "Establishes informed consent.",
+                                    }
+                                ],
+                            }
+                        ),
+                        "role": "assistant",
+                    }
+                },
+            ],
+        }
+
+    decision = await review_approval_request(
+        provider_connection(),
+        ApprovalReviewRequest(
+            action="additional_permissions",
+            arguments={"command": "docker compose up -d --build"},
+            cwd=tmp_path,
+            tool_name="shell_command",
+            user_request="确认",
+            transcript=[
+                ApprovalTranscriptEntry(
+                    role="assistant",
+                    content=(
+                        "This will recreate the dev container, write to the Docker "
+                        "socket, and briefly interrupt the local service."
+                    ),
+                ),
+                ApprovalTranscriptEntry(role="user", content="确认"),
+            ],
+            write_paths=[tmp_path / "docker.sock"],
+        ),
+        completion=fake_completion,
+    )
+
+    assert decision.decision == "approved"
+    assert decision.risk_level == "low"
+    assert decision.risk_score == 25
+    assert "informed of the concrete risk" in str(captured_messages[0]["content"])
+    payload = json.loads(str(captured_messages[-1]["content"]))
+    assert payload["user_request"] == "确认"
+    assert payload["transcript"][-1] == {"role": "user", "content": "确认"}
+
+
+@pytest.mark.anyio
+async def test_concrete_docker_socket_confirmation_can_be_approved(tmp_path) -> None:
+    async def fake_completion(**request: object) -> object:
+        return {
+            "choices": [
+                {
+                    "message": {
+                        "content": json.dumps(
+                            {
+                                "risk_level": "medium",
+                                "risk_score": 55,
+                                "rationale": (
+                                    "The user approved after being told the command "
+                                    "will recreate the dev container through Docker."
+                                ),
+                                "evidence": [],
+                            }
+                        ),
+                        "role": "assistant",
+                    }
+                }
+            ]
+        }
+
+    decision = await review_approval_request(
+        provider_connection(),
+        ApprovalReviewRequest(
+            action="additional_permissions",
+            arguments={
+                "command": "docker compose up -d --force-recreate flowent",
+            },
+            cwd=tmp_path,
+            tool_name="shell_command",
+            user_request="确认",
+            transcript=[
+                ApprovalTranscriptEntry(
+                    role="assistant",
+                    content=(
+                        "This will recreate the Flowent dev container through "
+                        "Docker and may briefly interrupt the running service."
+                    ),
+                ),
+                ApprovalTranscriptEntry(role="user", content="确认"),
+            ],
+            write_paths=[tmp_path / "docker.sock"],
+        ),
+        completion=fake_completion,
+    )
+
+    assert decision.decision == "approved"
+    assert decision.risk_level == "medium"
+    assert decision.risk_score == 55
+
+
+@pytest.mark.anyio
+async def test_vague_confirmation_without_concrete_risk_context_is_denied(
+    tmp_path,
+) -> None:
+    captured_payload: dict[str, object] = {}
+
+    async def fake_completion(**request: object) -> object:
+        captured_payload.update(json.loads(str(request["messages"][-1]["content"])))
+        return {
+            "choices": [
+                {
+                    "message": {
+                        "content": json.dumps(
+                            {
+                                "risk_level": "high",
+                                "risk_score": 82,
+                                "rationale": (
+                                    "The transcript only contains a vague confirmation "
+                                    "and no concrete Docker risk explanation."
+                                ),
+                                "evidence": [],
+                            }
+                        ),
+                        "role": "assistant",
+                    }
+                }
+            ]
+        }
+
+    decision = await review_approval_request(
+        provider_connection(),
+        ApprovalReviewRequest(
+            action="additional_permissions",
+            arguments={
+                "command": "docker compose up -d --force-recreate flowent",
+            },
+            cwd=tmp_path,
+            tool_name="shell_command",
+            user_request="确认",
+            transcript=[ApprovalTranscriptEntry(role="user", content="确认")],
+            write_paths=[tmp_path / "docker.sock"],
+        ),
+        completion=fake_completion,
+    )
+
+    assert decision.decision == "denied"
+    assert decision.risk_level == "high"
+    assert decision.risk_score == 82
+    assert captured_payload["transcript"] == [{"role": "user", "content": "确认"}]
+
+
+@pytest.mark.anyio
+async def test_broad_destructive_action_with_vague_confirmation_is_denied(
+    tmp_path,
+) -> None:
+    async def fake_completion(**request: object) -> object:
+        return {
+            "choices": [
+                {
+                    "message": {
+                        "content": json.dumps(
+                            {
+                                "risk_level": "high",
+                                "risk_score": 96,
+                                "rationale": (
+                                    "The action can delete broad data and the user "
+                                    "did not approve that concrete destructive risk."
+                                ),
+                                "evidence": [
+                                    {
+                                        "message": "rm -rf /var/lib/postgresql",
+                                        "why": "Broad destructive write outside the task.",
+                                    }
+                                ],
+                            }
+                        ),
+                        "role": "assistant",
+                    }
+                }
+            ]
+        }
+
+    decision = await review_approval_request(
+        provider_connection(),
+        ApprovalReviewRequest(
+            action="sandbox_failure",
+            arguments={"command": "rm -rf /var/lib/postgresql"},
+            cwd=tmp_path,
+            tool_name="shell_command",
+            tool_result="Read-only file system",
+            user_request="确认",
+            transcript=[ApprovalTranscriptEntry(role="user", content="确认")],
+        ),
+        completion=fake_completion,
+    )
+
+    assert decision.decision == "denied"
+    assert decision.risk_level == "high"
+    assert decision.risk_score == 96
+
+
+@pytest.mark.anyio
+async def test_invalid_reviewer_json_is_denied(tmp_path) -> None:
+    async def fake_completion(**request: object) -> object:
+        return {
+            "choices": [
+                {"message": {"content": "approved", "role": "assistant"}},
+            ],
+        }
+
+    decision = await review_approval_request(
+        provider_connection(),
+        ApprovalReviewRequest(
+            action="sandbox_failure",
+            arguments={"command": "touch file.txt"},
+            cwd=tmp_path,
+            tool_name="shell_command",
+            tool_result="Read-only file system",
+        ),
+        completion=fake_completion,
+    )
+
+    assert decision.decision == "denied"
+    assert "valid JSON" in decision.reason
+
+
+@pytest.mark.anyio
+async def test_reviewer_call_failure_is_denied(tmp_path) -> None:
+    async def fake_completion(**request: object) -> object:
+        raise RuntimeError("model unavailable")
+
+    decision = await review_approval_request(
+        provider_connection(),
+        ApprovalReviewRequest(
+            action="edit",
+            arguments={"patch": "*** Begin Patch\n*** End Patch"},
+            cwd=tmp_path,
+            tool_name="apply_patch",
+            write_paths=[tmp_path / "outside"],
+        ),
+        completion=fake_completion,
+    )
+
+    assert decision.decision == "denied"
+    assert "model unavailable" in decision.reason