@miller-tech/uap 1.118.1 → 1.119.0

package/tools/agents/tests/test_cycle_break_exploration.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+"""Unit tests for cycle-break tool narrowing preserving the exploration escape hatch.
+
+Regression: the CYCLE BREAK narrowing removed every cycling tool by name (and the
+whole read-only class if any cycling tool was read-only). Because Bash can be a
+cycling tool, the agent lost the Bash tool entirely and could no longer run any
+bash command to explore the filesystem. The narrowing must keep the open-ended
+exploration escape hatch (Bash/WebFetch/Agent) available so the agent can make a
+DIFFERENT move — the cycle *hint* handles "vary the command".
+"""
+
+import importlib.util
+import unittest
+from pathlib import Path
+
+
+def _load_proxy_module():
+    proxy_path = Path(__file__).resolve().parents[1] / "scripts" / "anthropic_proxy.py"
+    spec = importlib.util.spec_from_file_location("anthropic_proxy", proxy_path)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+proxy = _load_proxy_module()
+
+
+def _tools(*names):
+    return [{"function": {"name": n}} for n in names]
+
+
+def _names(tools):
+    return sorted(t["function"]["name"] for t in tools)
+
+
+class TestCycleBreakExploration(unittest.TestCase):
+    def test_cycling_bash_keeps_bash(self):
+        """The core bug: cycling on Bash must NOT remove the Bash tool."""
+        tools = _tools("Bash", "Read", "Write")
+        narrowed, expanded = proxy._narrow_tools_for_cycle_break(tools, {"Bash"}, set())
+        self.assertIn("Bash", _names(narrowed))
+        self.assertFalse(expanded)  # Bash is not read-only, so no class expansion
+
+    def test_read_only_cycle_expands_class_but_keeps_exploration(self):
+        """Cycling a read-only tool drops the whole read-only class, yet the
+        exploration escape hatch (Bash/WebFetch/Agent) survives."""
+        tools = _tools("Read", "Grep", "Glob", "Bash", "WebFetch", "Agent", "Write")
+        narrowed, expanded = proxy._narrow_tools_for_cycle_break(tools, {"Grep"}, set())
+        self.assertTrue(expanded)
+        kept = _names(narrowed)
+        # read-only class dropped
+        for n in ("Read", "Grep", "Glob"):
+            self.assertNotIn(n, kept)
+        # exploration escape hatch + writes preserved
+        for n in ("Bash", "WebFetch", "Agent", "Write"):
+            self.assertIn(n, kept)
+
+    def test_session_ban_is_honored_even_for_exploration_tools(self):
+        """An explicit session ban is a stronger, deliberate signal than cycling
+        and IS honored — even for an exploration tool. (Only the cycling path
+        preserves the escape hatch.)"""
+        tools = _tools("Bash", "Read", "Write")
+        narrowed, _ = proxy._narrow_tools_for_cycle_break(tools, set(), {"Bash"})
+        self.assertNotIn("Bash", _names(narrowed))
+
+    def test_cycling_bash_kept_even_when_another_tool_is_banned(self):
+        """A cycling Bash is preserved even while an unrelated tool is banned."""
+        tools = _tools("Bash", "Read", "Task")
+        narrowed, _ = proxy._narrow_tools_for_cycle_break(tools, {"Bash"}, {"Task"})
+        kept = _names(narrowed)
+        self.assertIn("Bash", kept)       # cycling exploration hatch preserved
+        self.assertNotIn("Task", kept)    # explicit ban honored
+
+    def test_case_insensitive_matching(self):
+        """Lowercase 'bash' in the cycling set must still match a 'Bash' tool
+        (and keep it), and 'read' must drop a 'Read' tool."""
+        tools = _tools("Bash", "Read")
+        narrowed, expanded = proxy._narrow_tools_for_cycle_break(
+            tools, {"bash", "read"}, set()
+        )
+        kept = _names(narrowed)
+        self.assertIn("Bash", kept)      # exploration hatch kept despite cycling
+        self.assertNotIn("Read", kept)   # read-only dropped
+        self.assertTrue(expanded)
+
+    def test_floor_keeps_last_action_tool_when_ban_would_strip_it(self):
+        """Floor invariant: if honoring a ban would leave NO exploration tool and
+        NO write tool, keep the original set — a loop-breaker must never strand
+        the agent with no way to make a different move."""
+        tools = _tools("Bash", "Read")  # Read is read-only, not an action path
+        narrowed, _ = proxy._narrow_tools_for_cycle_break(tools, set(), {"Bash"})
+        self.assertIn("Bash", _names(narrowed))  # floor restored the escape hatch
+
+    def test_floor_not_triggered_when_a_write_tool_survives(self):
+        """The ban IS honored when another action path (a write tool) remains."""
+        tools = _tools("Bash", "Read", "Write")
+        narrowed, _ = proxy._narrow_tools_for_cycle_break(tools, set(), {"Bash"})
+        kept = _names(narrowed)
+        self.assertNotIn("Bash", kept)  # honored: Write still gives an action path
+        self.assertIn("Write", kept)
+
+    def test_tool_class_taxonomy_is_pairwise_disjoint(self):
+        """Regression insurance: the three tool-class sets must never overlap, or
+        convergence accounting / cycle-break scope would silently change."""
+        ro = {c.lower() for c in proxy._READ_ONLY_TOOL_CLASS}
+        wr = {c.lower() for c in proxy._WRITE_TOOL_CLASS}
+        ex = {c.lower() for c in proxy._EXPLORATION_ESCAPE_TOOLS}
+        self.assertEqual(ro & wr, set())
+        self.assertEqual(ro & ex, set())
+        self.assertEqual(wr & ex, set())
+
+    def test_non_exploration_non_readonly_tool_dropped(self):
+        """A cycling tool that is neither read-only nor an exploration hatch is
+        dropped, and the read-only class is NOT expanded."""
+        tools = _tools("SomeTool", "Read", "Bash")
+        narrowed, expanded = proxy._narrow_tools_for_cycle_break(
+            tools, {"SomeTool"}, set()
+        )
+        kept = _names(narrowed)
+        self.assertNotIn("SomeTool", kept)
+        self.assertIn("Read", kept)      # class not expanded, Read survives
+        self.assertIn("Bash", kept)
+        self.assertFalse(expanded)
+
+
+class TestAutoBanExemption(unittest.TestCase):
+    """The session-ban accumulator is cycling-derived (ban after N detections).
+    Exploration escape-hatch tools must never be auto-banned, or the 'cannot
+    explore the filesystem' bug reappears on the Nth cycle."""
+
+    def test_bash_never_auto_banned_even_far_past_threshold(self):
+        self.assertFalse(proxy._should_auto_ban("bash", cycle_count=99, ban_at=3))
+        self.assertFalse(proxy._should_auto_ban("Bash", cycle_count=3, ban_at=3))
+
+    def test_other_exploration_tools_never_auto_banned(self):
+        for name in ("webfetch", "WebSearch", "agent", "task"):
+            self.assertFalse(proxy._should_auto_ban(name, cycle_count=10, ban_at=3))
+
+    def test_non_exploration_tool_banned_at_threshold(self):
+        self.assertTrue(proxy._should_auto_ban("edit", cycle_count=3, ban_at=3))
+        self.assertTrue(proxy._should_auto_ban("read", cycle_count=3, ban_at=3))
+
+    def test_non_exploration_tool_not_banned_below_threshold(self):
+        self.assertFalse(proxy._should_auto_ban("edit", cycle_count=2, ban_at=3))
+
+
+if __name__ == "__main__":
+    unittest.main()

package/tools/agents/tests/test_deferral_break.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+"""Unit tests for the DEFERRAL-BREAK guardrail (Fix A) and the no-tool-turn
+recon-convergence counting (Fix B).
+
+Regression: a model could end a turn with plain prose that DEFERS the work --
+"I need more exploration cycles to complete the plan" -- with no tool call. That
+stall was invisible to STUCK-BREAK (not a loop-admission phrase) and to
+recon-convergence (a no-tool turn never advanced its streak), so it silently
+halted a hands-free build. The deferral-break detects the no-tool capitulation
+and forces the next turn to take a concrete action; Fix B makes prose-only turns
+count toward convergence so prolonged stalls also escalate.
+"""
+
+import importlib.util
+import unittest
+from pathlib import Path
+
+
+def _load_proxy_module():
+    proxy_path = Path(__file__).resolve().parents[1] / "scripts" / "anthropic_proxy.py"
+    spec = importlib.util.spec_from_file_location("anthropic_proxy", proxy_path)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+proxy = _load_proxy_module()
+
+
+class TestDeferralPhraseRegex(unittest.TestCase):
+    def test_matches_the_reported_stall(self):
+        self.assertTrue(
+            proxy._DEFERRAL_PHRASE_RE.search(
+                "I need more exploration cycles to complete the plan. "
+            )
+        )
+
+    def test_matches_close_kin(self):
+        for s in (
+            "Let me continue exploring the codebase before I proceed.",
+            "I'll need a few more passes to complete the plan.",
+            "I need to gather more research before writing anything.",
+            "Once I have explored more, I can start.",
+        ):
+            self.assertTrue(proxy._DEFERRAL_PHRASE_RE.search(s), s)
+
+    def test_does_not_match_progress_narration(self):
+        for s in (
+            "Done. I created src/index.ts and wired the routes.",
+            "Here is the summary of what I built.",
+            "Running the tests now.",
+            "The build passes and all items are complete.",
+        ):
+            self.assertIsNone(proxy._DEFERRAL_PHRASE_RE.search(s), s)
+
+    def test_does_not_match_generic_time_or_steps_question(self):
+        # A legitimate clarifying question must not trip the guard (was a false
+        # positive when bare "time"/"steps" were object tokens).
+        for s in (
+            "I need more time to review your requirements — could you clarify X?",
+            "There are a few more steps, let me know which environment to target.",
+        ):
+            self.assertIsNone(proxy._DEFERRAL_PHRASE_RE.search(s), s)
+
+
+class TestDeferralStreak(unittest.TestCase):
+    def test_no_tool_deferral_increments_and_fires(self):
+        mon = proxy.SessionMonitor()
+        mon.note_deferral_signal(
+            "I need more exploration cycles to complete the plan.", had_tool_call=False
+        )
+        self.assertEqual(mon.deferral_streak, 1)
+        should, reason = mon.should_force_deferral_break()
+        self.assertTrue(should)
+        self.assertIn("deferral", reason)
+
+    def test_tool_call_turn_resets_streak(self):
+        mon = proxy.SessionMonitor()
+        mon.note_deferral_signal("let me continue exploring", had_tool_call=False)
+        self.assertEqual(mon.deferral_streak, 1)
+        # Model actually acted this turn -> not deferring anymore.
+        mon.note_deferral_signal("let me continue exploring", had_tool_call=True)
+        self.assertEqual(mon.deferral_streak, 0)
+        self.assertFalse(mon.should_force_deferral_break()[0])
+
+    def test_non_matching_text_resets_streak(self):
+        mon = proxy.SessionMonitor()
+        mon.note_deferral_signal("I need more cycles", had_tool_call=False)
+        self.assertEqual(mon.deferral_streak, 1)
+        mon.note_deferral_signal("All done, files written.", had_tool_call=False)
+        self.assertEqual(mon.deferral_streak, 0)
+
+
+class TestDeferralInjection(unittest.TestCase):
+    def test_forces_tool_choice_and_directive(self):
+        mon = proxy.SessionMonitor()
+        mon.note_deferral_signal("I need more exploration cycles", had_tool_call=False)
+        body = {
+            "tools": [{"function": {"name": "Write"}}],
+            "tool_choice": "auto",
+            "messages": [{"role": "system", "content": "sys"}],
+        }
+        proxy._maybe_inject_deferral_break(body, mon)
+        self.assertEqual(body["tool_choice"], "required")
+        self.assertIn("CONTINUE AUTONOMOUSLY", body["messages"][0]["content"])
+        self.assertEqual(mon.deferral_break_fires, 1)
+
+    def test_does_not_fire_below_threshold(self):
+        mon = proxy.SessionMonitor()  # streak 0
+        body = {"tools": [{"function": {"name": "Write"}}], "tool_choice": "auto", "messages": []}
+        proxy._maybe_inject_deferral_break(body, mon)
+        self.assertEqual(body["tool_choice"], "auto")
+        self.assertEqual(mon.deferral_break_fires, 0)
+
+    def test_yields_to_stuck_break(self):
+        mon = proxy.SessionMonitor()
+        mon.deferral_streak = 5
+        mon.self_stuck_streak = 99  # stuck-break will fire this turn
+        body = {"tools": [{"function": {"name": "Write"}}], "tool_choice": "auto", "messages": []}
+        proxy._maybe_inject_deferral_break(body, mon)
+        # deferral defers to the more-urgent stuck-break (prose exit).
+        self.assertEqual(body["tool_choice"], "auto")
+        self.assertEqual(mon.deferral_break_fires, 0)
+
+    def test_disabled_when_off(self):
+        mon = proxy.SessionMonitor()
+        # Simulate the phrase but with the guard disabled.
+        original = proxy.PROXY_DEFERRAL_BREAK
+        try:
+            proxy.PROXY_DEFERRAL_BREAK = False
+            mon.note_deferral_signal("I need more exploration cycles", had_tool_call=False)
+            self.assertEqual(mon.deferral_streak, 0)
+            self.assertFalse(mon.should_force_deferral_break()[0])
+        finally:
+            proxy.PROXY_DEFERRAL_BREAK = original
+
+
+class TestDeferralYieldsAndToolAbsence(unittest.TestCase):
+    def test_yields_to_recon_convergence(self):
+        mon = proxy.SessionMonitor()
+        mon.deferral_streak = 5
+        # Push the no-write streak into recon-convergence territory.
+        mon.consecutive_no_write_turns = proxy.PROXY_RECON_CONVERGENCE_THRESHOLD
+        body = {
+            "tools": [{"function": {"name": "Write"}}],
+            "tool_choice": "auto",
+            "messages": [],
+        }
+        proxy._maybe_inject_deferral_break(body, mon)
+        # recon owns the turn -> deferral must not fire or touch tool_choice.
+        self.assertEqual(body["tool_choice"], "auto")
+        self.assertEqual(mon.deferral_break_fires, 0)
+
+    def test_no_tools_injects_directive_but_leaves_tool_choice(self):
+        mon = proxy.SessionMonitor()
+        mon.deferral_streak = 1  # recon NOT pending (streak stays 0)
+        body = {"tools": [], "tool_choice": "auto", "messages": []}
+        proxy._maybe_inject_deferral_break(body, mon)
+        # No tools to force, so tool_choice is left as-is, but the directive and
+        # the telemetry counter still apply (the nudge is still worth sending).
+        self.assertEqual(body["tool_choice"], "auto")
+        self.assertEqual(mon.deferral_break_fires, 1)
+        self.assertIn("CONTINUE AUTONOMOUSLY", body["messages"][0]["content"])
+
+
+class TestNoToolTurnConvergence(unittest.TestCase):
+    def test_no_tool_turn_advances_convergence_streak(self):
+        mon = proxy.SessionMonitor()
+        before = mon.consecutive_no_write_turns
+        mon.note_no_tool_turn()
+        mon.note_no_tool_turn()
+        self.assertEqual(mon.consecutive_no_write_turns, before + 2)
+
+
+if __name__ == "__main__":
+    unittest.main()