@miller-tech/uap 1.148.2 → 1.148.5

package/tools/agents/scripts/anthropic_proxy.py

@@ -274,6 +274,49 @@ _RATE_LIMITED_API_RE = re.compile(r"api\.github\.com", re.IGNORECASE)
 PROXY_STUCK_TEXT_THRESHOLD = int(os.environ.get("PROXY_STUCK_TEXT_THRESHOLD", "2"))
 PROXY_STUCK_API_THRESHOLD = int(os.environ.get("PROXY_STUCK_API_THRESHOLD", "3"))
 
+# ---------------------------------------------------------------------------
+# ERROR-LOOP guardrail: the model edits, runs a command, hits the SAME failure,
+# edits again (a DIFFERENT edit), runs, hits the same failure — for many turns.
+# The identical-tool-call loop/cycle detectors never trip because each edit
+# differs; the model looks productive while never addressing the real blocker
+# (observed live: a duplicate function declaration in test.js kept the test from
+# compiling, and the model chased an unrelated error in another file for 35 min).
+# Detect a repeated tool_RESULT error SIGNATURE (normalized: paths/line numbers/
+# digits/hex stripped so it's edit-invariant) and, past the threshold, inject a
+# nudge to STOP editing and re-read the failing file/output end-to-end.
+# PROXY_ERROR_LOOP=off to disable.
+PROXY_ERROR_LOOP = os.environ.get("PROXY_ERROR_LOOP", "on").lower() not in {
+    "0", "off", "false", "no",
+}
+PROXY_ERROR_LOOP_THRESHOLD = int(os.environ.get("PROXY_ERROR_LOOP_THRESHOLD", "3"))
+
+_ERROR_LINE_RE = re.compile(
+    r"^.*(?:Error|Exception|Traceback|FAILED|assert(?:ion)?|SyntaxError|"
+    r"TypeError|ReferenceError|not found|cannot find|undefined|panic|"
+    r"fatal|command not found|exit code [1-9]).*$",
+    re.IGNORECASE | re.MULTILINE,
+)
+
+
+def _error_signature(text: str) -> str:
+    """Edit-invariant signature of the first error line in a tool result.
+
+    Normalizes away paths, line:col numbers, hex, and bare digits so that the
+    SAME underlying failure produces the SAME signature across turns even as the
+    model edits different code around it. Returns "" when no error line is found
+    (a passing result resets the streak)."""
+    if not text:
+        return ""
+    m = _ERROR_LINE_RE.search(text)
+    if not m:
+        return ""
+    line = m.group(0)
+    line = re.sub(r"(/[^\s:]+)+", "<path>", line)          # unix paths
+    line = re.sub(r"\b[0-9a-fA-F]{6,}\b", "<hex>", line)    # hashes/addresses
+    line = re.sub(r"\d+", "#", line)                        # line numbers, counts
+    line = re.sub(r"\s+", " ", line).strip().lower()
+    return line[:200]
+
 # ---------------------------------------------------------------------------
 # DEFERRAL-BREAK guardrail (Fix A): a model can end a turn with plain prose that
 # DEFERS the work instead of doing it -- "I need more exploration cycles to
@@ -769,6 +812,19 @@ PROXY_MALFORMED_TOOL_RETRY_MAX_TOKENS = int(
 PROXY_MALFORMED_TOOL_RETRY_TEMPERATURE = float(
     os.environ.get("PROXY_MALFORMED_TOOL_RETRY_TEMPERATURE", "0")
 )
+# Empty-max_tokens recovery (Option 3): a reasoning model can consume the ENTIRE
+# output budget in reasoning_content and emit NO content/tool_calls
+# (finish_reason=length, empty message) — a thinking-runaway. The upstream
+# --reasoning-budget cap is the primary fix; this is the backstop: on detecting
+# such an empty truncation, retry ONCE with thinking disabled so the turn yields
+# a usable answer instead of an empty response the client blindly re-sends
+# (observed: 529 retry cascades). PROXY_EMPTY_MAXTOKENS_RECOVER=off to disable.
+PROXY_EMPTY_MAXTOKENS_RECOVER = os.environ.get(
+    "PROXY_EMPTY_MAXTOKENS_RECOVER", "on"
+).lower() not in {"0", "off", "false", "no"}
+PROXY_EMPTY_MAXTOKENS_RETRY_MAX_TOKENS = int(
+    os.environ.get("PROXY_EMPTY_MAXTOKENS_RETRY_MAX_TOKENS", "4096")
+)
 PROXY_TOOL_TURN_TEMPERATURE = float(
     os.environ.get("PROXY_TOOL_TURN_TEMPERATURE", "0.3")
 )
@@ -1358,6 +1414,10 @@ class SessionMonitor:
     catastrophic_ctx_streak: int = 0  # Fix F: consecutive turns raw ctx >= finalize ratio
     unexpected_end_turn_count: int = 0  # end_turn without tool_use in active loop
     self_stuck_streak: int = 0  # consecutive assistant texts self-reporting a loop
+    error_signature_streak: int = 0  # consecutive turns with the SAME tool_result error
+    last_error_signature: str = ""  # normalized signature of that recurring error
+    error_loop_fires: int = 0  # telemetry: error-loop nudges injected
+    empty_maxtokens_recoveries: int = 0  # telemetry: thinking-runaway recoveries
     rate_limited_api_streak: int = 0  # consecutive tool calls hitting a rate-limited API host
     stuck_break_fires: int = 0  # monotonic count of forced stuck-breaks
     deferral_streak: int = 0  # consecutive no-tool turns deferring the work (Fix A)
@@ -1599,6 +1659,24 @@ class SessionMonitor:
                     by_tool = self.tool_target_history.setdefault(name, {})
                     by_tool[target] = by_tool.get(target, 0) + 1
 
+    def note_tool_result_error(self, latest_result_text: str) -> None:
+        """Track a repeated tool_result error signature (ERROR-LOOP guardrail).
+
+        Same normalized error as last turn -> increment the streak; a new error
+        or a clean (error-free) result -> reset. Only a SUSTAINED same-failure
+        streak (despite the model's varied edits) trips the nudge."""
+        if not PROXY_ERROR_LOOP:
+            return
+        sig = _error_signature(latest_result_text or "")
+        if sig and sig == self.last_error_signature:
+            self.error_signature_streak += 1
+        elif sig:
+            self.error_signature_streak = 1
+            self.last_error_signature = sig
+        else:
+            self.error_signature_streak = 0
+            self.last_error_signature = ""
+
     def note_assistant_text(self, text: str) -> None:
         """Track the model self-reporting that it is stuck (STUCK-BREAK signal
         (a)). A matching turn increments the streak; a non-matching turn resets
@@ -4905,6 +4983,46 @@ def _maybe_inject_stuck_break(openai_body: dict, monitor: "SessionMonitor") -> N
     logger.warning("STUCK-BREAK: forced terminal turn (%s, fires=%d)", reason, monitor.stuck_break_fires)
 
 
+def _maybe_inject_error_loop_break(openai_body: dict, monitor: "SessionMonitor") -> None:
+    """Nudge the model out of a repeated-same-error loop (ERROR-LOOP guardrail).
+
+    When the SAME normalized tool_result error has recurred >= threshold times
+    despite the model's (varied) edits, its edits aren't addressing the real
+    blocker. Inject a directive to STOP editing and re-read the whole failing
+    file/output — the bug is likely somewhere it hasn't looked. Advisory: does
+    NOT release tool_choice (the model should keep acting, just diagnose first)."""
+    if not PROXY_ERROR_LOOP:
+        return
+    if monitor.error_signature_streak < PROXY_ERROR_LOOP_THRESHOLD:
+        return
+    monitor.error_loop_fires += 1
+    directive = (
+        "\n\nSTOP — the SAME failure has now recurred "
+        + str(monitor.error_signature_streak)
+        + " times in a row despite your edits: \""
+        + monitor.last_error_signature[:120]
+        + "\". Your edits are NOT addressing the real cause. Do NOT make another "
+        "edit yet. FIRST re-read the ENTIRE failing file (and the full error "
+        "output) top-to-bottom — the bug is very likely somewhere you have not "
+        "looked (a duplicate declaration, a wrong import, a different file). "
+        "Only after you can name the exact line causing THIS error, fix that."
+    )
+    msgs = openai_body.get("messages")
+    if not isinstance(msgs, list):
+        msgs = []
+    if msgs and msgs[0].get("role") == "system":
+        msgs[0]["content"] = (msgs[0].get("content") or "") + directive
+    else:
+        msgs.insert(0, {"role": "system", "content": directive.strip()})
+    openai_body["messages"] = msgs
+    logger.warning(
+        "ERROR-LOOP: same failure x%d — injected re-read nudge (sig=%r, fires=%d)",
+        monitor.error_signature_streak,
+        monitor.last_error_signature[:80],
+        monitor.error_loop_fires,
+    )
+
+
 def _maybe_inject_deferral_break(openai_body: dict, monitor: "SessionMonitor") -> None:
     """Convert a turn-ending deferral into forward motion (Fix A).
 
@@ -5897,6 +6015,9 @@ def build_openai_request(
 
     _maybe_inject_stuck_break(openai_body, monitor)
 
+    # ERROR-LOOP: same tool_result failure recurring despite varied edits.
+    _maybe_inject_error_loop_break(openai_body, monitor)
+
     # DEFERRAL-BREAK (Fix A): after stuck-break, drive a no-tool deferral turn
     # into a concrete action. Runs last so it can yield to stuck-break.
     _maybe_inject_deferral_break(openai_body, monitor)
@@ -6030,6 +6151,21 @@ def _record_last_assistant_tool_calls(
     with different arguments (e.g. read(file_a) vs read(file_b)) produces
     distinct fingerprints, preventing false cycle/stagnation detection."""
     messages = anthropic_body.get("messages", [])
+    # ERROR-LOOP tracking: feed the monitor the most recent tool_result text so a
+    # recurring same-failure signature can be detected across (varied) edits.
+    _latest_tr = ""
+    for _m in reversed(messages):
+        _c = _m.get("content")
+        if isinstance(_c, list):
+            _parts = [
+                _extract_text(b.get("content", ""))
+                for b in _c
+                if isinstance(b, dict) and b.get("type") == "tool_result"
+            ]
+            if _parts:
+                _latest_tr = "\n".join(p for p in _parts if p)
+                break
+    monitor.note_tool_result_error(_latest_tr)
     tool_fingerprints = []
     tool_targets: dict[str, str] = {}
     assistant_had_text = False  # Fix B: did the last assistant turn emit prose?
@@ -8159,6 +8295,71 @@ async def _apply_completion_contract_guardrail(
     return retried
 
 
+def _is_empty_maxtokens_response(openai_resp: dict) -> bool:
+    """A finish_reason=length turn that produced NO content and NO tool calls —
+    the thinking-runaway signature (all budget spent in reasoning_content)."""
+    choices = openai_resp.get("choices") or []
+    if not choices:
+        return False
+    choice = choices[0]
+    if (choice.get("finish_reason") or "").lower() != "length":
+        return False
+    msg = choice.get("message") or {}
+    if msg.get("tool_calls"):
+        return False
+    content = msg.get("content")
+    text = content if isinstance(content, str) else ""
+    return len(text.strip()) == 0
+
+
+async def _apply_empty_maxtokens_recovery(
+    client: httpx.AsyncClient,
+    openai_resp: dict,
+    openai_body: dict,
+    anthropic_body: dict,
+    monitor: SessionMonitor,
+    session_id: str,
+) -> dict:
+    """Option 3 backstop: recover an empty finish=length turn by retrying once
+    with thinking OFF, so a residual reasoning-runaway yields a usable answer
+    instead of an empty response the client blindly re-sends (529 cascade)."""
+    if not PROXY_EMPTY_MAXTOKENS_RECOVER:
+        return openai_resp
+    if not _is_empty_maxtokens_response(openai_resp):
+        return openai_resp
+    retry_body = dict(openai_body)
+    retry_body["enable_thinking"] = False
+    ctk = dict(retry_body.get("chat_template_kwargs") or {})
+    ctk["enable_thinking"] = False
+    retry_body["chat_template_kwargs"] = ctk
+    requested = int(openai_body.get("max_tokens") or PROXY_EMPTY_MAXTOKENS_RETRY_MAX_TOKENS)
+    retry_body["max_tokens"] = min(requested, PROXY_EMPTY_MAXTOKENS_RETRY_MAX_TOKENS)
+    logger.warning(
+        "EMPTY-MAX_TOKENS recovery: thinking-runaway (empty finish=length) for "
+        "session %s — retrying once with thinking OFF (max_tokens=%d)",
+        session_id, retry_body["max_tokens"],
+    )
+    try:
+        retry_resp = await client.post(
+            f"{LLAMA_CPP_BASE}/chat/completions",
+            json=retry_body,
+            headers={"Content-Type": "application/json"},
+        )
+        if retry_resp.status_code != 200:
+            return openai_resp
+        retried = retry_resp.json()
+        if not _is_empty_maxtokens_response(retried):
+            monitor.empty_maxtokens_recoveries += 1
+            logger.info(
+                "EMPTY-MAX_TOKENS recovery succeeded for session %s (recoveries=%d)",
+                session_id, monitor.empty_maxtokens_recoveries,
+            )
+            return retried
+    except Exception as exc:  # noqa: BLE001 — recovery is best-effort
+        logger.warning("EMPTY-MAX_TOKENS recovery errored for session %s: %s", session_id, exc)
+    return openai_resp
+
+
 def _sanitize_assistant_messages_for_retry(messages: list[dict]) -> list[dict]:
     """Strip malformed tool-like text from assistant messages to prevent copy-contamination.
 
@@ -10672,6 +10873,14 @@ async def messages(request: Request):
                 monitor,
                 session_id,
             )
+            openai_resp = await _apply_empty_maxtokens_recovery(
+                client,
+                openai_resp,
+                strict_body,
+                body,
+                monitor,
+                session_id,
+            )
 
             openai_resp, was_degenerate = _detect_and_truncate_degenerate_repetition(openai_resp)
             if was_degenerate:
@@ -11116,6 +11325,14 @@ async def messages(request: Request):
             monitor,
             session_id,
         )
+        openai_resp = await _apply_empty_maxtokens_recovery(
+            client,
+            openai_resp,
+            openai_body,
+            body,
+            monitor,
+            session_id,
+        )
 
         choice, _ = _extract_openai_choice(openai_resp)
         finish_reason = choice.get("finish_reason", "")

package/tools/agents/tests/test_empty_maxtokens_recovery.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+"""Empty-max_tokens recovery: a thinking-runaway (finish=length, empty content,
+no tool calls) is retried once with thinking OFF so the turn yields an answer
+instead of an empty response the client blindly re-sends."""
+import asyncio
+import importlib.util
+import unittest
+from pathlib import Path
+
+
+def _load_proxy():
+    p = Path(__file__).resolve().parents[1] / "scripts" / "anthropic_proxy.py"
+    spec = importlib.util.spec_from_file_location("anthropic_proxy", p)
+    m = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(m)
+    return m
+
+
+proxy = _load_proxy()
+
+
+def empty_length_resp():
+    return {"choices": [{"finish_reason": "length", "message": {"role": "assistant", "content": ""}}]}
+
+
+def good_resp(text="here is the answer"):
+    return {"choices": [{"finish_reason": "stop", "message": {"role": "assistant", "content": text}}]}
+
+
+class DetectTests(unittest.TestCase):
+    def test_detects_empty_length(self):
+        self.assertTrue(proxy._is_empty_maxtokens_response(empty_length_resp()))
+
+    def test_ignores_length_with_content(self):
+        self.assertFalse(proxy._is_empty_maxtokens_response(
+            {"choices": [{"finish_reason": "length", "message": {"content": "partial answer"}}]}))
+
+    def test_ignores_length_with_tool_calls(self):
+        self.assertFalse(proxy._is_empty_maxtokens_response(
+            {"choices": [{"finish_reason": "length", "message": {"content": "", "tool_calls": [{"id": "1"}]}}]}))
+
+    def test_ignores_normal_stop(self):
+        self.assertFalse(proxy._is_empty_maxtokens_response(good_resp()))
+
+
+class FakeResp:
+    def __init__(self, body):
+        self.status_code = 200
+        self._body = body
+    def json(self):
+        return self._body
+
+
+class FakeClient:
+    def __init__(self, body):
+        self._body = body
+        self.last_json = None
+    async def post(self, url, json=None, headers=None):  # noqa: A002
+        self.last_json = json
+        return FakeResp(self._body)
+
+
+class RecoveryTests(unittest.TestCase):
+    def setUp(self):
+        self.m = proxy.SessionMonitor()
+
+    def _run(self, resp, client_body, body_in=None):
+        client = FakeClient(client_body)
+        out = asyncio.run(proxy._apply_empty_maxtokens_recovery(
+            client, resp, body_in or {"max_tokens": 16384}, {}, self.m, "t"))
+        return out, client
+
+    def test_recovers_with_thinking_off(self):
+        out, client = self._run(empty_length_resp(), good_resp("recovered!"))
+        self.assertEqual(out["choices"][0]["message"]["content"], "recovered!")
+        # retry request had thinking disabled + a bounded max_tokens
+        self.assertIs(client.last_json["chat_template_kwargs"]["enable_thinking"], False)
+        self.assertIs(client.last_json["enable_thinking"], False)
+        self.assertLessEqual(client.last_json["max_tokens"], proxy.PROXY_EMPTY_MAXTOKENS_RETRY_MAX_TOKENS)
+        self.assertEqual(self.m.empty_maxtokens_recoveries, 1)
+
+    def test_no_retry_on_a_healthy_response(self):
+        out, client = self._run(good_resp("fine"), good_resp("should-not-be-used"))
+        self.assertEqual(out["choices"][0]["message"]["content"], "fine")
+        self.assertIsNone(client.last_json)  # never called upstream
+
+    def test_keeps_original_if_retry_also_empty(self):
+        out, _ = self._run(empty_length_resp(), empty_length_resp())
+        self.assertTrue(proxy._is_empty_maxtokens_response(out))  # unchanged
+        self.assertEqual(self.m.empty_maxtokens_recoveries, 0)
+
+
+if __name__ == "__main__":
+    unittest.main()

package/tools/agents/tests/test_error_loop_break.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""ERROR-LOOP guardrail: detect a repeated tool_result error signature across
+varied edits and nudge the model to re-read the failing file.
+
+The identical-tool-call loop detectors miss this because each edit differs; this
+keys on the normalized ERROR, which is edit-invariant.
+"""
+import importlib.util
+import unittest
+from pathlib import Path
+
+
+def _load_proxy_module():
+    proxy_path = Path(__file__).resolve().parents[1] / "scripts" / "anthropic_proxy.py"
+    spec = importlib.util.spec_from_file_location("anthropic_proxy", proxy_path)
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    return mod
+
+
+proxy = _load_proxy_module()
+
+
+class ErrorSignatureTests(unittest.TestCase):
+    def test_same_error_different_paths_lines_shares_signature(self):
+        a = proxy._error_signature("SyntaxError: Identifier 'testPageLoads' has already been declared\n  at /home/x/test.js:75:1")
+        b = proxy._error_signature("SyntaxError: Identifier 'testPageLoads' has already been declared\n  at /other/path/test.js:128:9")
+        self.assertTrue(a)
+        self.assertEqual(a, b)  # edit-invariant
+
+    def test_distinct_errors_differ(self):
+        a = proxy._error_signature("TypeError: cannot read property 'x' of undefined")
+        b = proxy._error_signature("SyntaxError: unexpected token")
+        self.assertNotEqual(a, b)
+
+    def test_clean_output_has_no_signature(self):
+        self.assertEqual(proxy._error_signature("ok\nall 12 tests passed"), "")
+        self.assertEqual(proxy._error_signature(""), "")
+
+
+class MonitorStreakTests(unittest.TestCase):
+    def setUp(self):
+        self.m = proxy.SessionMonitor()
+
+    def test_streak_accumulates_on_repeat_and_resets_on_pass(self):
+        err = "TypeError: cannot find module 'foo'\n at a.js:1"
+        for _ in range(3):
+            self.m.note_tool_result_error(err)
+        self.assertGreaterEqual(self.m.error_signature_streak, 3)
+        # a passing result resets
+        self.m.note_tool_result_error("ok, 5 passed")
+        self.assertEqual(self.m.error_signature_streak, 0)
+
+    def test_new_error_resets_streak_to_one(self):
+        self.m.note_tool_result_error("Error: A\n at a:1")
+        self.m.note_tool_result_error("Error: A\n at a:2")
+        self.assertEqual(self.m.error_signature_streak, 2)
+        self.m.note_tool_result_error("Error: B different\n at b:1")
+        self.assertEqual(self.m.error_signature_streak, 1)
+
+
+class InjectionTests(unittest.TestCase):
+    def setUp(self):
+        self.m = proxy.SessionMonitor()
+
+    def test_nudge_injected_past_threshold(self):
+        for _ in range(proxy.PROXY_ERROR_LOOP_THRESHOLD):
+            self.m.note_tool_result_error("SyntaxError: already declared\n at x:1")
+        body = {"messages": [{"role": "user", "content": "go"}], "tool_choice": "required"}
+        proxy._maybe_inject_error_loop_break(body, self.m)
+        sysmsg = body["messages"][0]
+        self.assertEqual(sysmsg["role"], "system")
+        self.assertIn("SAME failure", sysmsg["content"])
+        self.assertIn("re-read", sysmsg["content"].lower())
+        # advisory: does NOT release tool_choice
+        self.assertEqual(body["tool_choice"], "required")
+        self.assertEqual(self.m.error_loop_fires, 1)
+
+    def test_no_nudge_below_threshold(self):
+        self.m.note_tool_result_error("Error: once\n at x:1")
+        body = {"messages": [{"role": "user", "content": "go"}]}
+        proxy._maybe_inject_error_loop_break(body, self.m)
+        self.assertEqual(len(body["messages"]), 1)  # unchanged
+        self.assertEqual(self.m.error_loop_fires, 0)
+
+
+if __name__ == "__main__":
+    unittest.main()