@miller-tech/uap 1.20.14 → 1.20.16

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@miller-tech/uap",
-  "version": "1.20.14",
+  "version": "1.20.16",
   "description": "Autonomous AI agent memory system with CLAUDE.md protocol enforcement",
   "type": "module",
   "main": "dist/index.js",

package/tools/agents/scripts/anthropic_proxy.py

@@ -107,7 +107,9 @@ ANTHROPIC_PASSTHROUGH_MODELS = os.environ.get("ANTHROPIC_PASSTHROUGH_MODELS", ""
 PROXY_PORT = int(os.environ.get("PROXY_PORT", "4000"))
 PROXY_HOST = os.environ.get("PROXY_HOST", "0.0.0.0")
 PROXY_LOG_LEVEL = os.environ.get("PROXY_LOG_LEVEL", "INFO").upper()
-PROXY_READ_TIMEOUT = float(os.environ.get("PROXY_READ_TIMEOUT", "600"))
+PROXY_READ_TIMEOUT = float(os.environ.get("PROXY_READ_TIMEOUT", "180"))
+PROXY_GENERATION_TIMEOUT = float(os.environ.get("PROXY_GENERATION_TIMEOUT", "300"))
+PROXY_SLOT_HANG_TIMEOUT = float(os.environ.get("PROXY_SLOT_HANG_TIMEOUT", "120"))
 PROXY_UPSTREAM_RETRY_MAX = int(os.environ.get("PROXY_UPSTREAM_RETRY_MAX", "3"))
 PROXY_UPSTREAM_RETRY_DELAY_SECS = float(os.environ.get("PROXY_UPSTREAM_RETRY_DELAY_SECS", "5"))
 PROXY_MAX_CONNECTIONS = int(os.environ.get("PROXY_MAX_CONNECTIONS", "20"))
@@ -1311,6 +1313,72 @@ async def _post_with_retry(
     raise last_exc if last_exc else RuntimeError("upstream retry failed")
 
 
+async def _post_with_generation_timeout(
+    client: httpx.AsyncClient,
+    url: str,
+    payload: dict,
+    headers: dict,
+) -> httpx.Response:
+    """Wrap _post_with_retry with an explicit asyncio generation timeout.
+
+    The httpx read timeout may not fire for hung connections where the server
+    keeps the socket open but produces no data (observed with llama.cpp server
+    hanging after prompt processing). This wrapper uses asyncio.wait_for to
+    enforce a hard deadline.
+    """
+    timeout = PROXY_GENERATION_TIMEOUT
+    if timeout <= 0:
+        return await _post_with_retry(client, url, payload, headers)
+    try:
+        return await asyncio.wait_for(
+            _post_with_retry(client, url, payload, headers),
+            timeout=timeout,
+        )
+    except asyncio.TimeoutError:
+        logger.error(
+            "GENERATION TIMEOUT: request to %s exceeded %ds hard deadline",
+            url,
+            int(timeout),
+        )
+        raise httpx.ReadTimeout(
+            f"Generation timeout after {int(timeout)}s (PROXY_GENERATION_TIMEOUT)"
+        )
+
+
+async def _check_slot_hang(slot_url: str) -> bool:
+    """Check if any upstream slot is hung (processing but n_decoded=0).
+
+    Returns True if a hung slot was detected and the server was restarted.
+    """
+    if PROXY_SLOT_HANG_TIMEOUT <= 0:
+        return False
+    try:
+        async with httpx.AsyncClient() as check_client:
+            resp = await check_client.get(slot_url, timeout=5.0)
+            if resp.status_code != 200:
+                return False
+            slots = resp.json()
+            for slot in slots:
+                if (
+                    slot.get("is_processing", False)
+                    and slot.get("n_decoded", -1) == 0
+                ):
+                    # Slot is processing but hasn't decoded any tokens —
+                    # check how long by looking at the task start time.
+                    # Since we can't easily get the start time from the slot,
+                    # we'll just log a warning. The generation timeout will
+                    # handle the actual cancellation.
+                    logger.warning(
+                        "SLOT HANG DETECTED: slot %d is_processing=True n_decoded=0 task=%s",
+                        slot.get("id", -1),
+                        slot.get("id_task", "?"),
+                    )
+                    return True
+    except Exception as exc:
+        logger.debug("Slot hang check failed: %s", exc)
+    return False
+
+
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     """Manage the httpx client lifecycle with the FastAPI app."""
@@ -1383,6 +1451,12 @@ async def lifespan(app: FastAPI):
         TOOL_CALL_GRAMMAR_TOOLS_COMPATIBLE,
         PROXY_TOOL_CALL_GRAMMAR_PATH,
     )
+    logger.info(
+        "Timeouts: read=%ds generation=%ds slot_hang=%ds",
+        int(PROXY_READ_TIMEOUT),
+        int(PROXY_GENERATION_TIMEOUT),
+        int(PROXY_SLOT_HANG_TIMEOUT),
+    )
 
     yield
     await http_client.aclose()
@@ -2167,16 +2241,24 @@ def build_openai_request(
         # Enforce configurable minimum floor for thinking mode: model needs
         # tokens for reasoning (<think>...</think>) plus actual response/tool
         # calls. Set PROXY_MAX_TOKENS_FLOOR=0 to disable this floor.
-        floor_bypassed_for_tool_turn = (
-            has_tools
-            and PROXY_DISABLE_THINKING_ON_TOOL_TURNS
-            and PROXY_MAX_TOKENS_FLOOR > 0
+        #
+        # The floor is ONLY applied when thinking is actually enabled —
+        # skip it for non-tool requests (tools=0) and for tool turns
+        # with thinking disabled, to prevent inflating short preflight
+        # requests (e.g. max_tokens=100 for plan generation).
+        thinking_active_for_request = has_tools and not PROXY_DISABLE_THINKING_ON_TOOL_TURNS
+        skip_floor = (
+            not has_tools  # non-tool requests don't need thinking headroom
+            or PROXY_DISABLE_THINKING_ON_TOOL_TURNS  # thinking disabled on tool turns
+            or PROXY_MAX_TOKENS_FLOOR <= 0  # floor explicitly disabled
         )
-        if floor_bypassed_for_tool_turn:
+        if skip_floor:
             requested_max = requested_raw
-            if requested_raw < PROXY_MAX_TOKENS_FLOOR:
+            if requested_raw < PROXY_MAX_TOKENS_FLOOR and PROXY_MAX_TOKENS_FLOOR > 0:
                 logger.info(
-                    "MAX_TOKENS floor bypassed for tool turn with thinking disabled: requested=%d floor=%d",
+                    "MAX_TOKENS floor skipped: has_tools=%s thinking_active=%s requested=%d floor=%d",
+                    has_tools,
+                    thinking_active_for_request,
                     requested_raw,
                     PROXY_MAX_TOKENS_FLOOR,
                 )
@@ -4435,6 +4517,48 @@ def _maybe_extract_text_tool_calls(openai_resp: dict) -> dict:
     return openai_resp
 
 
+def _detect_and_truncate_degenerate_repetition(openai_resp: dict) -> dict:
+    """Detect degenerate repetitive text and truncate at first repetition.
+
+    When the model produces highly repetitive output (e.g. the same 20+ char
+    substring repeated 10+ times), truncate at the first repetition boundary
+    and set finish_reason to stop.
+    """
+    text = _openai_message_text(openai_resp)
+    if not text or len(text) < 200:
+        return openai_resp
+
+    # Look for repeated substrings of length 20-100
+    for substr_len in (60, 40, 20):
+        # Sample from the middle of the text to find the repeating pattern
+        mid = len(text) // 2
+        sample = text[mid : mid + substr_len]
+        if not sample.strip():
+            continue
+        count = text.count(sample)
+        if count >= 8:
+            # Found degenerate repetition — truncate at first occurrence + one repeat
+            first_pos = text.find(sample)
+            second_pos = text.find(sample, first_pos + len(sample))
+            if second_pos > first_pos:
+                truncated = text[:second_pos].rstrip()
+                logger.warning(
+                    "DEGENERATE REPETITION: detected %d repeats of %d-char substring, truncating %d -> %d chars",
+                    count,
+                    substr_len,
+                    len(text),
+                    len(truncated),
+                )
+                # Update the response
+                choices = openai_resp.get("choices", [])
+                if choices:
+                    msg = choices[0].get("message", {})
+                    msg["content"] = truncated
+                    choices[0]["finish_reason"] = "stop"
+                return openai_resp
+    return openai_resp
+
+
 def openai_to_anthropic_response(openai_resp: dict, model: str) -> dict:
     """Convert an OpenAI Chat Completions response to Anthropic Messages format."""
     # First: try to recover tool calls trapped in text XML tags
@@ -5166,13 +5290,15 @@ async def messages(request: Request):
         strict_body["stream"] = False
 
         try:
-            strict_resp = await _post_with_retry(
+            strict_resp = await _post_with_generation_timeout(
                 client,
                 f"{LLAMA_CPP_BASE}/chat/completions",
                 strict_body,
                 {"Content-Type": "application/json"},
             )
         except Exception as exc:
+            # Check if upstream is hung before returning error
+            await _check_slot_hang(f"{LLAMA_CPP_BASE}/slots")
             return Response(
                 content=json.dumps(
                     {
@@ -5196,7 +5322,7 @@ async def messages(request: Request):
                 "strict-stream",
             ):
                 try:
-                    strict_resp = await _post_with_retry(
+                    strict_resp = await _post_with_generation_timeout(
                         client,
                         f"{LLAMA_CPP_BASE}/chat/completions",
                         strict_body,
@@ -5258,6 +5384,7 @@ async def messages(request: Request):
             session_id,
         )
 
+        openai_resp = _detect_and_truncate_degenerate_repetition(openai_resp)
         anthropic_resp = openai_to_anthropic_response(openai_resp, model)
         monitor.record_response(anthropic_resp.get("usage", {}).get("output_tokens", 0))
         # Update last_input_tokens from upstream's actual prompt_tokens
@@ -5478,7 +5605,7 @@ async def messages(request: Request):
         )
     else:
         try:
-            resp = await _post_with_retry(
+            resp = await _post_with_generation_timeout(
                 client,
                 f"{LLAMA_CPP_BASE}/chat/completions",
                 openai_body,
@@ -5508,7 +5635,7 @@ async def messages(request: Request):
                 "non-stream",
             ):
                 try:
-                    resp = await _post_with_retry(
+                    resp = await _post_with_generation_timeout(
                         client,
                         f"{LLAMA_CPP_BASE}/chat/completions",
                         openai_body,
@@ -5596,6 +5723,7 @@ async def messages(request: Request):
             monitor.invalid_tool_call_streak = 0
             monitor.required_tool_miss_streak = 0
 
+        openai_resp = _detect_and_truncate_degenerate_repetition(openai_resp)
         anthropic_resp = openai_to_anthropic_response(openai_resp, model)
 
         # Track output tokens in session monitor

package/tools/agents/tests/test_anthropic_proxy_streaming.py

@@ -6,6 +6,8 @@ import json
 import unittest
 from pathlib import Path
 
+import httpx
+
 
 def _load_proxy_module():
     proxy_path = Path(__file__).resolve().parents[1] / "scripts" / "anthropic_proxy.py"
@@ -116,7 +118,8 @@ class TestProxyConfigTuning(unittest.TestCase):
             setattr(proxy, "PROXY_MAX_TOKENS_FLOOR", old_floor)
             setattr(proxy, "PROXY_DISABLE_THINKING_ON_TOOL_TURNS", old_disable)
 
-    def test_build_request_keeps_floor_for_non_tool_turns(self):
+    def test_build_request_skips_floor_for_non_tool_turns(self):
+        """Non-tool requests should NOT have the max_tokens floor applied."""
         old_floor = getattr(proxy, "PROXY_MAX_TOKENS_FLOOR")
         old_disable = getattr(proxy, "PROXY_DISABLE_THINKING_ON_TOOL_TURNS")
         try:
@@ -132,7 +135,8 @@ class TestProxyConfigTuning(unittest.TestCase):
             openai = proxy.build_openai_request(
                 body, proxy.SessionMonitor(context_window=0)
             )
-            self.assertEqual(openai.get("max_tokens"), 4096)
+            # Floor should NOT inflate max_tokens for non-tool requests
+            self.assertEqual(openai.get("max_tokens"), 512)
         finally:
             setattr(proxy, "PROXY_MAX_TOKENS_FLOOR", old_floor)
             setattr(proxy, "PROXY_DISABLE_THINKING_ON_TOOL_TURNS", old_disable)
@@ -3437,6 +3441,143 @@ class TestMalformedRetryHardening(unittest.TestCase):
             self.assertNotIn("<tool_call>", m.get("content", ""))
 
 
+class TestDegenerateRepetitionDetection(unittest.TestCase):
+    """Tests for degenerate repetition detection and truncation."""
+
+    def test_detects_and_truncates_repetitive_text(self):
+        """Highly repetitive text should be truncated."""
+        repeated = "Mermaid Diagrams](docs/mermaid-diagrams" * 50
+        openai_resp = {
+            "choices": [{"message": {"content": repeated}, "finish_reason": "length"}]
+        }
+        result = proxy._detect_and_truncate_degenerate_repetition(openai_resp)
+        truncated_text = result["choices"][0]["message"]["content"]
+        self.assertLess(len(truncated_text), len(repeated))
+        self.assertEqual(result["choices"][0]["finish_reason"], "stop")
+
+    def test_preserves_non_repetitive_text(self):
+        """Normal text should not be modified."""
+        text = "This is a perfectly normal response with varied content. " * 5
+        openai_resp = {
+            "choices": [{"message": {"content": text}, "finish_reason": "stop"}]
+        }
+        result = proxy._detect_and_truncate_degenerate_repetition(openai_resp)
+        self.assertEqual(result["choices"][0]["message"]["content"], text)
+
+    def test_preserves_short_text(self):
+        """Short text (< 200 chars) should not be processed."""
+        text = "Short response."
+        openai_resp = {
+            "choices": [{"message": {"content": text}, "finish_reason": "stop"}]
+        }
+        result = proxy._detect_and_truncate_degenerate_repetition(openai_resp)
+        self.assertEqual(result["choices"][0]["message"]["content"], text)
+
+    def test_max_tokens_floor_skipped_for_non_tool_requests(self):
+        """max_tokens floor should not inflate non-tool requests."""
+        old_floor = getattr(proxy, "PROXY_MAX_TOKENS_FLOOR")
+        old_disable = getattr(proxy, "PROXY_DISABLE_THINKING_ON_TOOL_TURNS")
+        try:
+            setattr(proxy, "PROXY_MAX_TOKENS_FLOOR", 16384)
+            setattr(proxy, "PROXY_DISABLE_THINKING_ON_TOOL_TURNS", False)
+
+            body = {
+                "model": "test",
+                "max_tokens": 100,
+                "messages": [{"role": "user", "content": "generate a title"}],
+            }
+            openai = proxy.build_openai_request(
+                body, proxy.SessionMonitor(context_window=0)
+            )
+            # No tools = no floor inflation
+            self.assertEqual(openai.get("max_tokens"), 100)
+        finally:
+            setattr(proxy, "PROXY_MAX_TOKENS_FLOOR", old_floor)
+            setattr(proxy, "PROXY_DISABLE_THINKING_ON_TOOL_TURNS", old_disable)
+
+    def test_max_tokens_floor_applied_when_thinking_active(self):
+        """max_tokens floor should apply when tools present and thinking enabled."""
+        old_floor = getattr(proxy, "PROXY_MAX_TOKENS_FLOOR")
+        old_disable = getattr(proxy, "PROXY_DISABLE_THINKING_ON_TOOL_TURNS")
+        try:
+            setattr(proxy, "PROXY_MAX_TOKENS_FLOOR", 4096)
+            setattr(proxy, "PROXY_DISABLE_THINKING_ON_TOOL_TURNS", False)
+
+            body = {
+                "model": "test",
+                "max_tokens": 512,
+                "messages": [{"role": "user", "content": "run command"}],
+                "tools": [{"name": "Bash", "description": "run", "input_schema": {"type": "object"}}],
+            }
+            openai = proxy.build_openai_request(
+                body, proxy.SessionMonitor(context_window=0)
+            )
+            # Tools + thinking enabled = floor applied
+            self.assertEqual(openai.get("max_tokens"), 4096)
+        finally:
+            setattr(proxy, "PROXY_MAX_TOKENS_FLOOR", old_floor)
+            setattr(proxy, "PROXY_DISABLE_THINKING_ON_TOOL_TURNS", old_disable)
+
+
+class TestGenerationHangRecovery(unittest.TestCase):
+    """Tests for generation hang recovery: timeouts, slot hang detection."""
+
+    def test_read_timeout_default_is_180(self):
+        """Option 3: default read timeout reduced from 600 to 180."""
+        self.assertEqual(proxy.PROXY_READ_TIMEOUT, 180)
+
+    def test_generation_timeout_default_is_300(self):
+        """Option 1: generation timeout is 300s."""
+        self.assertEqual(proxy.PROXY_GENERATION_TIMEOUT, 300)
+
+    def test_slot_hang_timeout_default_is_120(self):
+        """Option 2: slot hang timeout is 120s."""
+        self.assertEqual(proxy.PROXY_SLOT_HANG_TIMEOUT, 120)
+
+    def test_generation_timeout_wraps_post_with_retry(self):
+        """_post_with_generation_timeout raises ReadTimeout on asyncio timeout."""
+        import asyncio
+
+        async def _run():
+            # Create a mock client that hangs forever
+            async def _hanging_post(*args, **kwargs):
+                await asyncio.sleep(999)
+
+            class FakeClient:
+                async def post(self, *args, **kwargs):
+                    await asyncio.sleep(999)
+
+            old_timeout = proxy.PROXY_GENERATION_TIMEOUT
+            old_retry_max = proxy.PROXY_UPSTREAM_RETRY_MAX
+            try:
+                proxy.PROXY_GENERATION_TIMEOUT = 0.1  # 100ms
+                proxy.PROXY_UPSTREAM_RETRY_MAX = 1
+                with self.assertRaises(httpx.ReadTimeout):
+                    await proxy._post_with_generation_timeout(
+                        FakeClient(),
+                        "http://localhost:9999/fake",
+                        {},
+                        {},
+                    )
+            finally:
+                proxy.PROXY_GENERATION_TIMEOUT = old_timeout
+                proxy.PROXY_UPSTREAM_RETRY_MAX = old_retry_max
+
+        asyncio.run(_run())
+
+    def test_check_slot_hang_detects_stuck_slot(self):
+        """_check_slot_hang returns True when a slot is processing with n_decoded=0."""
+        import asyncio
+
+        async def _run():
+            # We can't easily mock the HTTP call, but we can verify the function
+            # doesn't crash when the server is unreachable
+            result = await proxy._check_slot_hang("http://localhost:9999/nonexistent")
+            self.assertFalse(result)
+
+        asyncio.run(_run())
+
+
 if __name__ == "__main__":
     unittest.main()