@miller-tech/uap 1.179.4 → 1.179.5

package/docs/guides/PROXY.md

@@ -179,6 +179,8 @@ set is below.
 | Env (default) | Behavior |
 |---|---|
 | `PROXY_MAX_CONNECTIONS` (20) | httpx pool size; a large pool + 529 backoff absorbs connection churn gracefully |
+| `PROXY_UPSTREAM_RETRY_MAX` (3) / `PROXY_UPSTREAM_RETRY_DELAY_SECS` (5) | Transient-failure retry budget for upstream calls, on both the buffered and the streaming path. Values below 1 are clamped to 1 |
+| **503 "Loading model"** (no toggle) | llama-server answers 503 while a GGUF is still being mapped. Both paths treat it as transient: wait for `/health` (up to 60s), then retry. A wait that times out surfaces the 503 rather than stacking another — so a cold start costs at most ~60s of extra latency per attempt, not `RETRY_MAX × 60s` of dead air |
 | **529 backpressure** (no toggle) | A pool timeout returns HTTP **529 `overloaded_error`** with `retry-after` — pure graceful degradation, not a hard failure |
 | `PROXY_CLOSEWAIT_REAP_INTERVAL` (0 = **off**) | Opt-in CLOSE-WAIT reaper (pool self-heal); off by default because pool-swap churn can harm a saturated upstream |
 | `PROXY_TOOL_NARROWING` (**off**) | Opt-in: drop cycling/banned tools from the set on loops — always keeps the Bash/WebFetch/Agent escape hatch + write tools (a floor invariant that never strands the agent) |

package/package.json

@@ -1,7 +1,6 @@
 {
   "name": "@miller-tech/uap",
-  "version": "1.179.4",
-  "version": "1.179.4",
+  "version": "1.179.5",
   "description": "Autonomous AI agent memory system with CLAUDE.md protocol enforcement",
   "type": "module",
   "main": "dist/index.js",

package/tools/agents/scripts/anthropic_proxy.py

@@ -3560,6 +3560,92 @@ class DisconnectAwareClient(httpx.AsyncClient):
                 task.cancel()
 
 
+async def _send_stream_with_retry(
+    client: httpx.AsyncClient,
+    url: str,
+    payload: dict,
+) -> httpx.Response:
+    """POST a streaming completion, retrying transient failures and 503 "Loading model".
+
+    The streaming sibling of `_post_with_retry_inner`. Both retry the
+    `_UPSTREAM_RETRY_EXCEPTIONS` set — which is wider than connect errors; a
+    ReadTimeout/ReadError mid-setup is retried too — so a llama-server restart
+    doesn't fail the turn. Both also treat a 503 "Loading model" as transient:
+    llama-server answers that while a GGUF is still being mapped, so the right
+    move is to wait for /health and retry rather than surface a fatal error.
+
+    Raises the last transient exception when every attempt failed. Otherwise
+    returns the response, which may itself be a non-200 the caller must handle —
+    including a loading 503 that outlived the retry budget, or one whose health
+    wait timed out.
+    """
+    # Clamp: a configured 0 would skip the loop entirely and fall through to the
+    # terminal raise with last_exc unset, escaping the caller's narrow except as
+    # an unhandled 500. One attempt is the floor.
+    retry_max = max(1, PROXY_UPSTREAM_RETRY_MAX)
+    last_exc: Exception | None = None
+    for attempt in range(retry_max):
+        try:
+            resp = await client.send(
+                client.build_request(
+                    "POST",
+                    url,
+                    json=payload,
+                    headers={"Content-Type": "application/json"},
+                ),
+                stream=True,
+            )
+            setattr(resp, "_uap_client", client)
+            if resp.status_code == 503:
+                # _is_loading_model_503 reads .text, which raises ResponseNotRead
+                # on an unread streaming body — and its bare `except` turns that
+                # into a silent False, so without this read every loading 503
+                # would look permanent and the retry would never fire. httpx
+                # caches the body, so the caller's non-200 handler re-reads free.
+                try:
+                    await resp.aread()
+                except BaseException:
+                    # Never leave the connection checked out: a ReadError here is
+                    # retryable and would otherwise loop with resp unreferenced
+                    # and unclosed, accruing CLOSE-WAIT until the pool saturates.
+                    _detach_aclose(resp)
+                    raise
+                if _is_loading_model_503(resp) and attempt < retry_max - 1:
+                    logger.warning(
+                        "Upstream 503 Loading model (stream attempt %d/%d) – waiting for health",
+                        attempt + 1,
+                        retry_max,
+                    )
+                    healthy = await _wait_for_upstream_health(client, max_wait=60.0)
+                    if not healthy:
+                        # Matches _post_with_retry_inner: once a health wait has
+                        # timed out, stacking another one just buys the client
+                        # minutes of dead air before the same failure. Surface it.
+                        return resp
+                    _detach_aclose(resp)
+                    continue
+            return resp
+        except _UPSTREAM_RETRY_EXCEPTIONS as exc:
+            last_exc = exc
+            if attempt < retry_max - 1:
+                logger.warning(
+                    "Upstream transient error (stream attempt %d/%d): %s – retrying in %.0fs",
+                    attempt + 1,
+                    retry_max,
+                    type(exc).__name__,
+                    PROXY_UPSTREAM_RETRY_DELAY_SECS,
+                )
+                await asyncio.sleep(PROXY_UPSTREAM_RETRY_DELAY_SECS)
+            else:
+                logger.error(
+                    "Upstream stream failed after %d attempts: %s: %s",
+                    retry_max,
+                    type(exc).__name__,
+                    exc,
+                )
+    raise last_exc if last_exc else RuntimeError("upstream stream retry failed")
+
+
 async def _post_with_retry_inner(
     client: httpx.AsyncClient,
     url: str,
@@ -11676,43 +11762,17 @@ async def messages(request: Request):
         # Retry upstream connection with backoff to handle
         # llama-server restarts gracefully instead of 500-ing to the client.
         MAX_UPSTREAM_RETRIES = PROXY_UPSTREAM_RETRY_MAX
-        RETRY_DELAY_SECS = PROXY_UPSTREAM_RETRY_DELAY_SECS
         last_exc: Exception | None = None
         resp: httpx.Response | None = None
 
-        for attempt in range(MAX_UPSTREAM_RETRIES):
-            try:
-                resp = await client.send(
-                    client.build_request(
-                        "POST",
-                        f"{LLAMA_CPP_BASE}/chat/completions",
-                        json=openai_body,
-                        headers={"Content-Type": "application/json"},
-                    ),
-                    stream=True,
-                )
-                setattr(resp, "_uap_client", client)
-                # Connection succeeded – break out of retry loop
-                last_exc = None
-                break
-            except _UPSTREAM_RETRY_EXCEPTIONS as exc:
-                last_exc = exc
-                if attempt < MAX_UPSTREAM_RETRIES - 1:
-                    logger.warning(
-                        "Upstream connect failed (attempt %d/%d): %s – retrying in %.0fs",
-                        attempt + 1,
-                        MAX_UPSTREAM_RETRIES,
-                        type(exc).__name__,
-                        RETRY_DELAY_SECS,
-                    )
-                    await asyncio.sleep(RETRY_DELAY_SECS)
-                else:
-                    logger.error(
-                        "Upstream connect failed after %d attempts: %s: %s",
-                        MAX_UPSTREAM_RETRIES,
-                        type(exc).__name__,
-                        exc,
-                    )
+        try:
+            resp = await _send_stream_with_retry(
+                client,
+                f"{LLAMA_CPP_BASE}/chat/completions",
+                openai_body,
+            )
+        except _UPSTREAM_RETRY_EXCEPTIONS as exc:
+            last_exc = exc
 
         if last_exc is not None:
             return Response(

package/tools/agents/tests/test_anthropic_proxy_streaming.py

@@ -5834,3 +5834,228 @@ class TestPassthroughTimeout(unittest.IsolatedAsyncioTestCase):
 
         self.assertIsNotNone(captured.get("timeout"))
         self.assertEqual(captured["timeout"].read, proxy.PROXY_PASSTHROUGH_TIMEOUT)
+
+
+class _FakeStreamResponse:
+    """Stand-in for a streamed httpx.Response, faithful to two contracts.
+
+    Both are load-bearing for _send_stream_with_retry, and a fake that models
+    neither lets the fix be deleted with the suite still green:
+
+    1. `.text` raises ResponseNotRead until the body has been read. That is why
+       the production code calls aread() before _is_loading_model_503 — whose
+       bare `except` would otherwise swallow the raise into a silent False and
+       silently disable the retry.
+    2. aread() caches. The caller re-reads the same response in its non-200
+       handler, which only works because httpx serves the second read from
+       cache rather than raising StreamConsumed.
+    """
+
+    def __init__(self, status_code, text=""):
+        self.status_code = status_code
+        self._body = text
+        self._read = False
+        self.read_count = 0
+        self.closed = False
+
+    @property
+    def text(self):
+        if not self._read:
+            raise httpx.ResponseNotRead()
+        return self._body
+
+    async def aread(self):
+        if self._read:
+            return self._body.encode()  # cached: no second stream consumption
+        self._read = True
+        self.read_count += 1
+        return self._body.encode()
+
+    async def aclose(self):
+        self.closed = True
+
+
+class _FakeStreamClient:
+    """Replays a queued script of responses/exceptions for client.send()."""
+
+    def __init__(self, scripted):
+        self._scripted = list(scripted)
+        self.sends = 0
+        self.requests = []
+        self.stream_flags = []
+
+    def build_request(self, method, url, **kwargs):
+        return {"method": method, "url": url, **kwargs}
+
+    async def send(self, request, stream=False):
+        self.sends += 1
+        self.requests.append(request)
+        self.stream_flags.append(stream)
+        if not self._scripted:
+            raise AssertionError("no scripted response left")
+        nxt = self._scripted.pop(0)
+        if isinstance(nxt, Exception):
+            raise nxt
+        return nxt
+
+
+class TestSendStreamWithRetry(unittest.TestCase):
+    """The streaming path must treat 503 'Loading model' as transient.
+
+    Regression guard: this path used to break out of its retry loop on any HTTP
+    response, so a streamed turn that began while llama-server was still mapping
+    a GGUF surfaced 'AI_APICallError: Loading model' to the client as a fatal
+    error. The non-streaming sibling (_post_with_retry_inner) retried it —
+    health-gated and bounded by PROXY_UPSTREAM_RETRY_MAX — while the streaming
+    path, the one every opencode turn actually uses, did not retry it at all.
+    """
+
+    def _run(self, client, health_ok=True, retry_max=3):
+        waited = []
+
+        async def _fake_health(_client, max_wait=60.0, poll_interval=5.0):
+            waited.append(max_wait)
+            return health_ok
+
+        async def _no_sleep(_secs):
+            return None
+
+        def _sync_close(closeable):
+            # Production schedules the close on a detached task; inside a test
+            # the loop closes before it runs. Close eagerly so `.closed` is a
+            # deterministic assertion rather than a race.
+            closeable.closed = True
+
+        with unittest.mock.patch.object(proxy, "_wait_for_upstream_health", _fake_health), \
+                unittest.mock.patch.object(proxy, "_detach_aclose", _sync_close), \
+                unittest.mock.patch.object(proxy, "PROXY_UPSTREAM_RETRY_MAX", retry_max), \
+                unittest.mock.patch.object(asyncio, "sleep", _no_sleep):
+            resp = asyncio.run(
+                proxy._send_stream_with_retry(client, "http://x/v1/chat/completions", {})
+            )
+        return resp, waited
+
+    def test_retries_loading_model_503_then_returns_success(self):
+        loading = _FakeStreamResponse(503, "Loading model")
+        ok = _FakeStreamResponse(200, "")
+        client = _FakeStreamClient([loading, ok])
+
+        resp, waited = self._run(client)
+
+        self.assertIs(resp, ok)
+        self.assertEqual(client.sends, 2, "should have retried after the 503")
+        self.assertEqual(len(waited), 1, "should have waited for upstream health")
+        self.assertTrue(loading.closed, "the discarded 503 must be closed")
+
+    def test_does_not_retry_a_non_loading_503(self):
+        # A 503 that is not a model load is a real upstream failure; passing it
+        # straight through preserves the existing error contract.
+        overloaded = _FakeStreamResponse(503, "server is overloaded")
+        client = _FakeStreamClient([overloaded, _FakeStreamResponse(200, "")])
+
+        resp, waited = self._run(client)
+
+        self.assertIs(resp, overloaded)
+        self.assertEqual(client.sends, 1)
+        self.assertEqual(waited, [])
+
+    def test_returns_503_after_exhausting_retry_budget(self):
+        responses = [_FakeStreamResponse(503, "Loading model") for _ in range(3)]
+        client = _FakeStreamClient(list(responses))
+
+        resp, waited = self._run(client, retry_max=3)
+
+        self.assertEqual(client.sends, 3)
+        self.assertEqual(resp.status_code, 503)
+        self.assertIs(resp, responses[-1], "last 503 is surfaced to the caller")
+        self.assertEqual(len(waited), 2, "waits between attempts, not after the last")
+
+    def test_surfaces_the_503_when_the_health_wait_times_out(self):
+        # Mirrors _post_with_retry_inner: once upstream is still unhealthy after
+        # a full wait, stacking more waits only buys the client dead air. Without
+        # this gate a dead upstream costs retry_max x 60s before the same failure.
+        loading = _FakeStreamResponse(503, "Loading model")
+        client = _FakeStreamClient([loading, _FakeStreamResponse(200, "")])
+
+        resp, waited = self._run(client, health_ok=False)
+
+        self.assertIs(resp, loading, "unhealthy upstream surfaces the 503 now")
+        self.assertEqual(client.sends, 1, "must not retry a still-unhealthy upstream")
+        self.assertEqual(len(waited), 1)
+
+    def test_reads_the_body_before_classifying_a_503(self):
+        # The fix hinges on aread() preceding _is_loading_model_503, whose bare
+        # `except` would turn httpx's ResponseNotRead into a silent False and
+        # disable the retry entirely. Deleting that aread() must fail here.
+        loading = _FakeStreamResponse(503, "Loading model")
+        ok = _FakeStreamResponse(200, "")
+        client = _FakeStreamClient([loading, ok])
+
+        resp, _ = self._run(client)
+
+        self.assertIs(resp, ok)
+        self.assertEqual(loading.read_count, 1, "body read exactly once, before .text")
+
+    def test_closes_the_response_when_the_body_read_fails(self):
+        # A ReadError while reading the 503 body is retryable; if the discarded
+        # response is not closed the connection accrues as CLOSE-WAIT until the
+        # pool saturates, with the client seeing nothing wrong.
+        class _ExplodingResponse(_FakeStreamResponse):
+            async def aread(self):
+                raise httpx.ReadError("reset mid-body")
+
+        boom = _ExplodingResponse(503, "Loading model")
+        ok = _FakeStreamResponse(200, "")
+        client = _FakeStreamClient([boom, ok])
+
+        resp, _ = self._run(client)
+
+        self.assertIs(resp, ok, "the read failure is retried")
+        self.assertTrue(boom.closed, "the abandoned response must be closed")
+
+    def test_still_retries_connect_errors(self):
+        # Regression guard for the pre-existing behaviour the extraction replaced.
+        ok = _FakeStreamResponse(200, "")
+        client = _FakeStreamClient([httpx.ConnectError("boom"), ok])
+
+        resp, _ = self._run(client)
+
+        self.assertIs(resp, ok)
+        self.assertEqual(client.sends, 2)
+
+    def test_raises_when_every_connect_attempt_fails(self):
+        client = _FakeStreamClient([httpx.ConnectError("boom") for _ in range(3)])
+
+        with self.assertRaises(httpx.ConnectError):
+            self._run(client, retry_max=3)
+
+        self.assertEqual(client.sends, 3)
+
+    def test_retry_max_of_zero_still_makes_one_attempt(self):
+        # An unclamped 0 skipped the loop and raised a bare RuntimeError, which
+        # the call site's narrow `except` misses — an unhandled 500 where the
+        # old code returned a clean 529.
+        ok = _FakeStreamResponse(200, "")
+        client = _FakeStreamClient([ok])
+
+        resp, _ = self._run(client, retry_max=0)
+
+        self.assertIs(resp, ok)
+        self.assertEqual(client.sends, 1)
+
+    def test_preserves_the_streaming_contract_the_caller_depends_on(self):
+        # stream=True keeps the turn incremental (buffering it would silently
+        # reintroduce full-generation latency), and _uap_client drives the
+        # inflight accounting that the pool-retire logic waits on.
+        ok = _FakeStreamResponse(200, "")
+        client = _FakeStreamClient([ok])
+
+        resp, _ = self._run(client)
+
+        self.assertEqual(client.stream_flags, [True])
+        self.assertIs(getattr(resp, "_uap_client"), client)
+        self.assertEqual(client.requests[0]["url"], "http://x/v1/chat/completions")
+
+
+if __name__ == "__main__":
+    unittest.main()