npm - @tiens.nguyen/gonext-local-worker - Versions diffs - 1.0.320 → 1.0.322 - Mend

@tiens.nguyen/gonext-local-worker 1.0.320 → 1.0.322

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (2) hide show

package/gonext_agent_chat.py +173 -7
package/package.json +1 -1

package/gonext_agent_chat.py CHANGED Viewed

@@ -25,6 +25,7 @@ Emits NDJSON lines on stdout:
 """
 import contextlib
 import json
+import random
 import re
 import sys
 import threading
@@ -820,6 +821,22 @@ class _AgentConfigError(RuntimeError):
     worth a plain-reply degrade — the user needs the message itself to fix Settings."""
+class _AgentBackendOverloaded(RuntimeError):
+    """The coding backend refused every attempt with 429/rate-limit (task #114). Carries
+    the user-facing explanation; like a gateway outage it must NOT degrade to a plain
+    reply, which would answer from the chat model as if the work had been done."""
+def _host_of(url) -> str:
+    """Just the hostname of a base URL ('https://api.moonshot.ai/v1' → 'api.moonshot.ai'),
+    for user-facing messages. Falls back to the raw string."""
+    try:
+        from urllib.parse import urlparse
+        return urlparse(str(url or "")).netloc or str(url or "")
+    except Exception:  # noqa: BLE001
+        return str(url or "")
 def _list_model_ids(base_url, api_key=""):
     """Return the model ids an OpenAI-compatible server reports at {base_url}/models.
@@ -1452,8 +1469,92 @@ def _plain_reply(messages: list, base_url: str, api_key: str, model_id: str,
 #   • a genuine model/agent error — worth the plain-reply degrade.
 # `str(e)` for a proxy error is the raw nginx HTML ("<html>…502 Bad Gateway…"), so match
 # on the status text/codes and the common connection-failure phrases.
+def _is_backend_overloaded(err) -> bool:
+    """True when the model server ACCEPTED the connection but REFUSED the work because it
+    is saturated or we are over a quota — 429 / engine_overloaded / rate limit (task #114).
+    Deliberately separate from _is_backend_unavailable: that one means "the box or proxy
+    is not answering" (retry the same request, it will probably land), while this means
+    "the server is telling us to slow down" (re-sending immediately makes it worse and can
+    deepen the limit). The two get different retry budgets and different user-facing copy.
+    Live trigger: Moonshot returned
+      Error code: 429 - {'error': {'message': 'The engine is currently overloaded, please
+      try again later', 'type': 'engine_overloaded_error'}}
+    A local Ollama/MLX box does not produce these, so nothing keyed off this fires there."""
+    s = str(err or "").lower()
+    if "429" in s or "too many requests" in s:
+        return True
+    return (
+        "engine_overloaded" in s
+        or "overloaded" in s
+        or "rate limit" in s
+        or "rate_limit" in s
+        # OpenAI's billing/quota code. NOT a bare "quota" — a tool hitting "Disk quota
+        # exceeded" would otherwise be reported to the user as a model overload.
+        or "insufficient_quota" in s
+        or "over capacity" in s
+        or "at capacity" in s
+    )
+def _retry_after_seconds(err):
+    """The provider's Retry-After hint in seconds, or None. Reads the header off an
+    openai-python APIStatusError (`err.response.headers`) and falls back to a
+    'retry after 12 seconds' phrase in the message. Values are clamped to a sane range so
+    a bogus header can't park the turn for an hour."""
+    val = None
+    resp = getattr(err, "response", None)
+    headers = getattr(resp, "headers", None)
+    if headers is not None:
+        try:
+            val = headers.get("retry-after") or headers.get("Retry-After")
+        except Exception:  # noqa: BLE001
+            val = None
+    if val is None:
+        m = re.search(r"retry[\s-]?after[\s:]+(\d+(?:\.\d+)?)", str(err or ""), re.I)
+        val = m.group(1) if m else None
+    if val is None:
+        return None
+    try:
+        secs = float(str(val).strip())
+    except (TypeError, ValueError):
+        return None  # HTTP-date form — not worth parsing; fall back to our own backoff
+    if secs <= 0:
+        return None
+    return max(1.0, min(60.0, secs))
+def _retry_delay(attempt: int, overloaded: bool, gateway: bool, err=None) -> float:
+    """Seconds to wait before retry #attempt (1-based). Pure, so the policy can be
+    replayed offline (task #114).
+      overloaded (429) — PATIENT and jittered: ~5s → 10s → 20s → 30s (capped), plus up to
+                         25% jitter so parallel workers don't retry in lock-step. The
+                         provider's own Retry-After wins outright when it sends one.
+      gateway (502…)   — unchanged from #94: prompt re-send, 2s/4s/6s/8s capped at 8.
+      otherwise        — unchanged: 1.5s per attempt.
+    Only the `overloaded` branch is new; the other two must stay byte-for-byte, since they
+    are the path an Ollama/MLX coder takes."""
+    if overloaded:
+        hinted = _retry_after_seconds(err)
+        if hinted:
+            return hinted
+        delay = min(30.0, 5.0 * (2 ** (attempt - 1)))
+        return delay + random.uniform(0, min(3.0, delay * 0.25))
+    if gateway:
+        return min(8.0, 2.0 * attempt)
+    return 1.5 * attempt
 def _is_backend_unavailable(err) -> bool:
     s = str(err or "").lower()
+    # An overload (429) is its own class — never let it fall into the gateway bucket, whose
+    # policy (re-send promptly, up to 6 times) is exactly wrong for a server saying "slow
+    # down". Checked FIRST because a 429 body can also contain the word "timeout".
+    if _is_backend_overloaded(err):
+        return False
     return (
         "502 bad gateway" in s
         or "503 service" in s
@@ -5164,16 +5265,31 @@ def run_agent_chat(cfg):
         def _generate_once(self, *args, **kwargs):
             """One model call. Streams + reassembles when _stream_agent, else the normal
-            non-streaming path. Streaming is a pure transport optimization: on ANY failure
-            it falls back to the non-streaming call, and it's skipped for tool-calling
-            mode (tools_to_call_from), whose tool_call deltas we don't reassemble."""
+            non-streaming path. Streaming is a pure transport optimization: on a TRANSPORT
+            failure it falls back to the non-streaming call, and it's skipped for
+            tool-calling mode (tools_to_call_from), whose tool_call deltas we don't
+            reassemble.
+            Exception (task #114): an OVERLOAD (429 / engine_overloaded / rate limit) is
+            NOT a transport failure — the server refused the work. Re-sending the identical
+            request unstreamed, instantly and with no backoff, is the worst possible answer:
+            it can deepen the limit, and because a non-streamed call emits no first token it
+            then sits silent behind the 600s client timeout (live: 167s of "Thinking…" until
+            the user pressed Ctrl+C). Re-raise instead, so generate()'s retry loop applies a
+            real backoff and tells the user what is going on."""
             tools_to_call_from = kwargs.get("tools_to_call_from")
             if not self._stream_agent or tools_to_call_from:
                 return super().generate(*args, **kwargs)
             try:
                 return self._streamed_generate(*args, **kwargs)
             except Exception as e:  # noqa: BLE001
+                if _is_backend_overloaded(e):
+                    _log(f"streamed generate refused ({_clip(str(e), 200)}) → NOT falling "
+                         "back to non-streaming; letting the retry loop back off (#114)")
+                    raise
                 _log(f"streamed generate failed ({e}) → falling back to non-streaming")
+                _emit({"type": "step",
+                       "text": "Streaming unavailable — retrying without streaming…"})
                 return super().generate(*args, **kwargs)
         def _run_stream(self, completion_kwargs):
@@ -5505,18 +5621,45 @@ def run_agent_chat(cfg):
                                 "names (for Ollama use the tag, e.g. qwen3:14b)."
                             ) from e
                         last_err = e
-                        gateway = _is_backend_unavailable(e)
-                        if gateway and max_attempts < 6:
+                        # An OVERLOAD (429/rate limit, task #114) is its own class: the
+                        # server answered and told us to slow down, so it needs a PATIENT,
+                        # jittered backoff rather than the gateway policy's prompt re-send.
+                        # Checked first — _is_backend_unavailable already excludes it, but
+                        # the ordering keeps the intent obvious.
+                        overloaded = _is_backend_overloaded(e)
+                        gateway = (not overloaded) and _is_backend_unavailable(e)
+                        if (overloaded or gateway) and max_attempts < 6:
                             max_attempts = 6  # stretch the budget for a flaky backend
                         attempt += 1
                         if attempt < max_attempts:
                             # Longer, capped backoff for gateway errors (up to ~8s) so a
-                            # cold/loading model has time to come back; short for the rest.
-                            delay = min(8.0, 2.0 * attempt) if gateway else 1.5 * attempt
+                            # cold/loading model has time to come back; short for the rest;
+                            # patient + jittered for an overload. See _retry_delay.
+                            delay = _retry_delay(attempt, overloaded, gateway, e)
                             kind = "gateway/backend unavailable" if gateway else "transient"
+                            if overloaded:
+                                kind = "backend overloaded (429/rate limit)"
+                                # SURFACE IT: without this the REPL keeps showing a playful
+                                # heartbeat word and the user has no idea the provider is
+                                # refusing work (live: 167s of "Thinking…" → Ctrl+C).
+                                _emit({"type": "step",
+                                       "text": "Coding model is overloaded (429) — waiting "
+                                               f"{delay:.0f}s, then retrying "
+                                               f"(attempt {attempt + 1}/{max_attempts})…"})
                             _log(f"generate {kind} error (attempt {attempt}/{max_attempts}): "
                                  f"{_clip(emsg, 160)} — retrying in {delay:.0f}s")
                             time.sleep(delay)
+                        elif overloaded:
+                            # Out of attempts: end the turn with a cause the user can act on
+                            # instead of a bare stack trace (same spirit as #94's 502 copy).
+                            _log("backend overloaded after all attempts → failing honestly")
+                            raise _AgentBackendOverloaded(
+                                "The agent coding model backend "
+                                f"({_host_of(coding_base_url)}) is overloaded right now and "
+                                f"refused {max_attempts} attempts (HTTP 429). Nothing ran. "
+                                "Please try again in a minute, or switch the coding backend "
+                                "in Settings → Agent."
+                            ) from e
                 if last_err is not None:
                     raise last_err
             finally:
@@ -7010,6 +7153,29 @@ def run_agent_chat(cfg):
         # failure: it hides the outage and, being persisted to history, misleads the next
         # turn. Emit a clear, standalone "backend is down, retry" and stop — the checkpoint
         # is kept (not cleared) so a later "continue" resumes once the box is back.
+        # Coding-model OVERLOAD (429/rate limit, task #114): same reasoning as the outage
+        # branch below — the tools never ran, so degrading to _plain_reply would invent a
+        # confident how-to for work that didn't happen. Reported separately because the
+        # cause and the advice differ: the server is UP, it's just saturated or we're over
+        # quota, so "try again shortly" is the honest fix. Checked BEFORE the outage branch
+        # (_is_backend_unavailable deliberately excludes overloads).
+        if _is_backend_overloaded(e):
+            _log(f"agent error (coding-model backend OVERLOADED, NOT degrading): {_clip(str(e), 160)}")
+            _ov = None
+            _cur, _depth = e, 0
+            while _cur is not None and _depth < 6:
+                if isinstance(_cur, _AgentBackendOverloaded):
+                    _ov = _cur
+                    break
+                _cur, _depth = (getattr(_cur, "__cause__", None)
+                                or getattr(_cur, "__context__", None)), _depth + 1
+            _emit({"type": "final", "text": "⚠️ " + (str(_ov) if _ov else (
+                "The coding model backend is overloaded right now (HTTP 429 — it asked me "
+                "to slow down), so I couldn't run the tools to do this. The server is up, "
+                "it's just saturated or over quota. Please try again in a minute, or switch "
+                "the coding backend in Settings → Agent. (I didn't make any changes.)"
+            ))})
+            return
         if _is_backend_unavailable(e):
             _log(f"agent error (coding-model backend unavailable, NOT degrading): {_clip(str(e), 160)}")
             _emit({"type": "final", "text": (

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@tiens.nguyen/gonext-local-worker",
-  "version": "1.0.320",
+  "version": "1.0.322",
   "description": "Polls GoNext cloud API for async local LLM jobs and runs them against Ollama/OpenAI-compatible servers on this Mac",
   "type": "module",
   "license": "MIT",