@tiens.nguyen/gonext-local-worker 1.0.327 → 1.0.330

package/gonext_agent_chat.py

@@ -1528,6 +1528,49 @@ def _is_backend_overloaded(err) -> bool:
     )
 
 
+def _is_auth_failure(err) -> bool:
+    """True when the model server REJECTED OUR CREDENTIAL — a deterministic config error
+    that must fail fast rather than retry (task #117 companion).
+
+    Deliberately narrow:
+      • 401 is always an auth failure.
+      • 403 is NOT, on its own. Several providers return 403 for QUOTA EXHAUSTED as well as
+        for permission-denied, and reporting "you're out of credit" as "your API key is
+        wrong" sends the user to fix the wrong thing. A 403 counts only when the body does
+        NOT read like a quota/rate problem — those belong to _is_backend_overloaded.
+      • 402 (payment required) is a billing problem, not a credential one — excluded here
+        so it can carry its own message.
+    Falls back to message text only when the exception carries no HTTP status (a bare
+    string, or a wrapper that lost its cause)."""
+    if _is_backend_overloaded(err):
+        return False  # a 429 (or a quota-flavoured 403) is never an auth failure
+    s = str(err or "").lower()
+    code = _http_status_of(err)
+    if code is not None:
+        if code == 401:
+            return True
+        if code == 403:
+            return not re.search(r"quota|rate.?limit|billing|credit|capacity", s)
+        return False
+    return (
+        "401" in s
+        or "invalid_api_key" in s
+        or "invalid api key" in s
+        or "incorrect api key" in s
+        or "unauthorized" in s
+        or "authentication" in s and "fail" in s
+    )
+
+
+def _is_billing_failure(err) -> bool:
+    """True for 402 / explicit out-of-credit responses — deterministic like an auth failure
+    but with a different fix, so it gets its own message instead of 'check your API key'."""
+    if _http_status_of(err) == 402:
+        return True
+    s = str(err or "").lower()
+    return "insufficient_quota" in s or "exceeded your current quota" in s
+
+
 def _retry_after_seconds(err):
     """The provider's Retry-After hint in seconds, or None. Reads the header off an
     openai-python APIStatusError (`err.response.headers`) and falls back to a
@@ -1585,12 +1628,15 @@ def _is_backend_unavailable(err) -> bool:
     # down". Checked FIRST because a 429 body can also contain the word "timeout".
     if _is_backend_overloaded(err):
         return False
-    # A structured 5xx IS a gateway/backend outage regardless of wording: a provider that
-    # returns `Error code: 502 - {"error": …}` carries no "bad gateway" text for the
+    # A structured gateway/timeout status IS an outage regardless of wording: a provider
+    # that returns `Error code: 502 - {"error": …}` carries no "bad gateway" text for the
     # phrase list below to match, and would otherwise be misfiled as a generic error.
+    # 408 and Cloudflare's 522/524 are included so a real timeout STATUS keeps the same
+    # answer the "timed out" text below has always given it — without them, adding the
+    # structured check would have quietly demoted those.
     code = _http_status_of(err)
     if code is not None:
-        return code in (502, 503, 504)
+        return code in (502, 503, 504, 408, 522, 524)
     return (
         "502 bad gateway" in s
         or "503 service" in s
@@ -1859,12 +1905,68 @@ def _looks_like_tool_code(code: str):
     return True, ""
 
 
+_CODE_PAIR_RE = r"<code[^>]*>[\s\S]*?</code>"
+
+
+def _close_dangling_code_block(text):
+    """Close a code block the model OPENED but never CLOSED, when what follows it is a
+    runnable tool call — the normal shape whenever the server honors `stop` (task #119).
+
+    smolagents appends its closing tag to the stop_sequences of every CodeAgent turn
+    (agents.py: `stop_sequences.append(self.code_block_tags[1])`), so any backend that
+    honors `stop` — i.e. every OpenAI-compatible API — returns the block WITHOUT its
+    closer. Confirmed against the raw responses saved from a live run: what the model
+    actually sends is `<code>\\nrun_command("npm run build", …)` and nothing after it.
+    smolagents re-appends the tag itself once it has the text (agents.py, "This adds the
+    end code sequence … to the history"), which is why a trace shows a tidy `…</code>`
+    that never came off the wire — but every check WE run in between tested for the PAIR
+    `<code>…</code>`, a delimiter this backend structurally cannot produce.
+
+    What that cost, live: a perfectly good turn was read as a "dangling code tag", burned
+    a second model call on the CODE-NOW retry, and the retry's equally good opener-only
+    reply was then DISCARDED for failing the same pair test — so every other step died on
+    a parse error the model never caused, while the user watched "Error in code parsing"
+    scroll past on steps that had in fact called the right tool.
+
+    So: canonicalize first, decide second. Everything after the LAST opener is taken as
+    the block, a trailing "Thought:" line and stray tags are cut, and the result is
+    substituted ONLY when _looks_like_tool_code passes (it compiles AND calls something)
+    — the same #113 validity gate used for reasoning-channel recovery, so prose that
+    merely trails a stray tag is never promoted into executable code.
+
+    No-op when the text already holds a pair, has no opener, or the tail isn't runnable;
+    idempotent. Deliberately backend-agnostic: `stop` truncation is a property of the
+    protocol, not of one vendor, and the gate makes it inert everywhere else."""
+    if not isinstance(text, str) or not text:
+        return text
+    if re.search(_CODE_PAIR_RE, text, re.I):
+        return text                      # a real block already — never touch it
+    opens = list(re.finditer(r"<code[^>]*>", text, re.I))
+    if not opens:
+        return text                      # prose turn, or a lone stray closer (see #111)
+    last = opens[-1]
+    body = re.split(r"\n\s*Thought\s*:\s", text[last.end():], maxsplit=1)[0]
+    body = re.sub(r"</?code[^>]*>", "", body, flags=re.I).strip()
+    ok, _why = _looks_like_tool_code(body)
+    if not ok:
+        return text                      # not runnable — leave the step to fail honestly
+    # Keep the Thought prose that preceded the block, minus any earlier stray tag (an
+    # unbalanced opener left in the head would let the pair regex start from the wrong
+    # place and swallow the prose into the code).
+    head = re.sub(r"</?code[^>]*>", "", text[:last.start()], flags=re.I).rstrip()
+    return (head + "\n" if head else "") + "<code>\n" + body + "\n</code>"
+
+
 def _has_runnable_block(text) -> bool:
     """True if `text` contains a <code>…</code> block whose python actually compiles —
     i.e. this step produced something that can run. Used by the #113 no-code loop-breaker
-    (openai backend only) to tell "the step worked" from "the model wrote prose again"."""
+    (openai backend only) to tell "the step worked" from "the model wrote prose again".
+
+    Closes a stop-truncated block first (#119), so a step that DID call a tool is never
+    counted toward the no-code streak just because the API ate the closing tag."""
     if not isinstance(text, str) or not text:
         return False
+    text = _close_dangling_code_block(text)
     blk = re.search(r"<code[^>]*>([\s\S]*?)</code>", text, re.I)
     if not blk:
         return False
@@ -1881,7 +1983,10 @@ def _code_nudge_reason(content, openai_backend: bool):
     replayed offline against real traces (task #113).
 
       1. empty-content   — the whole turn went to the reasoning channel, no message.
-      2. dangling tag    — a stray <code>/</code> with no valid pair.
+      2. dangling tag    — a stray <code>/</code> with no valid pair. A block the server
+         merely TRUNCATED at the `</code>` stop sequence is not that (task #119): it is
+         closed first and accepted as-is, so a step that already called a tool no longer
+         pays for a second model call to be told to do what it just did.
       3. prose-only plan — narration of what it is ABOUT to do, no tag at all. This one
          is openai-only and additionally requires _looks_like_scratchpad, so a FINISHED
          prose answer is never nudged into inventing another tool call. It exists because
@@ -1893,8 +1998,8 @@ def _code_nudge_reason(content, openai_backend: bool):
     text = (content or "").strip()
     if not text:
         return "empty-content"
-    if re.search(r"<code[^>]*>[\s\S]*?</code>", content, re.I):
-        return None  # a real block — nothing to nudge
+    if re.search(_CODE_PAIR_RE, _close_dangling_code_block(content), re.I):
+        return None  # a real block (closing a stop-truncated one first) — nothing to nudge
     if re.search(r"</?code[\s>/]", content, re.I):
         return "dangling code tag, no valid block"
     if openai_backend and _looks_like_scratchpad(text):
@@ -1919,9 +2024,12 @@ def _recover_code_from_reasoning(content, reasoning):
     Candidates are tried NEWEST-first and the first runnable one wins; if none is
     runnable we return `content` untouched and let the step fail normally, which
     is what happened before #111 existed."""
-    _pair = r"<code[^>]*>[\s\S]*?</code>"
-    if re.search(_pair, content or "", re.I):
-        return content  # content already has a usable block — never override it
+    _pair = _CODE_PAIR_RE
+    if re.search(_pair, _close_dangling_code_block(content) or "", re.I):
+        # Content already has a usable block — never override it. A block the server
+        # truncated at the `</code>` stop sequence counts as usable (#119): the visible
+        # channel is the model's actual answer and must win over a reasoning-channel draft.
+        return content
     reasoning = reasoning or ""
     if not reasoning.strip():
         return content  # nothing to recover from (Ollama / non-reasoning model)
@@ -5323,6 +5431,14 @@ def run_agent_chat(cfg):
                     _log(f"streamed generate refused ({_clip(str(e), 200)}) → NOT falling "
                          "back to non-streaming; letting the retry loop back off (#114)")
                     raise
+                if _is_auth_failure(e) or _is_billing_failure(e):
+                    # Same reasoning as the overload above: the credential was rejected, so
+                    # streaming was never the problem. Re-sending unstreamed would cost a
+                    # second identical request per attempt AND print "Streaming unavailable
+                    # — retrying without streaming…", which is simply untrue here.
+                    _log(f"streamed generate rejected ({_clip(str(e), 200)}) → NOT falling "
+                         "back to non-streaming; this is a credential/billing error")
+                    raise
                 _log(f"streamed generate failed ({e}) → falling back to non-streaming")
                 _emit({"type": "step",
                        "text": "Streaming unavailable — retrying without streaming…"})
@@ -5454,6 +5570,12 @@ def run_agent_chat(cfg):
                 _emit({"type": "step", "text": "Model output ran away — restarting this step…"})
                 _log(f"streamed generate: discarded {len(content)} runaway chars → empty content")
                 return "", role, in_tok, out_tok, reasoning_len
+            # Task #119: smolagents passes its closing "</code>" as a stop sequence, so a
+            # server that honors `stop` returns the block UNCLOSED. Close it here, before
+            # anything downstream asks "did this turn produce a code block?" — otherwise a
+            # step that called a tool correctly reads as broken. Gated on the code actually
+            # compiling, so a genuinely dangling tag still falls through to the nudge below.
+            content = _close_dangling_code_block(content)
             # Task #111: reasoning models on an OpenAI-compatible API (e.g. Kimi K3 @ Moonshot)
             # can emit the actionable <code>…</code> into the REASONING channel, leaving only
             # prose + a stray tag in `content` → smolagents' parser can't find the pair. When
@@ -5519,7 +5641,11 @@ def run_agent_chat(cfg):
             #       in the trace is appended by SMOLAGENTS after we return (its stop-sequence
             #       handling), so the content WE see here carries no tag at all. Openai-only.
             # See _code_nudge_reason for the full decision (kept pure so it can be replayed).
-            _pair_re = r"<code[^>]*>[\s\S]*?</code>"
+            # NOTE (#119): none of these triggers may fire on a block the server merely
+            # truncated at the "</code>" stop sequence — that is the SUCCESS shape for an
+            # OpenAI-compatible backend, not a failure. _close_dangling_code_block runs in
+            # _run_stream (and again inside _code_nudge_reason) so it is already excluded.
+            _pair_re = _CODE_PAIR_RE
             why = _code_nudge_reason(content, self._openai_backend)
             if why:
                 _log(f"{why} turn (reasoning={reasoning_len} chars) → retrying once with a "
@@ -5531,7 +5657,13 @@ def run_agent_chat(cfg):
                     {"role": "user", "content": self._CODE_NOW_DIRECTIVE}
                 ]
                 r_content, r_role, r_in, r_out, _ = self._run_stream(retry_kwargs)
-                r_has_pair = re.search(_pair_re, r_content or "", re.I) is not None
+                # Close a stop-truncated block before judging the retry (#119). This test
+                # used to demand a literal pair, which the openai backend cannot return —
+                # so the retry we had just paid for was thrown away and the broken original
+                # was sent on to smolagents. (_run_stream already closed it; the call here
+                # is idempotent and keeps the rule visible at the point of decision.)
+                r_has_pair = re.search(
+                    _pair_re, _close_dangling_code_block(r_content) or "", re.I) is not None
                 # Use the retry when it yields a real code block, or (empty original) any
                 # content. If the retry is still broken and we HAD a Thought, keep the
                 # original and let smolagents' own retry take it from here — no regression.
@@ -5542,7 +5674,13 @@ def run_agent_chat(cfg):
                 out_tok += r_out
             if stop_sequences and not self.supports_stop_parameter:
                 from smolagents.models import remove_content_after_stop_sequences
+                # For a server that can't honor `stop`, smolagents emulates it by splitting
+                # on each stop sequence and keeping split[0] — and "</code>" IS one of those
+                # sequences, so this CUTS THE CLOSER back off a block that had one. Re-close
+                # afterwards (#119) so both kinds of backend hand back the same canonical
+                # shape; without it the emulated path re-creates the exact bug we just fixed.
                 content = remove_content_after_stop_sequences(content, stop_sequences)
+                content = _close_dangling_code_block(content)
             return ChatMessage(
                 role=role,
                 content=content,
@@ -5647,8 +5785,47 @@ def run_agent_chat(cfg):
                         # for this server, e.g. an MLX-style name against Ollama) —
                         # deterministic, so retrying is pointless. Fail fast with the
                         # server's actual model list so the user can fix Settings.
+                        # A rejected CREDENTIAL is deterministic — retrying re-sends the
+                        # same bad key, and letting it fall through to the turn-level
+                        # handler would degrade to _plain_reply, i.e. the chat model
+                        # answering confidently about work that never ran (#94's failure
+                        # mode). Fail fast with a message that says WHICH key went out:
+                        # python uses `coding_api_key or agent_api_key`, so an unset coding
+                        # key silently authenticates a cloud endpoint with the LOCAL agent
+                        # key (the Auto+cloud path in #117) — a completely different fix
+                        # from "your coding key is wrong". The key itself is never echoed.
+                        if _is_auth_failure(e) or _is_billing_failure(e):
+                            _host = _host_of(coding_base_url)
+                            if _is_billing_failure(e):
+                                raise _AgentConfigError(
+                                    f"The agent coding backend ({_host}) accepted the key "
+                                    "but refused the request for BILLING reasons (out of "
+                                    "credit / quota exhausted). Nothing ran. Top up the "
+                                    "account, or point 'Agent coding model' at another "
+                                    "backend in Settings → Agent."
+                                ) from e
+                            if coding_api_key:
+                                raise _AgentConfigError(
+                                    f"The agent coding backend ({_host}) REJECTED the "
+                                    "coding API key. Nothing ran. Re-enter 'Agent coding "
+                                    "model API key' in Settings → Agent (the stored key is "
+                                    "write-only, so a stale one can't be inspected — just "
+                                    "save a fresh one)."
+                                ) from e
+                            raise _AgentConfigError(
+                                f"The agent coding backend ({_host}) rejected the request: "
+                                "NO coding API key was sent, so the local agent key was "
+                                "used instead. Set 'Coding backend' to OpenAI-compatible "
+                                "in Settings → Agent and enter the API key — the key field "
+                                "only appears once that backend is selected."
+                            ) from e
                         if "404" in emsg and "not found" in emsg.lower():
-                            ids = _list_model_ids(coding_base_url, agent_api_key)
+                            # Probe with the CODING key when there is one: using the agent
+                            # key here 401s against a cloud coder, so the "This server has:
+                            # …" hint — the only useful part of this error — silently
+                            # vanished exactly when it was needed.
+                            ids = _list_model_ids(coding_base_url,
+                                                  coding_api_key or agent_api_key)
                             have = f" This server has: {', '.join(ids)}." if ids else ""
                             raise _AgentConfigError(
                                 f"The agent coding model {coding_model_id!r} does not "

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiens.nguyen/gonext-local-worker",
-  "version": "1.0.327",
+  "version": "1.0.330",
   "description": "Polls GoNext cloud API for async local LLM jobs and runs them against Ollama/OpenAI-compatible servers on this Mac",
   "type": "module",
   "license": "MIT",
@@ -17,6 +17,7 @@
     "gonext": "./gonext-repl.mjs"
   },
   "scripts": {
+    "test": "python3 -m unittest discover tests -v",
     "deploy:local": "npm install -g .",
     "run": "gonext-local-worker",
     "publish:org": "npm publish --access public"