@tiens.nguyen/gonext-local-worker 1.0.141 → 1.0.142

package/gonext_agent_chat.py

@@ -1805,6 +1805,85 @@ def run_agent_chat(cfg):
     # and any step memory it accumulates. completion_kwargs["messages"] here is the
     # literal messages array sent to /v1/chat/completions.
     class _LoggingModel(OpenAIServerModel):
+        def __init__(self, *a, stream_agent=False, **kw):
+            # stream_agent: request the completion with stream=True and reassemble the
+            # deltas into one message. Enabled for Ollama, where a long single-shot
+            # (non-stream) generation sends nothing until the very end — a reverse proxy
+            # in front of the box (ollama1.gomarsic.cc) then hits its idle read-timeout
+            # and kills the request mid-generation. Streaming keeps tokens flowing so the
+            # idle timer never fires. smolagents still receives the COMPLETE text, so its
+            # <code>/final_answer parsing is unaffected.
+            self._stream_agent = bool(stream_agent)
+            super().__init__(*a, **kw)
+
+        def _generate_once(self, *args, **kwargs):
+            """One model call. Streams + reassembles when _stream_agent, else the normal
+            non-streaming path. Streaming is a pure transport optimization: on ANY failure
+            it falls back to the non-streaming call, and it's skipped for tool-calling
+            mode (tools_to_call_from), whose tool_call deltas we don't reassemble."""
+            tools_to_call_from = kwargs.get("tools_to_call_from")
+            if not self._stream_agent or tools_to_call_from:
+                return super().generate(*args, **kwargs)
+            try:
+                return self._streamed_generate(*args, **kwargs)
+            except Exception as e:  # noqa: BLE001
+                _log(f"streamed generate failed ({e}) → falling back to non-streaming")
+                return super().generate(*args, **kwargs)
+
+        def _streamed_generate(
+            self, messages, stop_sequences=None, response_format=None,
+            tools_to_call_from=None, **kwargs,
+        ):
+            from smolagents.models import ChatMessage, TokenUsage  # local: version-safe
+            completion_kwargs = self._prepare_completion_kwargs(
+                messages=messages,
+                stop_sequences=stop_sequences,
+                response_format=response_format,
+                tools_to_call_from=tools_to_call_from,
+                model=self.model_id,
+                custom_role_conversions=self.custom_role_conversions,
+                convert_images_to_image_urls=True,
+                **kwargs,
+            )
+            completion_kwargs["stream"] = True
+            # Ask for a usage summary in the final chunk (OpenAI streaming convention;
+            # Ollama honors it). Harmless if the server ignores it — usage stays 0.
+            completion_kwargs["stream_options"] = {"include_usage": True}
+            self._apply_rate_limit()
+            stream = self.client.chat.completions.create(**completion_kwargs)
+            parts = []
+            role = "assistant"
+            in_tok = out_tok = 0
+            for chunk in stream:
+                usage = getattr(chunk, "usage", None)
+                if usage is not None:
+                    in_tok = getattr(usage, "prompt_tokens", 0) or in_tok
+                    out_tok = getattr(usage, "completion_tokens", 0) or out_tok
+                choices = getattr(chunk, "choices", None)
+                if not choices:
+                    continue
+                delta = getattr(choices[0], "delta", None)
+                if delta is None:
+                    continue
+                if getattr(delta, "role", None):
+                    role = delta.role
+                piece = getattr(delta, "content", None)
+                if piece:
+                    parts.append(piece)
+            content = "".join(parts)
+            if stop_sequences and not self.supports_stop_parameter:
+                from smolagents.models import remove_content_after_stop_sequences
+                content = remove_content_after_stop_sequences(content, stop_sequences)
+            _log(f"streamed generate assembled {len(content)} chars "
+                 f"(in={in_tok} out={out_tok} tokens)")
+            return ChatMessage(
+                role=role,
+                content=content,
+                tool_calls=None,
+                raw=None,
+                token_usage=TokenUsage(input_tokens=in_tok, output_tokens=out_tok),
+            )
+
         def generate(self, *args, **kwargs):
             # Two safeguards on each step's reply BEFORE smolagents parses it for the
             # code block / final_answer:
@@ -1843,7 +1922,7 @@ def run_agent_chat(cfg):
             try:
                 for attempt in range(3):
                     try:
-                        msg = super().generate(*args, **kwargs)
+                        msg = self._generate_once(*args, **kwargs)
                         break
                     except Exception as e:  # noqa: BLE001
                         emsg = str(e)
@@ -2014,13 +2093,17 @@ def run_agent_chat(cfg):
         # the live test — minutes per step on a slow GPU — so disable it explicitly.
         # Sniffed per-server because MLX might reject the unknown param.
         extra_model_kwargs = {}
-        if _is_ollama_server(coding_base_url):
-            _log("coding server is Ollama → sending reasoning_effort='none' to disable thinking")
+        coding_is_ollama = _is_ollama_server(coding_base_url)
+        if coding_is_ollama:
+            _log("coding server is Ollama → reasoning_effort='none' (disable thinking) + streaming")
             extra_model_kwargs["reasoning_effort"] = "none"
         model = _LoggingModel(
             model_id=coding_model_id,
             api_base=coding_base_url,
             api_key=agent_api_key,
+            # Stream for Ollama so a long generation keeps the connection warm and can't
+            # trip a reverse-proxy idle read-timeout. smolagents still gets the full text.
+            stream_agent=coding_is_ollama,
             # Cap each HTTP attempt at 600s so a genuinely slow single call (a
             # remote M6000/Vulkan box doing a cold ~4-5min prompt-eval) can finish
             # on the first try, while a truly hung request still fails into OUR

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiens.nguyen/gonext-local-worker",
-  "version": "1.0.141",
+  "version": "1.0.142",
   "description": "Polls GoNext cloud API for async local LLM jobs and runs them against Ollama/OpenAI-compatible servers on this Mac",
   "type": "module",
   "license": "MIT",