@tiens.nguyen/gonext-local-worker 1.0.143 → 1.0.144

package/gonext_agent_chat.py

@@ -1872,10 +1872,13 @@ def run_agent_chat(cfg):
             # Ollama honors it). Harmless if the server ignores it — usage stays 0.
             completion_kwargs["stream_options"] = {"include_usage": True}
             self._apply_rate_limit()
+            t0 = time.monotonic()
+            _log("streamed generate: request issued (stream=True), awaiting first token…")
             stream = self.client.chat.completions.create(**completion_kwargs)
             parts = []
             role = "assistant"
             in_tok = out_tok = 0
+            first_token_at = None
             for chunk in stream:
                 usage = getattr(chunk, "usage", None)
                 if usage is not None:
@@ -1891,6 +1894,17 @@ def run_agent_chat(cfg):
                     role = delta.role
                 piece = getattr(delta, "content", None)
                 if piece:
+                    if first_token_at is None:
+                        # First streamed byte. The gap t0→here is pure prompt-eval time
+                        # (server sent nothing before this); everything after is token
+                        # generation with bytes flowing, so the proxy idle timer resets.
+                        # If this line never appears in the log, the request died in
+                        # prompt-eval BEFORE any token — streaming can't help there
+                        # (shrink the model / warm the prompt cache instead).
+                        first_token_at = time.monotonic()
+                        _log(f"streamed generate: FIRST token after {first_token_at - t0:.1f}s "
+                             "(prompt-eval done; tokens now flowing)")
+                        _emit({"type": "step", "text": "Model is responding…"})
                     parts.append(piece)
             content = "".join(parts)
             if stop_sequences and not self.supports_stop_parameter:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiens.nguyen/gonext-local-worker",
-  "version": "1.0.143",
+  "version": "1.0.144",
   "description": "Polls GoNext cloud API for async local LLM jobs and runs them against Ollama/OpenAI-compatible servers on this Mac",
   "type": "module",
   "license": "MIT",