@tiens.nguyen/gonext-local-worker 1.0.188 → 1.0.189

package/gonext-local-worker.mjs

@@ -1815,6 +1815,7 @@ async function runAgentChatJob(job) {
 
     let inThink = false;
     let finalText = "";
+    let answerStreamed = false; // an answer_stream delta already delivered the visible text
     // Raw model-stream tokens (event.type === "stream") are Python code + `Thought:`
     // prose + <code>/<|"|> fragments. The web Thought panel renders reasoning as
     // markdown, which turns `#`-prefixed code comments into <h1> headings and
@@ -1866,6 +1867,20 @@ async function runAgentChatJob(job) {
           enqueueText("<think>");
         }
         enqueueText(event.text + "\n\n");
+      } else if (event.type === "answer_stream" && typeof event.text === "string") {
+        // Live delta of the FINAL user-facing answer (the plain-chat fast path streams
+        // its reply as it generates, instead of blocking for the whole thing) — this is
+        // the actual answer, NOT reasoning, so it must stay OUTSIDE <think> (wrapping it
+        // would hide it in the web's collapsed Thinking panel instead of the live main
+        // answer bubble). Close any open think/stream fence first in case a prior step
+        // left one dangling.
+        closeStreamFence();
+        if (inThink) {
+          inThink = false;
+          enqueueText("</think>");
+        }
+        enqueueText(event.text);
+        answerStreamed = true;
       } else if (event.type === "final" && typeof event.text === "string") {
         closeStreamFence();
         if (inThink) {
@@ -1873,7 +1888,12 @@ async function runAgentChatJob(job) {
           enqueueText("</think>");
         }
         finalText = event.text;
-        enqueueText(event.text);
+        // If the answer was already delivered live via answer_stream deltas, its text is
+        // ALREADY in resultText — appending it again here would duplicate the whole reply
+        // (streamed once, then dumped again in full). Only append when nothing streamed it.
+        if (!answerStreamed) {
+          enqueueText(event.text);
+        }
       }
     }, pdfEnv);
 

package/gonext-repl.mjs

@@ -54,10 +54,15 @@ const isCodeCloseLine = (line) => line.trim() === "</code>";
 // gonext-local-worker daemon's own terminal — swallow it here to keep the REPL terse.
 const isToolStepSummary = (line) => / \| → /.test(line);
 // The router's internal play-by-play ("Routing your request…", "→ Agent mode (needs
-// tools)", "→ Chat reply", "Choosing a tool…") is implementation detail, not something a
-// user needs — also visible in the daemon's own terminal. Swallowed; see routingAnnounced.
+// tools)", "→ Chat reply", "Choosing a tool…", "Composing a reply…") is implementation
+// detail, not something a user needs — also visible in the daemon's own terminal.
+// Swallowed; see routingAnnounced. "Composing a reply…" specifically must be included so
+// it doesn't print (undimmed, since plainReplyFlow is already true by then) right before
+// the real streamed answer, where it would visually blend into the answer text.
 const isRoutingLine = (line) =>
-  /^(Routing your request…|Choosing a tool…|→ Agent mode.*|→ Chat reply)$/.test(line.trim());
+  /^(Routing your request…|Choosing a tool…|Composing a reply…|→ Agent mode.*|→ Chat reply)$/.test(
+    line.trim()
+  );
 
 // Playful words for the local "thinking…" ticker, reused verbatim from the worker's
 // thinking_words.txt sibling (falls back to a tiny built-in list if it's missing).
@@ -271,7 +276,15 @@ async function runAgentTurn(history) {
   let statusShown = false; // a transient status line is currently on screen
   let warnedPending = false;
   let jobRunning = false; // only tick "thinking…" once the worker has claimed the job
+  let answerShownLive = false; // plain-reply content already printed — don't re-print it
   let routingAnnounced = false; // one friendly line replaces the whole routing play-by-play
+  // True once we've SPECIFICALLY seen "→ Chat reply" (the plain-reply/trivial-chat
+  // branch), as opposed to "→ Agent mode…" (the tool-use branch). Only the plain-reply
+  // path streams its answer via answer_stream with NO heartbeat/tool-summary/code lines
+  // ever interleaved — so only THERE is it safe to partial-flush trailing content live
+  // (see carryFlushedLen below); the agent path's code/tool-call lines must stay buffered
+  // until a full line is known, or we'd risk showing part of something we meant to hide.
+  let plainReplyFlow = false;
 
   // --- Live "thinking…" ticker ---------------------------------------------------
   // Between visible tokens (prompt-eval / awaiting the first token) we show ONE in-place
@@ -343,18 +356,25 @@ async function runAgentTurn(history) {
   // keep spacing; everything else commits dim to scrollback.
   let inCode = false;
   let lastWasBlank = true; // suppresses a leading blank line and collapses repeats
+  // Chars of the CURRENT trailing (newline-less) `carry` line already flushed live — lets
+  // a long plain-reply answer stream character-by-character instead of waiting for a "\n"
+  // that might not arrive until the very end (most short replies have none at all).
+  let carryFlushedLen = 0;
   const consume = (chunk) => {
     carry += stripThinkTags(chunk);
     let nl;
     while ((nl = carry.indexOf("\n")) >= 0) {
       const line = carry.slice(0, nl);
       carry = carry.slice(nl + 1);
+      const alreadyFlushed = carryFlushedLen;
+      carryFlushedLen = 0; // reset — `carry` now starts a fresh pending line
       if (isHeartbeatLine(line) || isFenceLine(line)) continue; // swallow, no trace
       if (isToolStepSummary(line)) { lastContentAt = Date.now(); continue; } // swallow
       if (isRoutingLine(line)) {
         // Replace the whole routing play-by-play with ONE friendly line, shown once
         // per turn — the raw labels are implementation detail (visible in the daemon's
         // own terminal if you need them).
+        if (line.trim() === "→ Chat reply") plainReplyFlow = true;
         if (!routingAnnounced) {
           routingAnnounced = true;
           onContent();
@@ -386,8 +406,24 @@ async function runAgentTurn(history) {
         }
         continue;
       }
-      onContent();
-      process.stdout.write(dim(line) + "\n");
+      if (alreadyFlushed === 0) onContent();
+      // Plain-reply content IS the answer (not background narration) — print it in the
+      // terminal's default color, not dim, and remember it's already visible so the
+      // caller doesn't print the same text again once the turn completes.
+      const paint = plainReplyFlow ? (s) => s : dim;
+      process.stdout.write(paint(line.slice(alreadyFlushed)) + "\n");
+      if (plainReplyFlow) answerShownLive = true;
+      lastWasBlank = false;
+    }
+    // Plain-reply answers often have NO internal newline at all (a short one-sentence
+    // reply) and can take many seconds to generate — without this, nothing would print
+    // until the whole thing finally lands, right back to the "wait then dump" problem
+    // this was meant to fix. Only safe in the plain-reply flow (see plainReplyFlow above).
+    if (plainReplyFlow && !inCode && carry.length > carryFlushedLen) {
+      if (carryFlushedLen === 0) onContent();
+      process.stdout.write(carry.slice(carryFlushedLen)); // default color — this IS the answer
+      carryFlushedLen = carry.length;
+      answerShownLive = true;
       lastWasBlank = false;
     }
     // A non-empty remainder is a partial CONTENT line (heartbeats always arrive whole with
@@ -402,7 +438,7 @@ async function runAgentTurn(history) {
         process.stdout.write(
           dim("\n(stopped following — the job keeps running on the worker)\n")
         );
-        return "";
+        return { text: "", shown: false };
       }
       const jr = await fetch(`${apiBase}/api/worker/jobs/${jobId}`, {
         headers: { "X-Worker-Key": workerKey },
@@ -412,10 +448,11 @@ async function runAgentTurn(history) {
       jobRunning = job.jobStatus === "running";
       const text = typeof job.resultText === "string" ? job.resultText : "";
       if (job.jobStatus === "completed") {
-        // Drop the transient status line and any partial reasoning tail; the bright answer
-        // prints right after via the caller.
+        // Drop the transient status line and any partial reasoning tail. If this was a
+        // plain reply, its text is ALREADY fully visible (streamed live above) — the
+        // caller must not print it again; `shown` tells it so.
         clearStatus();
-        return answerFrom(text);
+        return { text: answerFrom(text), shown: answerShownLive };
       }
       if (text.length > shownChars) {
         consume(text.slice(shownChars));
@@ -512,10 +549,12 @@ async function main() {
     }
     history.push({ role: "user", content: line });
     try {
-      const answer = await runAgentTurn(history);
+      const { text: answer, shown } = await runAgentTurn(history);
       if (answer) {
         history.push({ role: "assistant", content: answer });
-        console.log(`\n\n${answer}\n`);
+        // A plain-reply answer already streamed live in the loop above — printing it
+        // again here would just duplicate it; a blank line is enough for spacing.
+        console.log(shown ? "" : `\n\n${answer}\n`);
       } else {
         history.pop(); // aborted/failed turn — don't poison the next request's history
         console.log(red("\n(no answer — run aborted or failed)\n"));

package/gonext_agent_chat.py

@@ -690,6 +690,43 @@ def _chat_create(client, **kwargs):
         raise
 
 
+def _chat_create_stream(client, on_delta, **kwargs) -> str:
+    """Like _chat_create but STREAMS the completion: on_delta(text) is called for each
+    content piece AS IT ARRIVES (so the caller can emit it live), and the full assembled
+    text is returned once the stream ends. Applies the same mlx_lm model-not-found retry
+    as _chat_create — that failure always happens at/near the very first network
+    round-trip (before any token streams), so a clean retry from scratch never risks
+    re-emitting a partial answer twice."""
+    kwargs = dict(kwargs)
+    kwargs["stream"] = True
+
+    def _run(kw):
+        stream = client.chat.completions.create(**kw)
+        parts = []
+        for chunk in stream:
+            choices = getattr(chunk, "choices", None)
+            delta = getattr(choices[0], "delta", None) if choices else None
+            piece = getattr(delta, "content", None) if delta else None
+            if piece:
+                parts.append(piece)
+                on_delta(piece)
+        return "".join(parts)
+
+    try:
+        return _run(kwargs)
+    except Exception as e:  # noqa: BLE001
+        if kwargs.get("model") != "default_model" and _is_model_not_found_err(e):
+            _log(f"model {kwargs.get('model')!r} not found on server; retrying as "
+                 "'default_model' (mlx_lm preloaded model)")
+            try:
+                kwargs2 = dict(kwargs)
+                kwargs2["model"] = "default_model"
+                return _run(kwargs2)
+            except Exception:  # noqa: BLE001
+                raise e  # surface the clearer original error, not the retry's
+        raise
+
+
 def _route(task_text: str, base_url: str, api_key: str, model_id: str) -> bool:
     """Decide if the task needs the HTTP agent (True) or a plain chat reply (False).
 
@@ -787,9 +824,12 @@ def _summarize_result(task_text: str, agent_output: str,
 def _plain_reply(messages: list, base_url: str, api_key: str, model_id: str,
                  fallback_base_url: str = "", fallback_model_id: str = "") -> str:
     """Plain chat completion using the full conversation history — a SMALL prompt (no tool
-    preamble), so it returns fast. If the primary model is unreachable (e.g. the local MLX
-    server is down) and a distinct fallback model is given (the coding model), retry there
-    so a greeting still gets answered instead of erroring."""
+    preamble), so it returns fast. STREAMS the answer live via {"type":"answer_stream"}
+    deltas as the model generates it (instead of blocking until the whole reply is ready
+    and delivering it in one burst) — the worker renders these as the live-updating MAIN
+    answer, not the collapsible Thinking panel. If the primary model is unreachable (e.g.
+    the local MLX server is down) and a distinct fallback model is given (the coding
+    model), retry there so a greeting still gets answered instead of erroring."""
     _THINK_RE_LOCAL = re.compile(r"<think>.*?</think>", re.DOTALL | re.IGNORECASE)
     chat_messages = [{"role": "system", "content": "You are a helpful assistant."}]
     for m in messages:
@@ -812,20 +852,30 @@ def _plain_reply(messages: list, base_url: str, api_key: str, model_id: str,
 
     last_err = None
     for i, (b, mid) in enumerate(targets):
+        emitted_here = False
+
+        def _track(piece):
+            nonlocal emitted_here
+            emitted_here = True
+            _emit({"type": "answer_stream", "text": piece})
+
         try:
             from openai import OpenAI
             client = OpenAI(base_url=b, api_key=api_key or "local",
                             max_retries=0, timeout=60)
-            resp = _chat_create(
-                client,
+            return _chat_create_stream(
+                client, _track,
                 model=mid,
                 messages=chat_messages,
                 temperature=0.7,
                 max_tokens=512,
-            )
-            return (resp.choices[0].message.content or "").strip()
+            ).strip()
         except Exception as e:  # noqa: BLE001
             last_err = e
+            if emitted_here:
+                # Part of THIS attempt's answer already streamed to the user — don't
+                # silently splice a DIFFERENT model's output after a partial one.
+                break
             if i + 1 < len(targets):
                 nb, nmid = targets[i + 1]  # the model we're about to try next
                 _log(f"plain reply: {mid} @ {b} failed ({e}); "

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiens.nguyen/gonext-local-worker",
-  "version": "1.0.188",
+  "version": "1.0.189",
   "description": "Polls GoNext cloud API for async local LLM jobs and runs them against Ollama/OpenAI-compatible servers on this Mac",
   "type": "module",
   "license": "MIT",