@tiens.nguyen/gonext-local-worker 1.0.197 → 1.0.199

package/gonext-repl.mjs

@@ -54,11 +54,16 @@ const CLEAR_LINE = "\r\x1b[K"; // carriage-return + erase-to-end-of-line
 // also drop the worker's ~~~ stream fences (they only matter to the web renderer).
 const isHeartbeatLine = (line) => /…\s?\(\d+s\)\s*$/.test(line);
 const isFenceLine = (line) => line.trim() === "~~~";
-// The model's raw Python code blob is delimited by <code>/</code> lines. We don't print
-// the tag lines themselves, but the code IN BETWEEN prints in a distinct grey (codeColor)
+// The model's raw Python code blob is delimited by <code>/</code> tags. We don't print
+// the tags themselves, but the code IN BETWEEN prints in a distinct grey (codeColor)
 // instead of dim prose, so it visually separates from the "Thought:" reasoning.
-const isCodeOpenLine = (line) => line.trim() === "<code>";
-const isCodeCloseLine = (line) => line.trim() === "</code>";
+// Tolerant matching, NOT exact-line: the raw token stream often mangles the tags —
+// glued to content ("<codecreate_folder(...)"), missing the ">", duplicated, or the
+// close tag stuck on the code's last line — and exact matching printed those
+// fragments literally in mixed colors (user-reported).
+const CODE_OPEN_RE = /^<code(?:\s*>)?/; // applied to line.trim()
+const CODE_CLOSE_RE = /<\/code>?\s*$/;
+const isBareCloseTag = (t) => /^<\/code>?$/.test(t);
 // The step summary the worker emits right after a tool call finishes (e.g.
 // "unzip_file(...) | → Unzipped 256 files…") duplicates what's already visible in the
 // gonext-local-worker daemon's own terminal — swallow it here to keep the REPL terse.
@@ -142,16 +147,19 @@ if (!workerKey || !apiBase) {
 // for piped/scripted input, harmless for interactive TTYs. Both ask() and the main
 // loop consume from this queue (never rl.question, to avoid double-capture).
 //
-// terminal: false — we draw our OWN prompt (cyan ">> ") and colorize output ourselves;
-// readline's default terminal:true (auto-enabled whenever output is a TTY) makes it ALSO
-// track cursor position and redraw the input line on every keystroke using ITS OWN
-// (empty, since we never call rl.setPrompt/rl.prompt) prompt — that tracking goes stale
-// the moment we write anything readline doesn't know about (which is everything we print
-// between prompts), and its next redraw emits a stray leading "[" (a malformed/partial
-// cursor-position escape) right before typed input, e.g. "[>> hi". With terminal:false,
-// readline stops managing the line/cursor itself; the OS's own canonical-mode terminal
-// still echoes keystrokes normally, and 'line' events fire exactly the same.
-const rl = createInterface({ input: process.stdin, output: process.stdout, terminal: false });
+// terminal: true (on real TTYs) with readline OWNING the prompt via setPrompt/prompt().
+// History of this setting: plain terminal:true with a MANUALLY-written ">> " once caused
+// a stray "[" artifact — readline redrew the input line using its own (empty, never-set)
+// prompt and cursor tracking that our un-announced writes had made stale. The interim
+// fix (terminal:false) traded that for something worse: the OS's canonical line
+// discipline echoes raw CSI bytes, so arrows/Home/End showed literal "^[[D" and mid-line
+// editing didn't work at all (user-reported). Giving readline the prompt makes its
+// redraw math correct — rl.prompt() resets the cursor state after every burst of our
+// output — and raw mode means backspace/arrows/↑-history all behave like a normal shell.
+const IS_TTY = Boolean(process.stdin.isTTY && process.stdout.isTTY);
+const rl = createInterface({ input: process.stdin, output: process.stdout, terminal: IS_TTY });
+const PROMPT = cyan(">> ");
+rl.setPrompt(PROMPT);
 // Side effect of terminal:false above: readline no longer intercepts arrow keys for
 // command-history recall, so pressing ↑/↓/→/← (or Home/End/Delete) sends the RAW escape
 // bytes straight through as literal input — the OS's own canonical line discipline
@@ -189,8 +197,13 @@ const nextLine = () =>
       ? Promise.resolve(null)
       : new Promise((res) => (_lineWaiter = res));
 const ask = async (q) => {
-  process.stdout.write(q);
+  // Route the question through readline's own prompt so its redraw logic stays
+  // consistent (a raw stdout write here would re-create the stale-cursor artifact
+  // that terminal:true mode is designed to avoid). Restored to ">> " afterwards.
+  rl.setPrompt(q);
+  rl.prompt();
   const a = await nextLine();
+  rl.setPrompt(PROMPT);
   return (a ?? "").trim();
 };
 
@@ -510,14 +523,48 @@ async function runAgentTurn(history) {
         }
         inEditCard = false; // card ended — fall through to normal handling
       }
-      if (!inCode && isCodeOpenLine(line)) {
-        stampIfThinking(); // hold/refresh the stamp; the code body commits it below
-        inCode = true;
-        continue;
+      {
+        const t = line.trim();
+        const openM = !inCode || t.startsWith("<code") ? CODE_OPEN_RE.exec(t) : null;
+        if (openM) {
+          // Enter code mode (or stay in it on a duplicate tag). Anything glued after
+          // the tag on the same line is code — print it, minus a glued close tag.
+          stampIfThinking(); // hold/refresh the stamp; the code body commits it below
+          inCode = true;
+          let rest = t.slice(openM[0].length).trim();
+          if (CODE_CLOSE_RE.test(rest)) {
+            inCode = false;
+            rest = rest.replace(CODE_CLOSE_RE, "").trim();
+          }
+          if (rest) {
+            onContent();
+            process.stdout.write(codeColor(rest) + "\n");
+            lastWasBlank = false;
+          }
+          lastContentAt = Date.now();
+          continue;
+        }
+        if (!inCode && isBareCloseTag(t)) {
+          // Stray close tag with no open in sight — swallow, never show it.
+          lastContentAt = Date.now();
+          continue;
+        }
       }
       if (inCode) {
-        if (isCodeCloseLine(line)) { inCode = false; lastContentAt = Date.now(); continue; }
-        if (line.trim() === "") { lastContentAt = Date.now(); continue; } // skip blank lines in code
+        const t = line.trim();
+        if (CODE_CLOSE_RE.test(t)) {
+          // Close tag, possibly with the code's last line glued in front of it.
+          const before = t.replace(CODE_CLOSE_RE, "").trim();
+          if (before) {
+            onContent();
+            process.stdout.write(codeColor(before) + "\n");
+            lastWasBlank = false;
+          }
+          inCode = false;
+          lastContentAt = Date.now();
+          continue;
+        }
+        if (t === "") { lastContentAt = Date.now(); continue; } // skip blank lines in code
         onContent();
         process.stdout.write(codeColor(line) + "\n");
         lastWasBlank = false;
@@ -637,20 +684,29 @@ async function main() {
       dim(`Resumed session: ${turns} prior turn(s) from last time in this folder — /reset to start fresh.\n`)
     );
   }
-  // With terminal:false, Ctrl-C is no longer intercepted by readline's raw-mode byte
-  // scanning (rl.on("SIGINT", …) would never fire) — it now delivers a real OS SIGINT to
-  // the process instead, which we catch directly here. Same behavior as before.
-  process.on("SIGINT", () => {
+  // Ctrl-C arrives on ONE of two paths depending on the readline mode: with
+  // terminal:true (real TTY) readline's raw-mode byte scanning intercepts \x03 and
+  // emits rl "SIGINT" (no OS signal is delivered); with terminal:false (piped input)
+  // the OS delivers a real SIGINT to the process. Attach the same handler to both —
+  // only one can ever fire for a given mode, so there's no double-handling.
+  const onInterrupt = () => {
     if (following) {
       followAborted = true;
       process.stdout.write(red("\n^C "));
     } else {
-      process.stdout.write(dim("\n(/exit to quit)\n") + cyan(">> "));
+      process.stdout.write(dim("\n(/exit to quit)\n"));
+      // Discard anything half-typed (Ctrl-C at a prompt means "never mind") and
+      // redraw through readline so its cursor tracking stays right.
+      rl.line = "";
+      rl.cursor = 0;
+      rl.prompt();
     }
-  });
+  };
+  process.on("SIGINT", onInterrupt);
+  rl.on("SIGINT", onInterrupt);
 
   for (;;) {
-    process.stdout.write(cyan(">> "));
+    rl.prompt();
     // Absorb blank Enter-presses SILENTLY under the same prompt — without this, lines
     // typed while the agent was busy on a long turn (queued, not yet read) all drain at
     // once the moment the turn ends, each re-printing ">> " with no newline between them

package/gonext_agent_chat.py

@@ -610,6 +610,13 @@ _WS_ACTION_RE = re.compile(
     r"\b(?:create|make|add|new|delete|remove|rename|move|copy|list|show)\b"
     r"[^.?!]{0,60}?"
     r"\b(?:folders?|director(?:y|ies)|dirs?|files?|subfolders?)\b"
+    # Run/verify actions on the project itself ("start and test the project", "run
+    # the app", "build it and run the tests") — second reported miss (task #40):
+    # none of these words were keywords anywhere, so "help me start and test the
+    # project" got a step-by-step npm tutorial instead of the agent running them.
+    r"|\b(?:run|start|launch|build|test|install|compile|verify|stop|kill|restart)\b"
+    r"[^.?!]{0,60}?"
+    r"\b(?:project|app|application|server|site|website|tests?|build|dependenc(?:y|ies)|packages?)\b"
     r"|\bmkdir\b",
     re.IGNORECASE,
 )
@@ -825,6 +832,20 @@ def _route(task_text: str, base_url: str, api_key: str, model_id: str) -> bool:
         from openai import OpenAI
         client = OpenAI(base_url=base_url, api_key=api_key or "local",
                         max_retries=0, timeout=20)
+        # The classifier's question must reflect the capabilities that ACTUALLY exist
+        # this run — with a workspace registered, acting on files/code/commands is
+        # agent work too, and pronoun phrasings ("test it") that the deterministic
+        # keyword gates can't safely match land here. Without this clause the prompt
+        # was HTTP-only (it predates the workspace tools), so NO was the classifier's
+        # CORRECT answer to the wrong question for every workspace action.
+        ws_clause = (
+            "\nThe user also has a code workspace registered, so answer YES as well "
+            "when the task asks to read, edit, create, delete, or list files or "
+            "folders, run tests/builds/commands, or start, stop, or restart the "
+            "project or its server — even when phrased with pronouns like 'test it' "
+            "or 'run it'."
+        ) if _WS_ROOTS else ""
+        ws_q = ", files, code, or commands" if _WS_ROOTS else ""
         resp = _chat_create(
             client,
             model=model_id,
@@ -833,12 +854,13 @@ def _route(task_text: str, base_url: str, api_key: str, model_id: str) -> bool:
                     "You are a task classifier. Reply YES or NO only, no punctuation.\n"
                     "Answer YES if the task requires fetching data from an external network source "
                     "(URL, API, website, remote server), a web search / factual lookup, or the "
-                    "current date or time.\n"
-                    "Answer NO only if it is pure conversation, opinion, or simple text the "
-                    "assistant can answer directly without looking anything up."
+                    "current date or time."
+                    + ws_clause +
+                    "\nAnswer NO only if it is pure conversation, opinion, or simple text the "
+                    "assistant can answer directly without looking anything up or touching anything."
                 )},
                 {"role": "user", "content": (
-                    f"Does this task require fetching data from an external network source?\n\n"
+                    f"Does this task require using tools (network{ws_q})?\n\n"
                     f"Task: {task_text}\n\nYES or NO:"
                 )},
             ],
@@ -2064,6 +2086,77 @@ _WS_RUN_ALLOWED = {
     "rspec", "bundle", "php", "composer", "tsc", "jest", "vitest",
 }
 
+# ---- background servers (behavior-detected, command-agnostic) ----
+# run_command classifies a command by OBSERVED BEHAVIOR, never by its text: if the
+# process keeps running AND starts LISTENing on a TCP port, it IS a server — npm start,
+# bun dev, a custom `make serve`, anything. (An earlier version pattern-matched command
+# names like "npm start"; rejected as unwinnable hardcode — same lesson as task #37.)
+# Detected servers are left running in their own process group (start_new_session),
+# recorded in ~/.gonext/servers.json, and stoppable via the stop_server tool.
+
+
+def _ws_pgroup_pids(pgid: int) -> list:
+    """Live pids in a process group (the server + everything it spawned)."""
+    import subprocess
+    try:
+        out = subprocess.run(["pgrep", "-g", str(pgid)],
+                             stdout=subprocess.PIPE, text=True, timeout=5)
+        return [int(x) for x in out.stdout.split()]
+    except Exception:  # noqa: BLE001
+        return []
+
+
+def _ws_listening_ports(pgid: int) -> list:
+    """TCP ports the process group is LISTENing on — the behavioral definition of
+    'this command is a server'. Empty when none (or lsof/pgrep unavailable)."""
+    import subprocess
+    pids = _ws_pgroup_pids(pgid)
+    if not pids:
+        return []
+    try:
+        out = subprocess.run(
+            ["lsof", "-a", "-p", ",".join(map(str, pids)),
+             "-iTCP", "-sTCP:LISTEN", "-P", "-n"],
+            stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=5)
+        ports = set()
+        for line in out.stdout.splitlines()[1:]:
+            m = re.search(r":(\d+)\s+\(LISTEN\)", line)
+            if m:
+                ports.add(int(m.group(1)))
+        return sorted(ports)
+    except Exception:  # noqa: BLE001
+        return []
+
+
+def _ws_servers_path() -> str:
+    import os
+    base = os.path.join(os.path.expanduser("~"), ".gonext")
+    os.makedirs(base, exist_ok=True)
+    return os.path.join(base, "servers.json")
+
+
+def _ws_load_servers(prune: bool = True) -> list:
+    """Registry of background servers we started. `prune` drops entries whose
+    process group is gone (crashed or stopped outside of us)."""
+    import json as _json
+    try:
+        with open(_ws_servers_path(), encoding="utf-8") as fh:
+            servers = _json.load(fh).get("servers", [])
+    except Exception:  # noqa: BLE001
+        return []
+    if prune:
+        servers = [s for s in servers if _ws_pgroup_pids(int(s.get("pgid", 0)))]
+    return servers
+
+
+def _ws_save_servers(servers: list) -> None:
+    import json as _json
+    try:
+        with open(_ws_servers_path(), "w", encoding="utf-8") as fh:
+            _json.dump({"servers": servers}, fh, indent=2)
+    except OSError as e:
+        _log(f"servers.json write failed: {e}")
+
 
 def _ws_scrubbed_env() -> dict:
     """Minimal child env for run_command. The worker process env holds secrets
@@ -2072,7 +2165,11 @@ def _ws_scrubbed_env() -> dict:
     import os
     keep = ("PATH", "HOME", "LANG", "LC_ALL", "TMPDIR", "SHELL", "USER", "TERM",
             "JAVA_HOME", "GOPATH", "CARGO_HOME", "NVM_DIR")
-    return {k: os.environ[k] for k in keep if k in os.environ}
+    env = {k: os.environ[k] for k in keep if k in os.environ}
+    # CI=1 makes watch-mode test runners (CRA's `npm test`, jest --watch) run ONCE and
+    # exit instead of sitting in interactive watch mode until the timeout kills them.
+    env["CI"] = "1"
+    return env
 
 
 def _ws_distill_output(out: str, cap_head: int = 1200, cap_tail: int = 4000) -> str:
@@ -4154,17 +4251,19 @@ def run_agent_chat(cfg):
 
         @tool
         def run_command(command: str, workdir: str = "", timeout_seconds: int = 180) -> str:
-            """Run a build/test command inside a workspace that has run permission (e.g. 'npm test', 'mvn test', 'pytest'). Use to VERIFY code changes; read the output and fix failures.
+            """Run a command inside a workspace that has run permission (e.g. 'npm test', 'npm run build', 'npm start', 'mvn test'). One-shot commands return their output when they exit. If the command turns out to be a SERVER (keeps running and listens on a port), it is left RUNNING in the background and its URL is reported — tell the user the URL, and use stop_server to stop it when asked.
 
             Args:
-                command: the command line (build/test runners only; no shells or sudo).
+                command: the command line (allowlisted runners only; no shells or sudo).
                 workdir: directory to run in. Empty = first registered workspace.
-                timeout_seconds: kill the command after this many seconds (max 600).
+                timeout_seconds: give up after this many seconds if the command neither exits nor starts a server (max 600).
             """
             try:
                 import os
                 import shlex
+                import signal
                 import subprocess
+                import time as _t
                 wd = _rag_read_allowed((workdir or "").strip() or (_WS_ROOTS[0]["path"] if _WS_ROOTS else "."))
                 w = _ws_for_path(wd)
                 if w is None or not w.get("allowRun"):
@@ -4184,23 +4283,128 @@ def run_agent_chat(cfg):
                     return msg
                 _emit({"type": "step", "text": f"Running → {command[:70]}"})
                 t = max(5, min(int(timeout_seconds or 180), 600))
-                # Scrubbed env: repo test scripts must never see worker secrets.
-                proc = subprocess.run(
-                    argv, cwd=wd, env=_ws_scrubbed_env(), timeout=t,
-                    stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
+                # Own process group (start_new_session) + output to a log file. This is
+                # what lets a server OUTLIVE this job process, and what lets a timeout
+                # kill take the runner's whole child tree (npm's node child etc. — the
+                # old subprocess.run timeout killed only the direct child, leaking it).
+                logs_dir = os.path.join(os.path.expanduser("~"), ".gonext", "run-logs")
+                os.makedirs(logs_dir, exist_ok=True)
+                log_path = os.path.join(logs_dir, f"{_t.strftime('%Y%m%d-%H%M%S')}-{os.getpid()}.log")
+                log_fh = open(log_path, "wb")
+                # Scrubbed env: repo scripts must never see worker secrets.
+                proc = subprocess.Popen(
+                    argv, cwd=wd, env=_ws_scrubbed_env(),
+                    stdout=log_fh, stderr=subprocess.STDOUT,
+                    start_new_session=True,
                 )
-                out = _ws_distill_output(proc.stdout or "")
-                status = "PASSED (exit 0)" if proc.returncode == 0 else f"FAILED (exit {proc.returncode})"
-                _emit({"type": "step", "text": f"Command {status.split()[0].lower()} → {command[:50]}"})
-                result = f"{status}\n{out}" if out.strip() else status
-                _last_obs["text"] = result
-                return result
-            except subprocess.TimeoutExpired:
-                msg = f"Error: command timed out after {timeout_seconds}s."
+                start = _t.time()
+
+                def _read_log() -> str:
+                    log_fh.flush()
+                    try:
+                        with open(log_path, "r", encoding="utf-8", errors="replace") as fh:
+                            return fh.read()
+                    except OSError:
+                        return ""
+
+                # Behavior classification loop — no command-name matching anywhere:
+                # exited → one-shot result; still running AND listening on a TCP port →
+                # it's a server, leave it running; neither by the deadline → kill group.
+                while True:
+                    rc = proc.poll()
+                    if rc is not None:
+                        out = _ws_distill_output(_read_log())
+                        status = "PASSED (exit 0)" if rc == 0 else f"FAILED (exit {rc})"
+                        _emit({"type": "step", "text": f"Command {status.split()[0].lower()} → {command[:50]}"})
+                        result = f"{status}\n{out}" if out.strip() else status
+                        _last_obs["text"] = result
+                        return result
+                    if _t.time() - start >= 3:
+                        ports = _ws_listening_ports(proc.pid)
+                        if ports:
+                            servers = _ws_load_servers()
+                            servers.append({
+                                "pgid": proc.pid, "pid": proc.pid, "command": command,
+                                "workdir": wd, "ports": ports, "log": log_path,
+                                "started_at": _t.strftime("%Y-%m-%d %H:%M:%S"),
+                            })
+                            _ws_save_servers(servers)
+                            urls = ", ".join(f"http://localhost:{p}" for p in ports)
+                            _emit({"type": "step", "text": f"Server up → {urls} (left running)"})
+                            tail = _ws_distill_output(_read_log())
+                            result = (f"RUNNING: '{command}' is up and listening — {urls} "
+                                      f"(pid {proc.pid}, log {log_path}). It stays running in "
+                                      "the background; use stop_server to stop it. "
+                                      f"Startup output:\n{tail}")
+                            _last_obs["text"] = result
+                            return result
+                    if _t.time() - start >= t:
+                        try:
+                            os.killpg(proc.pid, signal.SIGKILL)
+                        except OSError:
+                            pass
+                        tail = _ws_distill_output(_read_log())
+                        msg = (f"Error: command neither exited nor started a server within "
+                               f"{t}s — killed. Output so far:\n{tail}")
+                        _last_obs["text"] = msg
+                        return msg
+                    _t.sleep(2)
+            except Exception as ex:  # noqa: BLE001
+                msg = f"Error: run_command failed: {type(ex).__name__}: {ex}"
                 _last_obs["text"] = msg
                 return msg
+            finally:
+                try:
+                    log_fh.close()
+                except Exception:  # noqa: BLE001
+                    pass
+
+        @tool
+        def stop_server(port_or_pid: str = "") -> str:
+            """Stop a background server previously started by run_command. Use when the user asks to stop/kill the server.
+
+            Args:
+                port_or_pid: the server's port or pid (empty = the most recently started one).
+            """
+            try:
+                import os
+                import signal
+                import time as _t
+                servers = _ws_load_servers()
+                if not servers:
+                    msg = "No background servers are running."
+                    _last_obs["text"] = msg
+                    return msg
+                key = (port_or_pid or "").strip()
+                target = None
+                if key:
+                    for s in servers:
+                        if key == str(s.get("pid")) or any(key == str(p) for p in s.get("ports", [])):
+                            target = s
+                            break
+                    if target is None:
+                        listing = "; ".join(
+                            f"pid {s['pid']} ports {s.get('ports')} ({s['command']})" for s in servers)
+                        msg = f"No running server matches '{key}'. Running: {listing}"
+                        _last_obs["text"] = msg
+                        return msg
+                else:
+                    target = servers[-1]
+                pgid = int(target["pgid"])
+                try:
+                    os.killpg(pgid, signal.SIGTERM)
+                    _t.sleep(1.5)
+                    if _ws_pgroup_pids(pgid):
+                        os.killpg(pgid, signal.SIGKILL)
+                except OSError:
+                    pass
+                _ws_save_servers([s for s in servers if s is not target])
+                _emit({"type": "step", "text": f"Server stopped → {target['command'][:50]}"})
+                out = f"Stopped '{target['command']}' (pid {target['pid']}, ports {target.get('ports')})."
+                _last_obs["text"] = out
+                return out
             except Exception as ex:  # noqa: BLE001
-                msg = f"Error: run_command failed: {type(ex).__name__}: {ex}"
+                msg = f"Error: stop_server failed: {type(ex).__name__}: {ex}"
                 _last_obs["text"] = msg
                 return msg
 
@@ -4218,7 +4422,7 @@ def run_agent_chat(cfg):
                             rag_index, rag_add, rag_search]
         if _WS_AVAILABLE:
             agent_tools += [grep_repo, read_file_lines, edit_lines, edit_file,
-                            create_file, create_folder, run_command]
+                            create_file, create_folder, run_command, stop_server]
             # list_dir/read_text_file are needed for workspace browsing even when RAG
             # (S3 indexing) isn't configured.
             if not _RAG_AVAILABLE:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiens.nguyen/gonext-local-worker",
-  "version": "1.0.197",
+  "version": "1.0.199",
   "description": "Polls GoNext cloud API for async local LLM jobs and runs them against Ollama/OpenAI-compatible servers on this Mac",
   "type": "module",
   "license": "MIT",