@tiens.nguyen/gu-cli 1.0.787 → 1.0.789

package/gonext_agent_chat.py

@@ -5183,6 +5183,90 @@ def _same_embed_model(a: str, b: str) -> bool:
 _RUNTOOL_PREFIX = "__RUNTOOL__::"
 
 
+# ---- the command that is running, WHOLE (task #331) ------------------------------------------
+# "Running → <cmd>" carries the first 70 characters, which is right for a history line and wrong
+# for the one moment someone is watching: a heredoc or a long pipeline was cut at the source, so
+# the terminal could not show the rest whatever its layout. Sent beside that line (which stays,
+# for the web app and for older terminals) as ONE line of JSON — a command can span lines.
+_RUNCMD_PREFIX = "__RUNCMD__::"
+# How much it has printed so far: "__RUNOUT__::<lines>". From the wait loop, only when it changed.
+_RUNOUT_PREFIX = "__RUNOUT__::"
+_RUNCMD_MAX_CHARS = 4000
+
+# SECRETS. The 70-character cut hid most credentials by accident; showing the whole command must
+# not undo that. Conservative and by SHAPE — a value after a name that says it is a secret, a
+# bearer header, user:password@ in a URL, sshpass -p, and the key formats gu itself hands out.
+# This is what goes to the SCREEN and the job stream; the command that runs is untouched.
+# "pass" and "token" only as whole words: --passes=3, bypass=true and --max-tokens 100 are not secrets.
+_SECRET_NAME = (r"(?:password|passwd|passphrase|pwd|(?<![a-z])pass(?![a-z])|secret|token(?![a-z])"
+                r"|api[_-]?key|access[_-]?key)")
+_CMD_SECRET_RES = (
+    (re.compile(r"(?i)(\b[\w.-]*" + _SECRET_NAME + r"[\w.-]*\s*[=:]\s*)(\"[^\"]*\"|'[^']*'|[^\s;&|]+)"), r"\1•••"),
+    (re.compile(r"(?i)(--?[\w-]*" + _SECRET_NAME + r"[\w-]*[ =])(\"[^\"]*\"|'[^']*'|[^\s;&|]+)"), r"\1•••"),
+    (re.compile(r"(?i)(\bsshpass\s+-p\s*)(\"[^\"]*\"|'[^']*'|[^\s;&|]+)"), r"\1•••"),
+    (re.compile(r"(?i)(authorization:\s*(?:bearer|basic)\s+)[^\s\"']+"), r"\1•••"),
+    (re.compile(r"(?i)(\b[a-z][a-z0-9+.-]*://[^\s/:@]+:)[^\s/@]+(@)"), r"\1•••\2"),
+    (re.compile(r"\b(?:sk-[A-Za-z0-9_-]{16,}|gh[pousr]_[A-Za-z0-9]{20,}|wk_[A-Za-z0-9_-]{16,}|gu1\.[A-Za-z0-9_.-]{20,})"), "•••"),
+)
+
+
+def _mask_command_secrets(command: str) -> str:
+    """The command as it may be SHOWN: credentials replaced by •••, everything else as written."""
+    text = str(command or "")
+    for rx, sub in _CMD_SECRET_RES:
+        text = rx.sub(sub, text)
+    return text
+
+
+def _command_label(command: str, limit: int = 70) -> str:
+    """A command as ONE LINE of a step label: masked, its first line only, cut to `limit`.
+
+    A step is a line of the stream. A heredoc's newlines made "Command finished → python3 - <<'EOF'"
+    followed by every line of the script as a bullet of its own — and, on the way in, ended the
+    live status row the moment the "Running →" line was read (#332). " …" says there is more.
+    """
+    lines = [ln.strip() for ln in _mask_command_secrets(command).replace("\r", "").split("\n") if ln.strip()]
+    if not lines:
+        return ""
+    return lines[0][:limit] + (" …" if len(lines) > 1 else "")
+
+
+def _running_command_line(command: str) -> str:
+    """The stream line that carries the whole command to the terminal (masked, capped)."""
+    shown = _mask_command_secrets(command)
+    if len(shown) > _RUNCMD_MAX_CHARS:
+        shown = shown[:_RUNCMD_MAX_CHARS] + "…"
+    return _RUNCMD_PREFIX + json.dumps({"cmd": shown}, ensure_ascii=False)
+
+
+def _count_output_lines(path: str, state: dict) -> int:
+    """Lines a running command has written to its log so far. Reads only what is NEW since the
+    last call (`state` keeps the offset), so a chatty ten-minute build is not re-read every tick."""
+    try:
+        with open(path, "rb") as fh:
+            fh.seek(state.get("off", 0))
+            chunk = fh.read()
+    except OSError:
+        return state.get("lines", 0)
+    if chunk:
+        state["off"] = state.get("off", 0) + len(chunk)
+        state["lines"] = state.get("lines", 0) + chunk.count(b"\n")
+        state["partial"] = not chunk.endswith(b"\n")
+    # a last line with no newline yet is still a line someone could read
+    return state.get("lines", 0) + (1 if state.get("partial") else 0)
+
+
+def _emit_command_progress(path: str, state: dict) -> None:
+    """Tell the terminal how much the running command has printed — only when that changed."""
+    try:
+        n = _count_output_lines(path, state)
+        if n != state.get("sent", 0):
+            state["sent"] = n
+            _emit({"type": "step", "text": f"{_RUNOUT_PREFIX}{n}"})
+    except Exception:  # noqa: BLE001 — a progress figure must never fail a command
+        pass
+
+
 def _fmt_call(name: str, **kw) -> str:
     """A tool call rendered the way the model writes it. Skips empty/default arguments."""
     parts = []
@@ -6391,6 +6475,114 @@ def _and_chain_parts(command):
     return parts
 
 
+def _parse_heredoc(command):
+    """A heredoc or here-string in `command` → what to run and what to feed it. (task #332)
+
+    THE BUG THIS EXISTS FOR. run_command has no shell, so
+
+        python3 - <<'EOF'
+        print("hello")
+        EOF
+
+    became argv ["python3", "-", "<<EOF", "print(hello)", "EOF"]: python read its script from an
+    EMPTY stdin and exited 0 with no output. Nothing ran and the step looked like a success —
+    the #204 failure again, in the one shape #204's check did not cover (its redirect pattern
+    matches "<" and ">", not "<<"). Seen live: the model lost two steps rewriting the work as
+    `python3 -c`. Heredocs are how a strong model runs a multi-line script without a file, so
+    this is the common case, not an edge.
+
+    SUPPORTED, still with no shell: ONE program fed a body.
+        PROG ARGS <<'TAG' … TAG      the body becomes the program's stdin
+        PROG ARGS <<-TAG … TAG       same, leading tabs stripped
+        PROG ARGS <<< 'text'         a here-string: text + newline on stdin
+    REFUSED, each with the one change that fixes it — a guess costs a model step:
+        an unquoted TAG (or a double-quoted here-string) whose text has $ or ` — a shell would
+        expand those and we cannot; anything after the closing TAG; no closing TAG; two heredocs.
+
+    Returns None when there is no heredoc (`echo "a << b"` and `python3 -c "print(1 << 3)"` are
+    not: the operator only counts OUTSIDE quotes), else {"command", "stdin"} or {"error"}.
+    Pure, so every shape is testable without spawning anything.
+    """
+    s = command or ""
+    q = None
+    i = 0
+    at = -1
+    while i < len(s):
+        c = s[i]
+        if q:
+            if c == q:
+                q = None
+            elif c == "\\" and q == '"':
+                i += 1
+        elif c in "'\"":
+            q = c
+        elif c == "\\":
+            i += 1
+        elif s.startswith("<<", i):
+            at = i
+            break
+        i += 1
+    if at < 0:
+        return None
+    _why = "Error: that command did NOT run — nothing was executed. "
+    before = s[:at]
+    if s.startswith("<<<", at):
+        # HERE-STRING: the next word is the input; anything after it stays an argument.
+        rest = s[at + 3:]
+        import shlex as _shlex
+        try:
+            words = _shlex.split(rest)
+        except ValueError as e:
+            return {"error": _why + f"The here-string is not closed: {e}."}
+        if not words:
+            return {"error": _why + "'<<<' needs the text to feed the program: PROG <<< 'text'."}
+        lead = rest.lstrip()
+        if not lead.startswith("'") and re.search(r"[$`]", words[0]):
+            return {"error": _why + "run_command has NO shell, so $variables and `commands` in a "
+                    "here-string are not expanded. Put the text in SINGLE quotes to pass it "
+                    "exactly as written, or write the final text yourself."}
+        tail = " ".join(_shlex.quote(w) for w in words[1:])
+        return {"command": (before.rstrip() + (" " + tail if tail else "")).strip(),
+                "stdin": words[0] + "\n"}
+    m = re.match(r"<<(-?)[ \t]*(?:'([^'\n]+)'|\"([^\"\n]+)\"|\\?([A-Za-z_][\w.-]*))", s[at:])
+    if not m:
+        return {"error": _why + "'<<' must be followed by a tag: PROG <<'EOF', the text, then "
+                "EOF on a line of its own."}
+    strip_tabs = bool(m.group(1))
+    tag = m.group(2) or m.group(3) or m.group(4)
+    quoted = bool(m.group(2) or m.group(3)) or s[at + m.end() - len(tag) - 1] == "\\"
+    after = s[at + m.end():]
+    line_rest, nl, body_and_more = after.partition("\n")
+    if "<<" in line_rest and _parse_heredoc(line_rest) is not None:
+        return {"error": _why + "Only one heredoc per command is supported. Write the second "
+                "text with create_file, or run the two commands in separate calls."}
+    lines = body_and_more.split("\n") if nl else []
+    end = -1
+    for k, ln in enumerate(lines):
+        if (ln.lstrip("\t") if strip_tabs else ln).rstrip("\r") == tag:
+            end = k
+            break
+    if end < 0:
+        return {"error": _why + f"The heredoc is never closed: put {tag} on a line of its own "
+                "after the text."}
+    body_lines = [(ln.lstrip("\t") if strip_tabs else ln).rstrip("\r") for ln in lines[:end]]
+    trailing = "\n".join(lines[end + 1:]).strip()
+    if trailing:
+        return {"error": _why + f"There is more after the closing {tag}, and run_command runs "
+                "ONE command per call (only && chains continue). Run what follows in a "
+                "separate run_command call."}
+    cmdline = (before.rstrip() + " " + line_rest.strip()).strip()
+    body = "\n".join(body_lines) + ("\n" if body_lines else "")
+    if not quoted and re.search(r"[$`]", body):
+        return {"error": _why + "run_command has NO shell, so the $variables and `commands` in "
+                f"this heredoc would not be expanded. Quote the tag — <<'{tag}' — to pass the "
+                "text exactly as written (that is what a script wants), or write the final "
+                "text yourself."}
+    if not cmdline:
+        return {"error": _why + "The heredoc has no program to feed: PROG <<'EOF' … EOF."}
+    return {"command": cmdline, "stdin": body}
+
+
 def _shell_syntax_error(command, argv):
     """Why this command cannot do what it says, or None. (task #204)
 
@@ -6963,7 +7155,7 @@ def _ws_request_approval(api_base, worker_key, job_id, command, reason, deadline
         _srv, _, _rest = command[len("__MCPTOOL__::"):].partition("\t")
         _emit({"type": "step", "text": f"Awaiting your approval: MCP tool {_srv}/{_rest.split(chr(9))[0]}"})
     else:
-        _emit({"type": "step", "text": f"Awaiting your approval to run: {command[:70]}"})
+        _emit({"type": "step", "text": f"Awaiting your approval to run: {_command_label(command, 70)}"})
     deadline = _t.time() + max(30, int(deadline_s))  # per-call; risky=180s, __MAXSTEP__ long
     while _t.time() < deadline:
         _t.sleep(1.2)
@@ -7341,6 +7533,12 @@ def _ws_scrubbed_env() -> dict:
     # CI=1 makes watch-mode test runners (CRA's `npm test`, jest --watch) run ONCE and
     # exit instead of sitting in interactive watch mode until the timeout kills them.
     env["CI"] = "1"
+    # WINDOWS: a python child reads and writes in the ANSI code page unless told otherwise, so a
+    # script fed on stdin (a heredoc, #332) with "é" or "→" in it is mis-decoded, and printing
+    # one raises UnicodeEncodeError. The worker's own python is started with PYTHONUTF8=1 for
+    # exactly this reason; the whitelist above dropped it for the commands it runs.
+    if _IS_WINDOWS:
+        env["PYTHONUTF8"] = "1"
     return env
 
 
@@ -12501,7 +12699,7 @@ def run_agent_chat(cfg):
             """Run a command inside a workspace that has run permission (e.g. 'npm test', 'npm run build', 'npm start', 'mvn test', 'python3 app.py'). One-shot commands return their output when they exit. If the command turns out to be a SERVER (keeps running and listens on a port), it is left RUNNING and its URL is reported. If it is a DESKTOP/GUI app (opens a window — tkinter, pygame, PyQt…), it is LAUNCHED and left running: its window appears on the user's Mac; tell them it's open. Both are stoppable via stop_server. Also allows DEPLOY commands (ssh/scp/rsync) — use non-interactive KEY auth (-o BatchMode=yes); never put a password in the command (there is no stdin). Note: an interactive CLI that reads stdin (input()) cannot be driven here — it will fail with EOFError; make it non-interactive or give the user the command to run themselves.
 
             Args:
-                command: the command line. Chain dependent steps with && in ONE call ("cd app && npm ci && npm test"): they run in order, cd sets the folder, the first failure stops the rest. No other shell syntax (no pipes/redirects/;). sudo/su are blocked.
+                command: the command line. Chain dependent steps with && in ONE call ("cd app && npm ci && npm test"): they run in order, cd sets the folder, the first failure stops the rest. To run a multi-line script without a file, feed ONE program a heredoc, tag quoted: "python3 - <<'EOF'\n...\nEOF". No other shell syntax (no pipes/redirects/;). sudo/su are blocked.
                 workdir: directory to run in. Empty = first registered workspace.
                 timeout_seconds: give up after this many seconds if the command neither exits nor starts a server (max 600).
             """
@@ -12546,7 +12744,24 @@ def run_agent_chat(cfg):
                            "gu-local-worker workspace add <path> --allow-run)")
                     _last_obs["text"] = msg
                     return msg
-                argv = shlex.split(command or "")
+                # A HEREDOC / HERE-STRING (task #332): the text is the program's stdin, and the
+                # command line is what is left. `command` stays the whole text for the risk gate,
+                # the status row and the log; `_cmdline` is what is split and judged as argv.
+                _here = _parse_heredoc(command)
+                if _here and _here.get("error"):
+                    _log(f"run_command refused heredoc: {_c(_clip(command, 120))}")
+                    _last_obs["text"] = _here["error"]
+                    return _here["error"]
+                _cmdline = _here["command"] if _here else (command or "")
+                _stdin_text = _here["stdin"] if _here else None
+                if _here and _and_chain_parts(_cmdline):
+                    msg = ("Error: that command did NOT run — nothing was executed. A heredoc "
+                           "cannot be part of an && chain here. Run the program that reads it in "
+                           "its own run_command call, and pass the folder as the cwd argument "
+                           "instead of `cd X &&`.")
+                    _last_obs["text"] = msg
+                    return msg
+                argv = shlex.split(_cmdline)
                 if not argv:
                     msg = "Error: empty command."
                     _last_obs["text"] = msg
@@ -12554,7 +12769,7 @@ def run_agent_chat(cfg):
                 # AN `&&` CHAIN IS ONE STEP (task #274): each part runs through THIS function —
                 # allowlist, risk gate, approval, server detection, all of it — in order, and
                 # the first failure stops the rest. `cd DIR` sets the folder for what follows.
-                _chain = _and_chain_parts(command)
+                _chain = None if _here else _and_chain_parts(command)
                 if _chain:
                     _chain_start = _t.time()
                     _cur_wd = wd
@@ -12604,7 +12819,7 @@ def run_agent_chat(cfg):
                 # executes the `cd` PROGRAM — which exits 0 with no output — and the step is
                 # reported as a success that did nothing. Refusing costs one step; the observed
                 # alternative cost two hour-long turns and a false report to the user.
-                _shell_err = _shell_syntax_error(command, argv)
+                _shell_err = _shell_syntax_error(_cmdline, argv)
                 if _shell_err:
                     _log(f"run_command refused shell syntax: {_c(_clip(command, 120))}")
                     _last_obs["text"] = _shell_err
@@ -12622,7 +12837,10 @@ def run_agent_chat(cfg):
                 # Risk gate: privilege escalation / denylisted / destructive. When the
                 # client can approve interactively (terminal), PAUSE and ask the user;
                 # otherwise (web) keep hard-blocking the privilege/denylist cases.
-                _hard_reason, _ask_reason = _ws_command_risk(command, argv, _WS_RUN_DENYLIST, wd)
+                # Judged on the COMMAND LINE, not on a heredoc's text: the text is data for the
+                # program (a script that prints "sudo …" is not a sudo), and what a script does
+                # is no more visible here than it is for `python3 -c`.
+                _hard_reason, _ask_reason = _ws_command_risk(_cmdline, argv, _WS_RUN_DENYLIST, wd)
                 # Shift+Tab may have switched it since the turn started (#47).
                 global _accept_edits
                 if _interactive_approval and _job_id:
@@ -12638,7 +12856,7 @@ def run_agent_chat(cfg):
                 # states it. "this needs root and sudo needs a password we cannot type" is a
                 # different question from "sudo is passwordless here", and the user can only
                 # answer the one they are shown.
-                _priv = _ws_priv_reason(command)
+                _priv = _ws_priv_reason(_cmdline)
                 _facts = _ws_access_facts() if _priv else None
                 if _priv and _facts and not _facts["can_elevate"]:
                     # REFUSE BEFORE SPAWNING (requirement 5). With no tty, `sudo x` dies with
@@ -12689,7 +12907,8 @@ def run_agent_chat(cfg):
                             _emit({"type": "step", "text": f"Server already up → {_urls} (reused)"})
                             _last_obs["text"] = msg
                             return msg
-                _emit({"type": "step", "text": f"Running → {command[:70]}"})
+                _emit({"type": "step", "text": f"Running → {_command_label(command, 70)}"})
+                _emit({"type": "step", "text": _running_command_line(command)})  # whole (#331)
                 t = max(5, min(int(timeout_seconds or 180), 600))
                 # Own process group (start_new_session) + output to a log file. This is
                 # what lets a server OUTLIVE this job process, and what lets a timeout
@@ -12705,13 +12924,30 @@ def run_agent_chat(cfg):
                 if _attrib_note:
                     _log(f"run_command: {_attrib_note}")
                 # Scrubbed env: repo scripts must never see worker secrets.
-                proc = subprocess.Popen(
-                    argv, cwd=wd, env=_ws_scrubbed_env(),
-                    stdout=log_fh, stderr=subprocess.STDOUT,
-                    **_ws_spawn_kwargs(),
-                )
+                # The heredoc's text as stdin, from an unnamed temp file: no pipe to keep fed (a
+                # pipe needs a writer thread or it deadlocks past 64 KB), and nothing left on
+                # disk — the file goes when the last handle on it closes.
+                _stdin_fh = None
+                if _stdin_text is not None:
+                    import tempfile
+                    _stdin_fh = tempfile.TemporaryFile()
+                    _stdin_fh.write(_stdin_text.encode("utf-8"))
+                    _stdin_fh.seek(0)
+                try:
+                    # stdin is passed ONLY for a heredoc — every other command keeps exactly
+                    # the stdin it has always had.
+                    proc = subprocess.Popen(
+                        argv, cwd=wd, env=_ws_scrubbed_env(),
+                        stdout=log_fh, stderr=subprocess.STDOUT,
+                        **({"stdin": _stdin_fh} if _stdin_fh is not None else {}),
+                        **_ws_spawn_kwargs(),
+                    )
+                finally:
+                    if _stdin_fh is not None:
+                        _stdin_fh.close()  # the child holds its own copy
                 _INFLIGHT_CMDS.add(proc.pid)  # killed with the agent until it is classified
                 start = _t.time()
+                _out_seen: dict = {}  # how much of its output the terminal has been told about
 
                 def _read_log() -> str:
                     log_fh.flush()
@@ -12744,7 +12980,7 @@ def run_agent_chat(cfg):
                             if rc == 0 else f"FAILED (exit {rc})"
                         )
                         _emit({"type": "step", "text":
-                               f"Command {'finished' if rc == 0 else 'failed'} → {command[:50]}"})
+                               f"Command {'finished' if rc == 0 else 'failed'} → {_command_label(command, 50)}"})
                         # SAY "no output" RATHER THAN SHOW NOTHING (task #204). An empty result
                         # was indistinguishable from a command that never ran — which is exactly
                         # what `cd X && Y` used to do — and the model read the silence as a
@@ -12818,6 +13054,8 @@ def run_agent_chat(cfg):
                                f"{t}s — killed. Output so far:\n{tail}")
                         _last_obs["text"] = msg
                         return msg
+                    log_fh.flush()
+                    _emit_command_progress(log_path, _out_seen)
                     _t.sleep(2)
             except Exception as ex:  # noqa: BLE001
                 msg = f"Error: run_command failed: {type(ex).__name__}: {ex}"