@tiens.nguyen/gu-cli 1.0.832 → 1.0.834

package/gu-repl.mjs

@@ -139,7 +139,8 @@ import {
 import { completedFlush, splitLiveStream } from "./completed-flush.mjs";
 import { isScaffoldStep, isGroupableStep, stepGroupRows, relativizePaths } from "./step-group.mjs";
 import { clearLogsNoticeText } from "./clear-logs.mjs";
-import { commandLogFile, killShell, logTail, readShells, renderShellDetails, shellPickerLabel, shellState, shellsNotice } from "./shell-details.mjs";
+import { browserReport, bridgeStatus, extensionDir, helpInstall, installExtensionFiles } from "./browser-command.mjs";
+import { commandLogFile, killShell, logTail, markAnnounced, readShells, recapQuestion, shellsToRecap, statusText as shellStatusText, renderShellDetails, shellPickerLabel, shellState, shellsNotice } from "./shell-details.mjs";
 import { discoverSkills, matchSkillCommand } from "./skills.mjs";
 import { appendNew, archivePathFor, archivedWorkspaceKey, markReset, seedArchive } from "./conversation-archive.mjs";
 import { addAllowRule, listServers as listMcpServers } from "./mcp-config.mjs";
@@ -953,6 +954,7 @@ const COMMANDS = [
   { name: "/open-presets", usage: "/open-presets [name]", desc: "your whole setup now, keys included, read-only in your editor — to copy to another computer; with a name, edit that preset" },
   { name: "/use-presets", usage: "/use-presets [name]", desc: "pick a saved preset and apply it — shows what changes, backs up your setup, asks first" },
   { name: "/doctor", desc: "check every configured model ANSWERS — especially the embedder, which a turn only reaches after several steps" },
+  { name: "/browser", desc: "the gu Chrome extension the agent uses to open and click web pages (UI tests) — is it connected, and how to install it" },
   { name: "/shells", desc: "long commands the agent left running in the background — status, runtime, command, output; stop one" },
   { name: "/tokens", desc: "break down the last prompt sent to the coding model — how much of it was RAG" },
   { name: "/clear-logs", desc: "show what gu's logs, RAG indexes and old API versions take on disk, then ask what to clear — the logs first" },
@@ -9503,6 +9505,33 @@ function refreshShells() {
   shellEntries = readShells().map((rec) => ({ rec, st: shellState(rec) }));
   const after = shellsNotice(shellEntries, { viewed: shellsViewed, since: SHELLS_SINCE })?.text ?? "";
   if (after !== before) paintBar();
+  wakeForShells();
+}
+
+/**
+ * WHEN A BACKGROUND SHELL FINISHES, THE AGENT HEARS ABOUT IT (#362) — Claude Code tells its model
+ * "background command completed" and it recaps or carries on. A shell that ends DURING a turn is
+ * told on the next step (the agent's step_callback); one that ends after the turn starts a short
+ * recap turn here — only at the empty main prompt (never into a question you are answering or a
+ * line you are typing), only for this folder's shells, and never for one you stopped yourself.
+ * GU_SHELL_RECAP=0 turns it off.
+ */
+let mainPromptWaiting = false;
+const SHELL_DONE_PREFIX = "\u0000shell-done:";
+function wakeForShells() {
+  if (process.env.GU_SHELL_RECAP === "0") return;
+  if (!mainPromptWaiting || following || listPickerActive || approvalActive || (rl?.line ?? "") !== "") return;
+  const due = shellsToRecap(shellEntries, { cwd: process.cwd(), since: SHELLS_SINCE });
+  if (!due.length) return;
+  const ids = due.map((r) => String(r.id));
+  try {
+    if (!markAnnounced(ids)) return; // another terminal got there first
+  } catch {
+    return;
+  }
+  for (const id of ids) shellsViewed.add(id);
+  mainPromptWaiting = false;
+  submitSyntheticLine(SHELL_DONE_PREFIX + ids.join(","));
 }
 
 /** The Shell details block for one shell, printed into the transcript. */
@@ -9534,6 +9563,23 @@ function printShellDetails(rec) {
  * details, then refresh / stop it / close. During a turn it only prints the details — a picker
  * would fight the turn for the keyboard, as showOtherAgents explains.
  */
+// /browser (#363): the extension's connection, else how to install it.
+async function showBrowser() {
+  let folder = extensionDir();
+  try {
+    folder = installExtensionFiles();
+  } catch (e) {
+    console.log(GUTTER + yellow(`could not copy the extension to ${folder}: ${e.message}`));
+  }
+  const st = await bridgeStatus().catch(() => ({ running: false, connected: false }));
+  const lines = browserReport(st, folder);
+  console.log("");
+  for (const [i, l] of lines.entries()) console.log(GUTTER + (i === 0 ? (st.connected ? accent(l) : l) : dim(l)));
+  if (!st.connected && st.running && helpInstall(folder)) console.log(GUTTER + dim("(Opened chrome://extensions and copied the folder to your clipboard.)"));
+  console.log("");
+  paintBar();
+}
+
 async function showShells() {
   if (listPickerActive || approvalActive) return;
   const width = Math.max(20, process.stdout.columns || 80);
@@ -14007,6 +14053,7 @@ async function main() {
     // the turn ends; a queued line carries no Enter keypress, so its re-prompt overwrites
     // in place (\r + refresh) rather than stacking ">> >> >> " on one row.
     let lineRaw;
+    mainPromptWaiting = true; // a finished background shell may start a recap turn now (#362)
     for (;;) {
       lineRaw = await nextLine();
       const echoed = enterEchoed;
@@ -14035,8 +14082,21 @@ async function main() {
       }
     }
     enterEchoed = false;
+    mainPromptWaiting = false;
     footerActive = false; // a line arrived from the queue (no Enter keypress) → disarm
     if (lineRaw === null) break; // stdin closed
+    // A RECAP TURN FOR A FINISHED BACKGROUND SHELL (#362). Nobody typed it, so it is not echoed as
+    // a question under ">>" — one dim line says what started it, then the turn runs as usual.
+    let shellRecap = false;
+    if (lineRaw.startsWith(SHELL_DONE_PREFIX)) {
+      const ids = lineRaw.slice(SHELL_DONE_PREFIX.length).split(",");
+      const entries = readShells().map((rec) => ({ rec, st: shellState(rec) })).filter((e) => ids.includes(String(e.rec.id)));
+      if (!entries.length) continue;
+      shellRecap = true;
+      alignOutputToLineStart();
+      console.log("\n" + GUTTER + dim(`⏺ background ${entries.length === 1 ? "shell" : "shells"} finished — ${entries.map((e) => `${e.rec.id} ${shellStatusText(e.st)}`).join(", ")} · gu asks the agent to recap`));
+      lineRaw = recapQuestion(entries);
+    }
     const line = lineRaw.trim();
     // #302: what was typed for THIS line (before expansion), and whether it was sent from a
     // stopped turn's restored question.
@@ -14051,7 +14111,7 @@ async function main() {
     // carries a NUL and a raw jobId, and echoing it as a question would put both in the
     // scrollback under a ">>" that claims the user wrote them. Its branch prints its own
     // "⇄ following: …" line, which is the readable version of the same event.
-    if (barPinned() && line && !line.startsWith(ATTACH_PREFIX)) {
+    if (barPinned() && line && !line.startsWith(ATTACH_PREFIX) && !shellRecap) {
       inputVisible = false; // the zone goes back to an empty prompt for the turn
       // START THIS LINE AT COLUMN 0, don't assume it is already there.
       //
@@ -14147,6 +14207,10 @@ async function main() {
       await showShells();
       continue;
     }
+    if (line === "/browser") {
+      await showBrowser();
+      continue;
+    }
     if (line === "/model") {
       await chooseModel();
       continue;

package/gu_browser.py

@@ -0,0 +1,263 @@
+"""The agent's browser tool, Python side (task #363, GitHub #144).
+
+The agent runs in a new process every turn, so it never holds the Chrome extension itself: it talks
+to browser-bridge.mjs (a small local Node process that does) over HTTP, starting it when needed.
+This module finds or starts the bridge, sends one command, and turns the answer into the text the
+model reads. No model or smolagents import here — it is tested on its own.
+"""
+
+import base64
+import json
+import os
+import shutil
+import subprocess
+import sys
+import time
+import urllib.error
+import urllib.request
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+EXT_SRC = os.path.join(HERE, "browser-extension")
+EXT_FILES = ("manifest.json", "background.js", "popup.html", "popup.js",
+             "icons/icon16.png", "icons/icon48.png", "icons/icon128.png")
+BRIDGE = os.path.join(HERE, "browser-bridge.mjs")
+
+
+def data_dir():
+    return os.path.join(os.path.expanduser("~"), ".gonext")
+
+
+def state_path():
+    return os.environ.get("GU_BROWSER_STATE") or os.path.join(data_dir(), "browser-bridge.json")
+
+
+def seen_path():
+    # Written the first time an extension answers: from then on a missing connection means "Chrome
+    # is asleep or closed", worth a longer wait, not "never installed".
+    return os.path.join(os.path.dirname(state_path()), "browser-extension-seen")
+
+
+def extension_dir():
+    """The folder the user loads into Chrome. A fixed place under ~/.gonext, not the npm package:
+    an update moves the package, and Chrome would lose an unpacked extension loaded from there."""
+    return os.environ.get("GU_BROWSER_EXT_DIR") or os.path.join(data_dir(), "browser-extension")
+
+
+def install_extension_files():
+    """Copy the extension to extension_dir() when it is missing or older. Returns the folder."""
+    dst = extension_dir()
+    try:
+        with open(os.path.join(EXT_SRC, "manifest.json"), encoding="utf-8") as f:
+            want = f.read()
+        try:
+            with open(os.path.join(dst, "manifest.json"), encoding="utf-8") as f:
+                have = f.read()
+        except OSError:
+            have = ""
+        def same(n):
+            try:
+                with open(os.path.join(dst, n), "rb") as a, open(os.path.join(EXT_SRC, n), "rb") as b:
+                    return a.read() == b.read()
+            except OSError:
+                return False
+        if have != want or not all(same(n) for n in EXT_FILES):
+            os.makedirs(os.path.join(dst, "icons"), exist_ok=True)
+            for n in EXT_FILES:
+                shutil.copyfile(os.path.join(EXT_SRC, n), os.path.join(dst, n))
+    except OSError:
+        pass
+    return dst
+
+
+def install_text(reason="not connected"):
+    """What the model is told when the extension is not there: stop and ask the user."""
+    folder = install_extension_files()
+    return (
+        f"BROWSER NOT AVAILABLE ({reason}): the gu Chrome extension is not installed, or Chrome is "
+        "not running. Do NOT try to work around it with curl or a headless script for a UI check — "
+        "STOP and ask the user to install it, then continue when they say it is done. Tell them:\n"
+        "  1. Open Chrome and go to chrome://extensions\n"
+        "  2. Turn on Developer mode (top right)\n"
+        f"  3. Click \"Load unpacked\" and choose the folder {folder}\n"
+        "  4. (Or type /browser in the gu terminal for these steps and the connection status.)\n"
+        "gu only uses its own tabs (a tab group named \"gu\"); the user's other tabs are never touched.")
+
+
+def _node():
+    return os.environ.get("GU_NODE") or shutil.which("node") or "node"
+
+
+def _read_state():
+    try:
+        with open(state_path(), encoding="utf-8") as f:
+            st = json.load(f)
+        return st if st.get("port") and st.get("token") else None
+    except (OSError, ValueError):
+        return None
+
+
+def _http(st, method, path, body=None, timeout=10):
+    req = urllib.request.Request(
+        f"http://127.0.0.1:{st['port']}{path}", method=method,
+        data=json.dumps(body).encode() if body is not None else None,
+        headers={"authorization": f"Bearer {st['token']}", "content-type": "application/json"})
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as r:
+            return json.loads(r.read() or b"{}")
+    except urllib.error.HTTPError as e:
+        try:
+            return json.loads(e.read() or b"{}") | {"_status": e.code}
+        except ValueError:
+            return {"_status": e.code, "error": str(e)}
+
+
+def _start_bridge(log=None):
+    """Start browser-bridge.mjs detached (it outlives this turn) and wait for its state file."""
+    os.makedirs(data_dir(), exist_ok=True)
+    logf = open(os.path.join(data_dir(), "browser-bridge.log"), "ab")
+    kw = {"start_new_session": True} if os.name != "nt" else {
+        "creationflags": getattr(subprocess, "DETACHED_PROCESS", 0) | getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)}
+    subprocess.Popen([_node(), BRIDGE], stdin=subprocess.DEVNULL, stdout=logf, stderr=logf,
+                     env=dict(os.environ), **kw)
+    logf.close()
+    if log:
+        log("browser: started the bridge")
+    end = time.time() + 6
+    while time.time() < end:
+        st = _read_state()
+        if st:
+            try:
+                _http(st, "GET", "/status", timeout=2)
+                return st
+            except OSError:
+                pass
+        time.sleep(0.15)
+    return None
+
+
+def connect(wait=True, log=None, on_wait=None):
+    """→ (state, None) when the extension is connected, else (None, reason)."""
+    st = _read_state()
+    started = False
+    try:
+        if not st:
+            raise OSError("no bridge")
+        _http(st, "GET", "/status", timeout=3)
+    except OSError:
+        st = _start_bridge(log)
+        started = True
+        if not st:
+            return None, "the gu browser bridge did not start (is node installed?)"
+    # A Chrome extension's worker sleeps; its alarm wakes it to reconnect within 30 s. Wait that
+    # long only when an extension has connected before — otherwise it is most likely not installed.
+    seen = os.path.exists(seen_path())
+    wait_ms = (35000 if (seen and started) else 8000 if seen else 4000) if wait else 0
+    if wait_ms >= 8000 and on_wait:
+        on_wait()
+    try:
+        s = _http(st, "GET", f"/status?wait_ms={wait_ms}", timeout=wait_ms / 1000 + 5)
+    except OSError as e:
+        return None, f"the gu browser bridge is not answering ({e})"
+    if not s.get("connected"):
+        return None, "extension not connected"
+    if not seen:
+        try:
+            with open(seen_path(), "w") as f:
+                f.write(str(s.get("version") or ""))
+        except OSError:
+            pass
+    return st, None
+
+
+def command(cmd, log=None, on_wait=None):
+    """Send one command. → (ok, result_or_error, not_connected)."""
+    st, why = connect(log=log, on_wait=on_wait)
+    if not st:
+        return False, why, True
+    timeout_s = 75 if cmd.get("action") in ("open", "wait", "js") else 40
+    try:
+        r = _http(st, "POST", "/cmd", body=dict(cmd, timeout_ms=(timeout_s - 5) * 1000), timeout=timeout_s)
+    except OSError as e:
+        return False, f"the browser bridge did not answer ({e})", False
+    if r.get("_status") == 503:
+        return False, "extension not connected", True
+    if not r.get("ok"):
+        return False, str(r.get("error") or "failed"), False
+    return True, r.get("result") or {}, False
+
+
+# ── what the model reads ────────────────────────────────────────────────────────────────────
+def _head(res):
+    out = f"Page: {res.get('title') or '(no title)'} — {res.get('url') or ''}"
+    for d in res.get("dialogs") or []:
+        out += f"\nA dialog opened and gu accepted it: {d}"
+    errs = res.get("errors") or []
+    if errs:
+        out += "\nNEW console errors:\n" + "\n".join("  " + e for e in errs)
+    return out
+
+
+def format_read(res):
+    els = res.get("elements") or []
+    lines = [_head(res), f"Viewport {res.get('viewport')}, scrolled {res.get('scroll')} px."]
+    if els:
+        lines.append("Interactive elements in view — click/type by [number] (good until the next read):")
+        lines += ["  " + e for e in els]
+    else:
+        lines.append("No interactive elements in view.")
+    if res.get("offscreen"):
+        lines.append(f"(+{res['offscreen']} more off screen — scroll to reach them)")
+    text = (res.get("text") or "").strip()
+    lines.append("Page text:\n" + (text or "(empty)"))
+    return "\n".join(lines)
+
+
+def format_result(action, res):
+    a = action
+    if a == "tabs":
+        tabs = res.get("tabs") or []
+        if not tabs:
+            return "gu has no tab open."
+        return "gu's tabs:\n" + "\n".join(
+            f"  {t['id']}{' (current)' if t.get('current') else ''}: {t.get('title') or ''} — {t.get('url')}" for t in tabs)
+    if a == "read":
+        return format_read(res)
+    if a == "close":
+        return f"Closed tab {res.get('closed')}."
+    out = _head(res)
+    if a in ("click", "double_click", "right_click", "hover"):
+        out = f"{a.replace('_', ' ').capitalize()} at {res.get('at')} on {res.get('hit')}.\n" + out
+    if a == "scroll":
+        out += f"\nScrolled to {res.get('scroll')} px."
+    if a == "js":
+        out += f"\nResult: {res.get('value')}"
+    if a == "console":
+        msgs = res.get("console") or []
+        out += ("\nConsole since the last read:\n" + "\n".join(f"  [{m.get('level')}] {m.get('text')}" for m in msgs)
+                if msgs else "\nConsole: nothing new.")
+    if a in ("open", "click", "double_click", "type", "key", "back", "reload"):
+        out += "\n(read or screenshot to see the page now)"
+    return out
+
+
+def decode_screenshot(res, max_edge=1568):
+    """The screenshot as a PIL image at the page's CSS size, so its pixels ARE click coordinates.
+    → (image, note). Chrome captures in device pixels (2× on a Retina screen)."""
+    from io import BytesIO
+    from PIL import Image
+    im = Image.open(BytesIO(base64.b64decode(res.get("image") or ""))).convert("RGB")
+    w, h = int(res.get("width") or im.width), int(res.get("height") or im.height)
+    if (w, h) != im.size and w > 0 and h > 0:
+        im = im.resize((w, h), Image.LANCZOS)
+    note = f"Screenshot {im.width}×{im.height} — its pixels are the page's x,y for click."
+    if max(im.size) > max_edge:
+        k = max_edge / max(im.size)
+        im = im.resize((max(1, int(im.width * k)), max(1, int(im.height * k))), Image.LANCZOS)
+        note = (f"Screenshot shrunk to {im.width}×{im.height}; the page is {w}×{h} — multiply a point "
+                f"in it by {1 / k:.3f} for click x,y.")
+    return im, note
+
+
+if __name__ == "__main__":  # tiny manual check: python3 gu_browser.py open https://example.com
+    ok, res, _ = command({"action": sys.argv[1], "url": sys.argv[2] if len(sys.argv) > 2 else ""})
+    print(format_result(sys.argv[1], res) if ok else res)