@cohortapp/agent-sdk 2.17.0 → 2.18.5

package/lib/engine/tools/bash-background.mjs

@@ -0,0 +1,533 @@
+/**
+ * lib/engine/tools/bash-background.mjs — the session's shell: a persistent
+ * working directory for Bash, background commands, BashOutput and KillShell.
+ *
+ * Persistent working directory. Each foreground Bash call runs in the
+ * directory the previous call ended in, so `cd build` in one call holds for
+ * the next. The command is followed by a capture line that prints a per-session
+ * marker and `pwd -P`; the marker line is removed from the output and the path
+ * becomes the next call's directory. A command that exits the shell early, is
+ * killed, or overflows the output limit leaves the directory unchanged. The
+ * permission check and hooks see the command as the model wrote it. A
+ * directory that no longer exists falls back to the project directory. Every
+ * subagent starts in the project directory with its own tracked directory.
+ *
+ * Background commands. `Bash {run_in_background:true}` starts `bash -c` in its
+ * own process group (same scrubbed environment as a foreground call:
+ * `scrubToolEnv`), with stdin closed, and returns a shell id at once.
+ *   BashOutput {bash_id, filter?}  the output produced since the last read,
+ *                                  optionally only the lines matching `filter`
+ *                                  (a regular expression; other lines are consumed)
+ *   KillShell {shell_id}           SIGTERM to the whole group, SIGKILL after 2 s
+ * Unread output is kept in memory up to 30,000 characters; beyond that it, and
+ * everything after it, is written to `<spill dir>/<shell id>.log`. Reads then
+ * return at most 30,000 bytes at a time (cut at a line end) and name the file.
+ * The spill file stops growing at 50 MB (`SPILL_FILE_CAP`); later output is
+ * dropped and reads say so. Every background process group is killed when the
+ * engine exits (`killAll`, and a process `exit` listener as the backstop).
+ *
+ * A kill never waits on a process that left the group. After SIGKILL and a
+ * short settle, the shell's pipes are destroyed and the kill returns, even if a
+ * `setsid` grandchild still holds them open.
+ *
+ * CF-48 (W5-B): that grandchild no longer survives. Every shell (background
+ * Bash, Monitor, a subagent's or workflow child's shell — they share this
+ * manager) is tracked by process tree, not only by group (process-tree.mjs):
+ * its environment carries a random COHORT_ENGINE_PROCESS_TAG, its descendants
+ * are recorded with their start identity shortly after it starts, while it
+ * runs and when it is stopped, and KillShell, a monitor's timeout, killAll and
+ * the exit backstop SIGTERM → SIGKILL every recorded process whose identity
+ * still matches — within the same grace period. killAll also reaps what shells
+ * that already finished left behind.
+ *
+ * `wrapWithCwdCapture`, `splitCwdCapture` and `filterLines` are pure.
+ *
+ * @module lib/engine/tools/bash-background
+ */
+
+import { spawn } from "node:child_process";
+import { randomBytes } from "node:crypto";
+import { appendFileSync, closeSync, existsSync, mkdirSync, openSync, readSync, statSync } from "node:fs";
+import path from "node:path";
+import { bashTool, scrubToolEnv } from "./bash.mjs";
+import { createProcessTracker, newProcessTag, PROCESS_TAG_ENV } from "../process-tree.mjs";
+
+export const BACKGROUND_OUTPUT_CAP = 30_000;
+export const SPILL_FILE_CAP = 50 * 1024 * 1024;
+const KILL_GRACE_MS = 2_000;
+/** After SIGKILL, how long the pipes may take to close before they are destroyed. */
+const PIPE_SETTLE_MS = 250;
+
+/** @param {string} command @param {string} marker */
+export function wrapWithCwdCapture(command, marker) {
+  return `${command}\n__cohort_status=$?\nprintf '\\n%s%s\\n' '${marker}' "$(pwd -P 2>/dev/null || pwd)"\nexit $__cohort_status`;
+}
+
+/**
+ * Remove the capture line from a command's output.
+ * @param {string} text @param {string} marker
+ * @returns {{text:string, cwd:string|null}}
+ */
+export function splitCwdCapture(text, marker) {
+  const at = text.lastIndexOf(`\n${marker}`);
+  if (at === -1) return { text, cwd: null };
+  const end = text.indexOf("\n", at + 1 + marker.length);
+  const cwd = text.slice(at + 1 + marker.length, end === -1 ? text.length : end);
+  const rest = end === -1 ? "" : text.slice(end + 1);
+  return { text: text.slice(0, at) + (rest ? `\n${rest}` : ""), cwd: cwd.startsWith("/") ? cwd : null };
+}
+
+/** @param {string} text @param {RegExp} re */
+export function filterLines(text, re) {
+  return text
+    .split("\n")
+    .filter((l) => re.test(l))
+    .join("\n");
+}
+
+/**
+ * @typedef {Object} ShellRecord
+ * @property {string} id
+ * @property {string} command
+ * @property {string|null} description
+ * @property {import('node:child_process').ChildProcess} child
+ * @property {'running'|'completed'|'failed'|'killed'} status
+ * @property {number|null} exitCode
+ * @property {string} unread                in-memory output not yet returned
+ * @property {number} chars                 characters produced so far
+ * @property {string|null} spillPath
+ * @property {number} readBytes             spill mode: bytes of the log already returned
+ * @property {number} spilledBytes          spill mode: bytes written to the log
+ * @property {number} droppedBytes          output not kept once the log reached its cap
+ * @property {Promise<void>} exited          resolves when the pipes close (or are destroyed)
+ * @property {boolean} [processExited]      bash itself has exited (its pipes may still be open)
+ * @property {boolean} [pipesClosed]
+ * @property {() => void} [release]         end `exited` without the pipes closing
+ */
+
+/**
+ * What a finished shell reports to `onExit` (W4-A1, CF-50). Pure.
+ * @param {ShellRecord} s
+ */
+export function shellSummary(s) {
+  return { id: s.id, command: s.command, description: s.description, kind: s.kind ?? "bash", status: s.status, exitCode: s.exitCode, spillPath: s.spillPath };
+}
+
+/**
+ * @param {{spillDir:string, env:NodeJS.ProcessEnv, cap?:number, spillFileCap?:number, killGraceMs?:number,
+ *          onExit?:(summary:ReturnType<typeof shellSummary>)=>void}} o
+ *   `onExit` (W4-A1, CF-50): called once when a shell ends on its own (completed or failed) —
+ *   not when it was killed (KillShell, TaskStop, a monitor's timeout, the end of the run).
+ */
+export function createShellManager({ spillDir, env, cap = BACKGROUND_OUTPUT_CAP, spillFileCap = SPILL_FILE_CAP, killGraceMs = KILL_GRACE_MS, onExit: onShellExit, passthrough = [], processTracker }) {
+  /** @type {Map<string, ShellRecord>} */
+  const shells = new Map();
+  let seq = 0;
+  // CF-48: the processes each shell started, including those that called setsid (process-tree.mjs).
+  const tracker = processTracker ?? createProcessTracker();
+
+  const killGroup = (/** @type {ShellRecord} */ s, /** @type {NodeJS.Signals} */ sig) => {
+    try {
+      process.kill(-(/** @type number */ (s.child.pid)), sig);
+    } catch {
+      /* the group already exited */
+    }
+  };
+  const groupAlive = (/** @type {ShellRecord} */ s) => {
+    try {
+      process.kill(-(/** @type number */ (s.child.pid)), 0);
+      return true;
+    } catch (e) {
+      return /** @type {NodeJS.ErrnoException} */ (e).code === "EPERM";
+    }
+  };
+  const onExit = () => {
+    for (const s of shells.values()) if (s.status === "running") killGroup(s, "SIGKILL");
+    tracker.killSync([...shells.keys()]);
+  };
+  process.on("exit", onExit);
+
+  /** Write to the spill file up to its cap; what does not fit is counted, not kept. */
+  const spill = (/** @type {ShellRecord} */ s, /** @type string */ text) => {
+    const bytes = Buffer.byteLength(text);
+    const room = spillFileCap - s.spilledBytes;
+    if (bytes <= room) {
+      appendFileSync(/** @type string */ (s.spillPath), text);
+      s.spilledBytes += bytes;
+      return;
+    }
+    if (room > 0) {
+      // Cut at a line end inside the room, so the file never ends mid-character.
+      const buf = Buffer.from(text).subarray(0, room);
+      const nl = buf.lastIndexOf(0x0a);
+      const keep = nl === -1 ? 0 : nl + 1;
+      if (keep > 0) appendFileSync(/** @type string */ (s.spillPath), buf.subarray(0, keep));
+      s.spilledBytes += keep;
+      s.droppedBytes += bytes - keep;
+      if (keep < room) s.spilledBytes = spillFileCap; // the file is closed to further output
+      return;
+    }
+    s.droppedBytes += bytes;
+  };
+
+  const append = (/** @type {ShellRecord} */ s, /** @type string */ chunk) => {
+    s.chars += chunk.length;
+    if (s.spillPath) {
+      spill(s, chunk);
+      return;
+    }
+    s.unread += chunk;
+    if (s.unread.length > cap) {
+      mkdirSync(spillDir, { recursive: true });
+      s.spillPath = path.join(spillDir, `${s.id}.log`);
+      const text = s.unread;
+      s.unread = "";
+      spill(s, text);
+    }
+  };
+
+  /**
+   * CF-48: record every process these shells started (tree, samples, tag) and SIGTERM
+   * the ones outside the group; call before the group signal, so parents are still there.
+   * @param {string[]} ids
+   */
+  const terminateTracked = (ids) => {
+    try {
+      tracker.capture(ids, { tags: true });
+      tracker.signal(ids, "SIGTERM");
+    } catch {
+      /* the group signal still goes out */
+    }
+  };
+
+  /**
+   * Wait for killed shells: SIGKILL at the grace period, then at most a short
+   * settle for its pipes; a pipe still held open (by a process that left the
+   * group) is destroyed so the wait always ends. The tracked processes of
+   * `ids` (CF-48) get the rest of the grace period, then SIGKILL if their
+   * identity still matches.
+   * @param {ShellRecord[]} list  running shells
+   * @param {string[]} [ids]      every shell whose tracked processes are reaped
+   */
+  const settle = async (list, ids = list.map((s) => s.id)) => {
+    const started = Date.now();
+    const reapTracked = async () => {
+      if (ids.length === 0) return;
+      try {
+        await tracker.reap(ids, killGraceMs - (Date.now() - started));
+      } catch {
+        /* bounded best effort */
+      }
+    };
+    if (list.length === 0) {
+      await reapTracked();
+      return;
+    }
+    /** @type {NodeJS.Timeout[]} */
+    const timers = [];
+    const closed = Promise.all(list.map((s) => s.exited));
+    const graceOver = new Promise((resolve) => {
+      timers.push(
+        setTimeout(() => {
+          for (const s of list) killGroup(s, "SIGKILL");
+          timers.push(setTimeout(resolve, PIPE_SETTLE_MS));
+        }, killGraceMs),
+      );
+    });
+    // Bash has exited and nothing is left in its group: only an escaped process holds the pipes.
+    const groupGone = new Promise((resolve) => {
+      const poll = setInterval(() => {
+        if (list.every((s) => s.pipesClosed || (s.processExited && !groupAlive(s)))) resolve(undefined);
+      }, 50);
+      timers.push(poll);
+    });
+    await Promise.race([closed, graceOver, groupGone]);
+    for (const t of timers) clearTimeout(t);
+    await reapTracked();
+    for (const s of list) {
+      if (s.pipesClosed) continue;
+      s.child.stdout?.destroy();
+      s.child.stderr?.destroy();
+      s.child.unref();
+      s.release?.();
+    }
+  };
+
+  return {
+    /**
+     * @param {{command:string, cwd:string, description?:string|null, kind?:string, onLine?:(line:string)=>void}} p
+     *   `onLine` (W4-A1, Monitor): each complete stdout line as it arrives (the output is still kept for BashOutput).
+     * @returns {{ok:true, id:string}|{ok:false, error:string}}
+     */
+    start({ command, cwd, description = null, kind = "bash", onLine }) {
+      const id = `${kind === "bash" ? "bash" : kind}_${++seq}`;
+      const tag = newProcessTag();
+      let child;
+      try {
+        child = spawn("/bin/bash", ["-c", command], { cwd, env: { ...scrubToolEnv(env, { passthrough }), [PROCESS_TAG_ENV]: tag }, stdio: ["ignore", "pipe", "pipe"], detached: true });
+      } catch (e) {
+        return { ok: false, error: `Could not start bash: ${e instanceof Error ? e.message : e}` };
+      }
+      if (child.pid) tracker.track({ id, pid: child.pid, tag });
+      /** @type {ShellRecord} */
+      const s = { id, command, description, kind, child, status: "running", exitCode: null, unread: "", chars: 0, spillPath: null, readBytes: 0, spilledBytes: 0, droppedBytes: 0, exited: Promise.resolve() };
+      let lineBuf = "";
+      const emitLine = (/** @type string */ line) => {
+        try {
+          onLine?.(line);
+        } catch {
+          /* a line observer must not break the shell */
+        }
+      };
+      let reported = false;
+      const reportExit = () => {
+        if (reported) return; // 'error' can be followed by 'close'
+        reported = true;
+        if (lineBuf !== "") emitLine(lineBuf);
+        lineBuf = "";
+        if (s.status === "killed" || !onShellExit) return;
+        try {
+          onShellExit(shellSummary(s));
+        } catch {
+          /* a notification observer must not break the shell */
+        }
+      };
+      s.exited = new Promise((resolve) => {
+        s.release = () => resolve();
+        child.on("error", (e) => {
+          append(s, `bash error: ${e.message}\n`);
+          if (s.status === "running") s.status = "failed";
+          s.pipesClosed = true;
+          reportExit();
+          resolve();
+        });
+        // 'exit' records the code (the pipes may stay open); 'close' ends the wait.
+        child.on("exit", (code) => {
+          if (s.exitCode === null) s.exitCode = code;
+          s.processExited = true;
+        });
+        child.on("close", (code) => {
+          if (s.exitCode === null) s.exitCode = code;
+          if (s.status === "running") s.status = s.exitCode === 0 ? "completed" : "failed";
+          s.pipesClosed = true;
+          tracker.rootExited(id); // stop sampling; what was recorded is still reaped at killAll
+          reportExit();
+          resolve();
+        });
+      });
+      child.stdout?.setEncoding("utf8");
+      child.stderr?.setEncoding("utf8");
+      child.stdout?.on("data", (d) => {
+        append(s, d);
+        if (!onLine) return;
+        lineBuf += d;
+        let nl;
+        while ((nl = lineBuf.indexOf("\n")) !== -1) {
+          emitLine(lineBuf.slice(0, nl).replace(/\r$/, ""));
+          lineBuf = lineBuf.slice(nl + 1);
+        }
+      });
+      child.stderr?.on("data", (d) => append(s, d));
+      shells.set(id, s);
+      return { ok: true, id };
+    },
+
+    /** @param {string} id */
+    has: (id) => shells.has(id),
+
+    /**
+     * New output since the last read.
+     * @param {string} id @param {string|undefined} filter
+     * @returns {{ok:true, content:string, status:string}|{ok:false, error:string}}
+     */
+    read(id, filter) {
+      const s = shells.get(id);
+      if (!s) return { ok: false, error: `There is no background shell "${id}". Known: ${[...shells.keys()].join(", ") || "none"}.` };
+      let re = null;
+      if (filter) {
+        try {
+          re = new RegExp(filter);
+        } catch (e) {
+          return { ok: false, error: `filter is not a valid regular expression: ${e instanceof Error ? e.message : e}` };
+        }
+      }
+      let text;
+      let more = 0;
+      if (s.spillPath) {
+        const size = statSync(s.spillPath).size;
+        const want = Math.min(size - s.readBytes, cap);
+        const buf = Buffer.alloc(want);
+        const fd = openSync(s.spillPath, "r");
+        try {
+          readSync(fd, buf, 0, want, s.readBytes);
+        } finally {
+          closeSync(fd);
+        }
+        // Never split a UTF-8 character: stop at the last newline when cut short.
+        let take = buf.length;
+        if (s.readBytes + want < size) {
+          const nl = buf.lastIndexOf(0x0a);
+          if (nl > 0) take = nl + 1;
+        }
+        text = buf.subarray(0, take).toString("utf8");
+        s.readBytes += take;
+        more = size - s.readBytes;
+      } else {
+        text = s.unread;
+        s.unread = "";
+      }
+      if (re) text = filterLines(text, re);
+      const state = s.status === "running" ? "running" : s.status === "killed" ? "killed" : `exited with code ${s.exitCode}`;
+      const notes = [`[shell ${id}: ${state}]`];
+      if (s.spillPath) notes.push(`[the full log is in ${s.spillPath}${more > 0 ? `; ${more} more bytes are waiting — read again` : ""}]`);
+      if (s.droppedBytes > 0) notes.push(`[the log reached its ${spillFileCap} byte limit; ${s.droppedBytes} bytes of later output were not kept]`);
+      return { ok: true, status: s.status, content: `${text === "" ? "(no new output)" : text}\n\n${notes.join("\n")}` };
+    },
+
+    /**
+     * Wait for a shell to exit, at most `ms`.
+     * @param {string} id @param {number} ms @param {AbortSignal} [signal]
+     */
+    async wait(id, ms, signal) {
+      const s = shells.get(id);
+      if (!s || s.status !== "running") return;
+      await new Promise((resolve) => {
+        const t = setTimeout(done, ms);
+        const onAbort = () => done();
+        signal?.addEventListener("abort", onAbort, { once: true });
+        s.exited.then(done);
+        function done() {
+          clearTimeout(t);
+          signal?.removeEventListener("abort", onAbort);
+          resolve(undefined);
+        }
+      });
+    },
+
+    /**
+     * @param {string} id
+     * @returns {Promise<{ok:true, content:string}|{ok:false, error:string}>}
+     */
+    async kill(id) {
+      const s = shells.get(id);
+      if (!s) return { ok: false, error: `There is no background shell "${id}".` };
+      if (s.status !== "running") return { ok: true, content: `Shell ${id} had already ${s.status === "killed" ? "been killed" : `exited with code ${s.exitCode}`}.` };
+      s.status = "killed";
+      terminateTracked([id]);
+      killGroup(s, "SIGTERM");
+      await settle([s]);
+      return { ok: true, content: `Killed shell ${id} (${s.command.slice(0, 80)}).` };
+    },
+
+    /**
+     * Kill every running group, and every tracked process of every shell this manager
+     * started — finished ones too (CF-48) — and wait for them; removes the exit listener.
+     */
+    async killAll() {
+      const running = [...shells.values()].filter((s) => s.status === "running");
+      const ids = [...shells.keys()];
+      terminateTracked(ids);
+      for (const s of running) {
+        s.status = "killed";
+        killGroup(s, "SIGTERM");
+      }
+      await settle(running, ids);
+      tracker.dispose();
+      process.removeListener("exit", onExit);
+    },
+
+    /** CF-48: the tracker (tests and the session runtime read what a shell recorded). */
+    tracker,
+
+    list: () => [...shells.values()].map((s) => ({ id: s.id, command: s.command, status: s.status, exitCode: s.exitCode, kind: s.kind ?? "bash", description: s.description })),
+  };
+}
+
+/**
+ * The session's Bash: bash.mjs's tool with `run_in_background` and a
+ * persistent working directory.
+ * @param {{shells:ReturnType<typeof createShellManager>, projectCwd:string}} o
+ */
+export function createSessionBash({ shells, projectCwd }) {
+  const marker = `__COHORT_CWD_${randomBytes(6).toString("hex")}__`;
+  const state = { cwd: projectCwd };
+  const tool = {
+    ...bashTool,
+    description:
+      `${bashTool.description} The working directory carries over between calls (a \`cd\` holds for the next call). ` +
+      "Set `run_in_background` to start a long-running command and get a shell id back at once; read its output with BashOutput and stop it with KillShell.",
+    input_schema: {
+      ...bashTool.input_schema,
+      properties: { ...bashTool.input_schema.properties, run_in_background: { type: "boolean", description: "Start the command in the background and return its shell id." } },
+    },
+    /**
+     * @param {{command:string, timeout?:number, description?:string, run_in_background?:boolean}} input
+     * @param {import('./index.mjs').ToolContext} ctx
+     */
+    async run(input, ctx) {
+      let note = "";
+      if (!existsSync(state.cwd)) {
+        note = `\n[${state.cwd} no longer exists; the command ran in ${projectCwd}]`;
+        state.cwd = projectCwd;
+      }
+      if (input.run_in_background) {
+        const r = shells.start({ command: input.command, cwd: state.cwd, description: input.description ?? null });
+        if (!r.ok) return r;
+        return { ok: true, content: `Started in the background as shell ${r.id} (in ${state.cwd}). Read its output with BashOutput {"bash_id":"${r.id}"}; stop it with KillShell {"shell_id":"${r.id}"}.${note}` };
+      }
+      const { run_in_background: _bg, ...rest } = input;
+      const r = await bashTool.run({ ...rest, command: wrapWithCwdCapture(input.command, marker) }, { ...ctx, cwd: state.cwd });
+      const { text, cwd } = splitCwdCapture(r.ok ? r.content : r.error, marker);
+      if (cwd && cwd !== state.cwd && existsSync(cwd)) state.cwd = cwd;
+      if (r.ok) return { ok: true, content: (text === "" ? "(no output)" : text) + note };
+      return { ok: false, error: text + note };
+    },
+    /** A subagent's Bash: the same background shells, its own directory (a workflow child's worktree, W4-A2). */
+    forChild: (/** @type {{cwd?:string}|undefined} */ o) => createSessionBash({ shells, projectCwd: o?.cwd ?? projectCwd }),
+    get cwd() {
+      return state.cwd;
+    },
+  };
+  return tool;
+}
+
+/** @param {ReturnType<typeof createShellManager>} shells */
+export function createBashOutputTool(shells) {
+  return {
+    name: "BashOutput",
+    readOnly: true,
+    description:
+      "Read the output a background shell (started with Bash run_in_background) has produced since the last read, and whether it is still running. " +
+      "`filter` is an optional regular expression: only matching lines are returned, and the rest of the new output is consumed.",
+    input_schema: {
+      type: "object",
+      properties: {
+        bash_id: { type: "string", description: "The shell id Bash returned." },
+        filter: { type: "string", description: "A regular expression selecting the lines to return." },
+      },
+      required: ["bash_id"],
+    },
+    async run(/** @type {{bash_id:string, filter?:string}} */ input) {
+      const r = shells.read(input.bash_id, input.filter);
+      return r.ok ? { ok: true, content: r.content } : r;
+    },
+  };
+}
+
+/** @param {ReturnType<typeof createShellManager>} shells */
+export function createKillShellTool(shells) {
+  return {
+    name: "KillShell",
+    // Stops only a process this run started itself.
+    readOnly: true,
+    description: "Stop a background shell started with Bash run_in_background, including every process it started.",
+    input_schema: {
+      type: "object",
+      properties: { shell_id: { type: "string", description: "The shell id Bash returned." } },
+      required: ["shell_id"],
+    },
+    async run(/** @type {{shell_id:string}} */ input) {
+      return shells.kill(input.shell_id);
+    },
+  };
+}