@brutalsystems/muster 3.0.2 → 3.1.0

package/README.md

@@ -70,6 +70,10 @@ muster run claude --prompt 'Review this project' --cwd .
 muster run codex --kind task --prompt 'Summarise the failing tests' --read-only
 muster output <id>
 
+# Knowing it has finished: block on it, or have muster run a command then.
+muster wait <id> --timeout 30m
+muster run codex --kind task --prompt 'Review' --on-exit 'notify "$MUSTER_STATE"'
+
 # What is running, and stopping it.
 muster list --format human
 muster stop <id>

package/dist/cli-options.js

@@ -0,0 +1,60 @@
+/**
+ * Every flag any command takes, in one table so `asksForHelp` can tell which
+ * flags consume the next word: `--prompt --help` is a prompt, not a request
+ * for help.
+ */
+export const CLI_OPTIONS = {
+    prompt: { type: "string" },
+    cwd: { type: "string" },
+    kind: { type: "string" },
+    host: { type: "string" },
+    open: { type: "boolean" },
+    terminal: { type: "string" },
+    format: { type: "string" },
+    permissions: { type: "string" },
+    sandbox: { type: "string" },
+    level: { type: "string" },
+    "read-only": { type: "boolean" },
+    "write-here": { type: "boolean" },
+    unrestricted: { type: "boolean" },
+    mcp: { type: "string", multiple: true },
+    options: { type: "string", multiple: true },
+    "no-mcp": { type: "boolean" },
+    plugin: { type: "string", multiple: true },
+    model: { type: "string" },
+    "idle-timeout": { type: "string" },
+    ttl: { type: "string" },
+    title: { type: "string" },
+    socket: { type: "string" },
+    home: { type: "string" },
+    "stop-now": { type: "boolean" },
+    "no-plugin": { type: "boolean" },
+    "request-key": { type: "string" },
+    project: { type: "string" },
+    identity: { type: "string" },
+    agent: { type: "string" },
+    "token-env": { type: "string" },
+    interactive: { type: "boolean" },
+    "on-exit": { type: "string" },
+    timeout: { type: "string" },
+};
+/**
+ * Whether a command's arguments ask for its help: `--help` or `-h` before any
+ * `--`, and not the value of a flag that takes one.
+ */
+export function asksForHelp(args) {
+    for (let i = 0; i < args.length; i++) {
+        const arg = args[i];
+        if (arg === "--")
+            return false;
+        if (arg === "--help" || arg === "-h")
+            return true;
+        const name = arg.startsWith("--") ? arg.slice(2) : undefined;
+        const option = name && !name.includes("=")
+            ? CLI_OPTIONS[name]
+            : undefined;
+        if (option?.type === "string")
+            i++;
+    }
+    return false;
+}

package/dist/duration.js

@@ -19,3 +19,20 @@ export function parseDuration(value, label) {
         throw new Error(`${label} must be a duration like 90s, 30m, 4h, or off (got ${JSON.stringify(value)})`);
     return seconds;
 }
+/**
+ * `wait --timeout`: seconds as a bare number, which is what `timeout(1)` and
+ * shell callers write, or a duration. A lifecycle limit refuses the bare
+ * number because a wrong unit reaps a session; here the unit is the obvious
+ * one. `off` is refused: no timeout is spelled by leaving the flag out.
+ */
+export function waitSeconds(value) {
+    const raw = value.trim();
+    const seconds = /^\d+$/.test(raw)
+        ? Number(raw)
+        : /^\d+[smh]$/i.test(raw)
+            ? parseDuration(raw, "--timeout")
+            : 0;
+    if (seconds <= 0)
+        throw new Error(`--timeout must be seconds or a duration like 90s, 30m, 4h (got ${JSON.stringify(value)})`);
+    return seconds;
+}

package/dist/format.js

@@ -38,6 +38,9 @@ function recordHuman(record, color) {
     const id = record.id ?? record.thread_id ?? record.session_id;
     if (record.stopped === true)
         return `${paint("Stopped", 90)} ${clean(id)}.`;
+    // `stop` on a launch that had already ended changes nothing and says so.
+    if (record.stopped === false)
+        return `Not stopped: ${clean(id)} had already ${clean(record.state)}.`;
     // `muster title` answers with the id and the title alone, no kind.
     if (record.kind === undefined && record.title !== undefined)
         return `Titled ${clean(id)}: ${clean(record.title)}.`;

package/dist/help.js

@@ -97,6 +97,10 @@ export const COMMANDS = [
                 "--request-key KEY",
                 "name the request; sending it again returns the same launch",
             ],
+            [
+                "--on-exit COMMAND",
+                "run COMMAND with /bin/sh once the agent exits, however it ends; muster runs it, outside any sandbox, with MUSTER_ID, MUSTER_RUNTIME, MUSTER_KIND, MUSTER_STATE, MUSTER_EXIT_CODE, MUSTER_SIGNAL, MUSTER_OUTPUT, MUSTER_ERROR and MUSTER_RECORD set",
+            ],
             ["--format FORMAT", "json (default) or human"],
             ["-- RUNTIME_OPTIONS", "everything after -- is passed to the runtime"],
         ],
@@ -105,11 +109,32 @@ export const COMMANDS = [
             "A sealed launch does not read your profile's or the project's settings, so it also loads no rules or CLAUDE.md.",
             "Use one of the three flags, --level, or --permissions/--sandbox; naming two is an error rather than a precedence rule.",
             "A pty launch keeps this process running and Ctrl-C stops it; a tmux launch survives this process exiting.",
+            "--on-exit: MUSTER_EXIT_CODE and MUSTER_SIGNAL are empty for a session, whose terminal keeps no exit status, and MUSTER_OUTPUT (the captured output's path) is empty for one too. The command's own output goes to ~/.muster/on-exit/<launch id>.log. A launch that fails at readiness runs it too, with MUSTER_STATE=failed and MUSTER_ERROR. wait and list show it as on_exit: {claimed_at, result}. It is not run if the machine restarts first.",
         ],
         examples: [
             "muster run claude --prompt 'Review this project' --cwd .",
             "muster run claude --read-only --prompt 'Audit this' --cwd .",
             "muster run codex --identity work --write-here --prompt 'Ship it'",
+            "muster run codex --kind task --prompt 'Review' --on-exit 'notify.sh'",
+        ],
+    },
+    {
+        name: "wait",
+        summary: "block until a session or task has ended",
+        synopsis: "muster wait <id> [--timeout SECONDS] [--format json|human]",
+        options: [
+            [
+                "--timeout SECONDS",
+                "give up after this long: seconds, or 90s, 30m, 4h",
+            ],
+            ["--format FORMAT", "json (default) or human"],
+        ],
+        notes: [
+            "Prints the final record, with state, exit_code and signal (null for a session, whose terminal keeps no exit status). Exits 0 when it has ended, 2 for an id that names nothing, and 124 on timeout, printing the record as it stands.",
+        ],
+        examples: [
+            "muster wait 6fae755b-8f5d-4425-8ed7-69e84bfac8c5",
+            "muster wait simple-cms-46 --timeout 30m",
         ],
     },
     {

package/dist/identity/processes.js

@@ -96,10 +96,13 @@ export async function descendants(root, deadline = Date.now() + 1000, group) {
 /** Whether anything still runs in a group we started. */
 export async function groupAlive(group, deadline = Date.now() + 1000) {
     try {
-        const rows = (await command("ps", ["-axo", "pid=,pgid="], Math.max(1, Math.min(1000, deadline - Date.now()))))
+        const rows = (await command("ps", ["-axo", "pid=,pgid=,stat="], Math.max(1, Math.min(1000, deadline - Date.now()))))
             .split("\n")
-            .map((l) => l.trim().split(/\s+/).map(Number));
-        return rows.some(([pid, pgid]) => pid && pid > 1 && pgid === group);
+            .map((l) => l.trim().split(/\s+/));
+        // A zombie is not alive, as `processRef` already holds: a group left with
+        // only an unreaped zombie in it has ended, and counting it kept `wait`
+        // waiting for as long as its parent chose not to reap.
+        return rows.some(([pid, pgid, stat]) => Number(pid) > 1 && Number(pgid) === group && !stat?.startsWith("Z"));
     }
     catch {
         return false;

package/dist/muster.js

@@ -5,7 +5,8 @@ import { Server } from "@modelcontextprotocol/sdk/server/index.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { CallToolRequestSchema, ListToolsRequestSchema, } from "@modelcontextprotocol/sdk/types.js";
 import { z } from "zod";
-import { Muster } from "./run.js";
+import { Muster, UnknownRun, durableId, failedLaunchId } from "./run.js";
+import { waitSeconds } from "./duration.js";
 import { delay } from "./identity/processes.js";
 import { formatHuman, humanColorEnabled, statusText } from "./format.js";
 import { checkPlugins, npmRegistry } from "./doctor.js";
@@ -17,6 +18,7 @@ import { toolGate } from "./mcp.js";
 import { ALLOW_LAUNCH, MCP_SUBCOMMANDS, mcpCommand, notEnabled, selfInvocation, } from "./mcp-install.js";
 import { normalizeOpenFlag, usesTerminalFlag } from "./guard.js";
 import { commandHelp, topLevelHelp } from "./help.js";
+import { CLI_OPTIONS, asksForHelp } from "./cli-options.js";
 import { promptForMissing, terminalReader } from "./prompt.js";
 const listSchema = z
     .object({ kind: z.enum(["session", "task"]).optional() })
@@ -67,9 +69,10 @@ async function main() {
     }
     // Recognised before parsing, not during it: the parser is strict and has no
     // `help` option, so `muster setup-identity --help` used to be answered with
-    // "Unknown option '--help'". Only in second position, so a `--help` that is
-    // some flag's value still reaches the command it was written for.
-    if ((argv[1] === "--help" || argv[1] === "-h") && commandHelp(argv[0])) {
+    // "Unknown option '--help'". Anywhere among the command's own flags, so
+    // `muster run codex --help` is help too; see `asksForHelp` for the `--help`
+    // that is some flag's value, which still reaches the command.
+    if (commandHelp(argv[0] ?? "") && asksForHelp(argv.slice(1))) {
         process.stdout.write(commandHelp(argv[0]));
         return;
     }
@@ -201,41 +204,13 @@ async function main() {
             args: normalizeOpenFlag(argv),
             allowPositionals: true,
             strict: true,
-            options: {
-                prompt: { type: "string" },
-                cwd: { type: "string" },
-                kind: { type: "string" },
-                host: { type: "string" },
-                open: { type: "boolean" },
-                terminal: { type: "string" },
-                format: { type: "string" },
-                permissions: { type: "string" },
-                sandbox: { type: "string" },
-                level: { type: "string" },
-                "read-only": { type: "boolean" },
-                "write-here": { type: "boolean" },
-                unrestricted: { type: "boolean" },
-                mcp: { type: "string", multiple: true },
-                options: { type: "string", multiple: true },
-                "no-mcp": { type: "boolean" },
-                plugin: { type: "string", multiple: true },
-                model: { type: "string" },
-                "idle-timeout": { type: "string" },
-                ttl: { type: "string" },
-                title: { type: "string" },
-                socket: { type: "string" },
-                home: { type: "string" },
-                "stop-now": { type: "boolean" },
-                "no-plugin": { type: "boolean" },
-                "request-key": { type: "string" },
-                project: { type: "string" },
-                identity: { type: "string" },
-                agent: { type: "string" },
-                "token-env": { type: "string" },
-                interactive: { type: "boolean" },
-            },
+            options: CLI_OPTIONS,
         });
-        const { format = "json", "no-mcp": noMcp, "no-plugin": noPlugin, "request-key": requestKey, "idle-timeout": idleTimeout, socket, home: homeFlag, "stop-now": stopNow, "read-only": readOnly, "write-here": writeHere, unrestricted, ...values } = parsed;
+        const { format = "json", "no-mcp": noMcp, "no-plugin": noPlugin, "request-key": requestKey, "idle-timeout": idleTimeout, socket, home: homeFlag, "stop-now": stopNow, "read-only": readOnly, "write-here": writeHere, unrestricted, "on-exit": onExit, timeout, ...values } = parsed;
+        if (onExit !== undefined && command !== "run")
+            throw new Error("--on-exit applies only to run");
+        if (timeout !== undefined && command !== "wait")
+            throw new Error("--timeout applies only to wait");
         // The plain spellings of the three sealed levels. Each is a level, so the
         // existing exclusivity with --permissions/--sandbox applies unchanged once
         // it is folded into `values.level`.
@@ -267,7 +242,9 @@ async function main() {
             ["--home", homeFlag],
             ["--stop-now", stopNow],
         ])
-            if (value !== undefined && command !== "reap-check")
+            if (value !== undefined &&
+                command !== "reap-check" &&
+                !(name === "--home" && command === "on-exit-watch"))
                 throw new Error(`${name} applies only to reap-check`);
         if (requestKey !== undefined && command !== "run")
             throw new Error("--request-key applies only to run");
@@ -297,13 +274,41 @@ async function main() {
                 process.stderr.write("[muster] --terminal is deprecated and will be removed in v2; pass the app to --open instead, or set terminal in config.toml\n");
             const runtime = positionals[0];
             const args = positionals.slice(1);
-            const record = await muster.run({
-                runtime,
-                ...values,
-                ...(requestKey === undefined ? {} : { requestKey }),
-                ...(idleTimeout === undefined ? {} : { idleTimeout }),
-                args,
-            });
+            let record;
+            try {
+                record = await muster.run({
+                    runtime,
+                    ...values,
+                    ...(requestKey === undefined ? {} : { requestKey }),
+                    ...(idleTimeout === undefined ? {} : { idleTimeout }),
+                    args,
+                });
+            }
+            catch (e) {
+                // A launch that failed still ended, and the caller is waiting on its
+                // command to say so: armed on the failed entry, the watcher finds it
+                // already over and runs it with MUSTER_STATE=failed and MUSTER_ERROR.
+                // The launch's own error is still what this command fails with.
+                const launchId = failedLaunchId(e);
+                if (onExit !== undefined && launchId)
+                    await muster
+                        .armOnExit(launchId, onExit, process.cwd())
+                        .catch((armError) => process.stderr.write(`[muster] --on-exit not run for the failed launch: ${armError.message}\n`));
+                throw e;
+            }
+            // Armed after the launch and before the record is printed, so a caller
+            // that reads the record knows the watcher exists. A launch that is up
+            // but could not be armed still prints its record — the session is real
+            // and needs stopping — and then fails, because the caller asked for a
+            // signal it will not get.
+            if (onExit !== undefined)
+                try {
+                    await muster.armOnExit(durableId(record), onExit, process.cwd());
+                }
+                catch (e) {
+                    print(record);
+                    throw e;
+                }
             print(record);
             if (record.kind === "session" && record.host === "pty") {
                 process.stderr.write("[muster] pty: not watchable or attachable; this process owns the session. Ctrl-C stops it.\n");
@@ -409,6 +414,41 @@ async function main() {
             const { id } = idSchema.parse({ id: positionals[0] });
             print(await muster.title(id, positionals[1]));
         }
+        else if (command === "wait") {
+            if (positionals.length !== 1 || Object.keys(values).length)
+                throw new Error("wait requires exactly one id");
+            const { id } = idSchema.parse({ id: positionals[0] });
+            try {
+                const { done, record } = await muster.wait(id, {
+                    ...(timeout === undefined
+                        ? {}
+                        : { timeoutMs: waitSeconds(timeout) * 1000 }),
+                });
+                print(record);
+                // 124 is timeout(1)'s status, which a shell caller already tests for.
+                if (!done)
+                    process.exitCode = 124;
+            }
+            catch (e) {
+                if (!(e instanceof UnknownRun))
+                    throw e;
+                process.stderr.write(`[muster] ${e.message}\n`);
+                process.exitCode = 2;
+            }
+        }
+        else if (command === "on-exit-watch") {
+            // Hidden, like reap-check: started detached by `run --on-exit`, never
+            // typed. Its own exit status is unread; the command's goes to the log.
+            if (positionals.length !== 1 || !homeFlag)
+                throw new Error("on-exit-watch requires --home and one launch id");
+            const watcher = await Muster.create({ home: homeFlag });
+            try {
+                await watcher.runOnExit(positionals[0]);
+            }
+            finally {
+                await watcher.close();
+            }
+        }
         else if (command === "reap-check") {
             // Hidden: armed by muster itself as a tmux job, never typed by a person.
             // NOTE: `Muster.create()` above already ran against the REAL home before
@@ -435,6 +475,13 @@ async function main() {
     }
 }
 main().catch((e) => {
-    process.stderr.write(`[muster] ${e.message}\n`);
+    const command = process.argv[2] ?? "";
+    // parseArgs's own text explains how to pass a positional that starts with a
+    // dash, which is never what a mistyped flag meant; the flag list is.
+    const hint = e.code === "ERR_PARSE_ARGS_UNKNOWN_OPTION" &&
+        commandHelp(command)
+        ? `\n[muster] see \`muster ${command} --help\` for its options`
+        : "";
+    process.stderr.write(`[muster] ${e.message}${hint}\n`);
     process.exitCode = 1;
 });

package/dist/on-exit.js

@@ -0,0 +1,76 @@
+/**
+ * Running an `--on-exit` command: what it is told, and where its output goes.
+ * Kept out of run.ts so the environment contract can be tested without a
+ * launch, since that contract is what every caller's script depends on.
+ */
+import { mkdir, open, stat } from "node:fs/promises";
+import { dirname } from "node:path";
+import { spawn } from "node:child_process";
+/**
+ * The variables the command receives. Strings only, and an empty string where
+ * the fact was not observed — a session's exit status, a session's output —
+ * because a shell cannot tell an unset variable from a mistyped one.
+ */
+export function onExitEnv(entry, record) {
+    return {
+        MUSTER_ID: String(record.id),
+        MUSTER_RUNTIME: entry.runtime,
+        MUSTER_KIND: entry.kind,
+        MUSTER_STATE: String(record.state),
+        MUSTER_EXIT_CODE: record.exit_code === null || record.exit_code === undefined
+            ? ""
+            : String(record.exit_code),
+        MUSTER_SIGNAL: record.signal ? String(record.signal) : "",
+        // A path rather than the text: a task's output can be megabytes, and the
+        // environment has a size limit that would fail the command outright.
+        MUSTER_OUTPUT: entry.kind === "task" && entry.outputPath ? entry.outputPath : "",
+        // Why a launch failed before it was ever up (`MUSTER_STATE=failed`).
+        MUSTER_ERROR: record.error ? String(record.error) : "",
+        MUSTER_RECORD: JSON.stringify(record),
+    };
+}
+/**
+ * Where the command will run and where its output will go, settled before the
+ * command is claimed, because a claim is spent whether or not it then starts.
+ * Neither failure is allowed to cost the signal: a directory removed since the
+ * launch falls back to `fallbackCwd`, and a log that cannot be opened leaves
+ * the output discarded. Each fallback is reported, for the launch log.
+ */
+export async function prepareOnExit(opts) {
+    const warnings = [];
+    let cwd = opts.cwd;
+    try {
+        if (!(await stat(cwd)).isDirectory())
+            throw new Error("not a directory");
+    }
+    catch (e) {
+        warnings.push(`cwd ${cwd} unusable (${e.message}); ran in ${opts.fallbackCwd}`);
+        cwd = opts.fallbackCwd;
+    }
+    try {
+        await mkdir(dirname(opts.logPath), { recursive: true, mode: 0o700 });
+        return { cwd, log: await open(opts.logPath, "a", 0o600), warnings };
+    }
+    catch (e) {
+        warnings.push(`log ${opts.logPath} unusable (${e.message}); output discarded`);
+        return { cwd, warnings };
+    }
+}
+/**
+ * Runs `command` through `/bin/sh -c`, resolving to how it ended, or to why it
+ * never started. `onSpawn` is called once the shell is running, so whatever it
+ * does cannot keep the command from starting.
+ */
+export async function runOnExitCommand(opts) {
+    const out = opts.log?.fd ?? "ignore";
+    return new Promise((resolve) => {
+        const child = spawn("/bin/sh", ["-c", opts.command], {
+            cwd: opts.cwd,
+            env: { ...opts.env, ...onExitEnv(opts.entry, opts.record) },
+            stdio: ["ignore", out, out],
+        });
+        child.once("spawn", () => opts.onSpawn?.());
+        child.once("error", (e) => resolve({ exit_code: null, signal: null, error: e.message }));
+        child.once("exit", (code, signal) => resolve({ exit_code: code, signal: signal ?? null }));
+    });
+}

package/dist/registry.js

@@ -5,6 +5,19 @@ import { execFile } from "node:child_process";
 import { promisify } from "node:util";
 import { processRef, isSame, delay, groupAlive, } from "./identity/processes.js";
 import { LOCAL, requesterKey } from "./requester.js";
+/**
+ * Whether a vanished launch was ended by its stop: a stop was asked, and no
+ * task exit file was written before it. The file is the worker's, published
+ * by rename, so its mtime is when the exit was recorded.
+ */
+async function stoppedBeforeExit(entry) {
+    if (!entry.stopRequestedAt)
+        return false;
+    if (entry.kind !== "task" || !entry.exitPath)
+        return true;
+    const written = await stat(entry.exitPath).then((s) => s.mtimeMs, () => undefined);
+    return written === undefined || written >= Date.parse(entry.stopRequestedAt);
+}
 /**
  * A request key arrived again with different parameters. Distinct from a launch
  * failure: nothing was attempted, and the caller reused a key it should have
@@ -114,7 +127,9 @@ export class Registry {
                     // concluding the session ended: it is the signal that survives both.
                     if (!(await Promise.all((entry.descendants ?? []).map(isSame))).some(Boolean) &&
                         !(entry.group && (await groupAlive(entry.group))))
-                        entry.status = "exited";
+                        entry.status = (await stoppedBeforeExit(entry))
+                            ? "stopped"
+                            : "exited";
                 }
             }
             const result = await fn(entries);
@@ -217,6 +232,71 @@ export class Registry {
             return { entry: { ...entry }, deduped: false };
         });
     }
+    /**
+     * One entry, read without the lock. Safe because every write lands by rename,
+     * so a reader sees the old file or the new one and never half of either. For
+     * polling only: it does not run the liveness sweep, so a status read here may
+     * be stale until the next transaction.
+     */
+    async peek(launchId) {
+        try {
+            const entries = JSON.parse(await readFile(join(this.home, "registry.json"), "utf8"));
+            return entries.find((e) => e.launchId === launchId);
+        }
+        catch (e) {
+            if (e.code === "ENOENT")
+                return undefined;
+            throw e;
+        }
+    }
+    /**
+     * True at most once per launch. The check and the write share a transaction,
+     * so two watchers racing over the same exit cannot both run the command.
+     * At most, not exactly: a watcher that dies after claiming runs nothing.
+     */
+    async claimOnExit(launchId) {
+        return this.transaction(async (entries) => {
+            const entry = entries.find((e) => e.launchId === launchId);
+            if (!entry?.onExit || entry.onExitFiredAt)
+                return undefined;
+            entry.onExitFiredAt = new Date().toISOString();
+            return { ...entry };
+        });
+    }
+    /**
+     * Records that `stop` was asked, unless the launch has already ended, in
+     * which case nothing changes and undefined is returned: stopping a finished
+     * launch must not rewrite how it finished. `ended` is asked inside the
+     * transaction, after its sweep, so a process that vanished is already
+     * `exited` by the time it looks.
+     */
+    async requestStop(launchId, ended) {
+        return this.transaction(async (entries) => {
+            const entry = entries.find((e) => e.launchId === launchId);
+            if (!entry)
+                throw new Error(`Unknown launch ${launchId}`);
+            if (await ended(entry))
+                return undefined;
+            entry.stopRequestedAt ??= new Date().toISOString();
+            return { ...entry };
+        });
+    }
+    /**
+     * Records the `--on-exit` command, refusing one already recorded. The check
+     * and the write share a transaction: two callers handed the same launch (a
+     * request key sent twice) would otherwise both see none and both succeed,
+     * with the second command silently replacing the first.
+     */
+    async armOnExit(launchId, onExit) {
+        return this.transaction(async (entries) => {
+            const entry = entries.find((e) => e.launchId === launchId);
+            if (!entry)
+                throw new Error(`Unknown launch ${launchId}`);
+            if (entry.onExit)
+                throw new Error(`launch ${entry.id} already has an --on-exit command; it is claimed at most once, so a second is refused rather than dropped`);
+            entry.onExit = onExit;
+        });
+    }
     async update(id, patch) {
         return this.transaction(async (entries) => {
             const byLaunch = entries.find((e) => e.launchId === id);

package/dist/run.js

@@ -17,7 +17,7 @@ import { hosts, selectHost } from "./hosts/index.js";
 import { AGENTS } from "./agents.js";
 import { assertOptionsSupported, screenWorkspace } from "./options.js";
 import { workspaceTrustKey } from "./identity/workspace-key.js";
-import { processRef, isSame, stopTree, delay, descendants, ownedGroup, } from "./identity/processes.js";
+import { processRef, isSame, stopTree, delay, descendants, ownedGroup, groupAlive, } from "./identity/processes.js";
 import { resolveCodex } from "./identity/codex.js";
 import { claudeConfigDir, claudeRegistryDiagnostic, claudeWorkspaceTrusted, resolveClaude, } from "./identity/claude.js";
 import { openCodeEndpointOwnedBy, resolveOpenCode, } from "./identity/opencode.js";
@@ -33,6 +33,7 @@ import { assignNames, resolvePeer, runtimeName, suffixOf, } from "./naming.js";
 import { selectMcp, prepareMcp, mcpEnvironment, } from "./mcp.js";
 import { pollInterval } from "./reap.js";
 import { selectPlugins } from "./plugins.js";
+import { prepareOnExit, runOnExitCommand } from "./on-exit.js";
 function taskHandle(entry, pid) {
     return {
         kind: "task",
@@ -66,6 +67,72 @@ function taskHandle(entry, pid) {
  * as "there is no such variable" via `?? entry.identityPath` and an absent
  * value is what that idiom expects.
  */
+/**
+ * A finished record always answers the three questions a waiter has, even
+ * where the answer is "not observed": a session's agent exits inside its
+ * terminal host, which keeps no exit status, so only a task carries real ones.
+ */
+/**
+ * Whether the `--on-exit` command was claimed and what became of it, on every
+ * record of a launch that has one: a claim with no result is a watcher that
+ * died mid-command, and the only place that shows is here.
+ */
+function onExitView(e) {
+    if (!e.onExit)
+        return {};
+    return {
+        on_exit: {
+            claimed_at: e.onExitFiredAt ?? null,
+            result: e.onExitResult ?? null,
+        },
+    };
+}
+function finalRecord(record) {
+    return {
+        ...record,
+        exit_code: record.exit_code ?? null,
+        signal: record.signal ?? null,
+    };
+}
+/**
+ * A task's exit file, and when it was written: undefined until it holds a
+ * record. Only a record that parses counts — a file that exists but cannot be
+ * read as one is never the end, whatever wrote it. The time is the file's,
+ * which the worker publishes by rename, so it is when the exit was recorded.
+ */
+async function readExit(path) {
+    if (!path)
+        return undefined;
+    try {
+        const [text, info] = await Promise.all([
+            readFile(path, "utf8"),
+            stat(path),
+        ]);
+        const exit = JSON.parse(text);
+        if (exit && typeof exit === "object" && "code" in exit)
+            return { ...exit, at: info.mtimeMs };
+    }
+    catch { }
+    return undefined;
+}
+/** Whether a stop, not the agent, ended a launch with this exit record. */
+function endedByStop(entry, exit) {
+    if (!entry.stopRequestedAt)
+        return false;
+    return !exit || exit.at >= Date.parse(entry.stopRequestedAt);
+}
+const FAILED_LAUNCH = Symbol("muster.failedLaunch");
+/**
+ * The launch a failed `run` had reserved, so the CLI can still fire its
+ * `--on-exit` for it. Undefined for a refusal made before anything was
+ * reserved — a bad option, or a request key already in use — which is not a
+ * launch that ended and must not be reported as one.
+ */
+export function failedLaunchId(e) {
+    return e && typeof e === "object"
+        ? e[FAILED_LAUNCH]
+        : undefined;
+}
 function configHomeVar(runtime) {
     return AGENTS[runtime].configHomeVar ?? undefined;
 }
@@ -73,6 +140,28 @@ function configHomeVar(runtime) {
 function provisionalTitle(req) {
     return sanitiseTitle(`${basename(req.cwd)} · ${req.runtime}`);
 }
+/**
+ * `wait` was given an id that names no launch. A class of its own because the
+ * CLI answers it with its own exit status: a caller polling for completion must
+ * be able to tell "never existed" from "still running", or a typo waits forever.
+ */
+export class UnknownRun extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "UnknownRun";
+    }
+}
+/**
+ * The id that names a launch for as long as the registry holds it. Not a
+ * name or `canonical_id`: those resolve only among running sessions, so a
+ * session that exits between readiness and `--on-exit` being armed would no
+ * longer be found, and would lose its command.
+ */
+export function durableId(record) {
+    if (record.kind === "task")
+        return record.id;
+    return record.runtime === "codex" ? record.thread_id : record.session_id;
+}
 export class Muster {
     config;
     home;
@@ -911,6 +1000,8 @@ export class Muster {
             });
         }
         catch (e) {
+            if (e && typeof e === "object")
+                e[FAILED_LAUNCH] = entry.launchId;
             await this.stopEntry(entry).catch(() => { });
             const message = e.message;
             await this.registry.update(entry.launchId, {
@@ -1105,10 +1196,35 @@ export class Muster {
     async stop(id, requester = LOCAL) {
         const entry = await this.find(id);
         assertMayAct(entry, requester);
-        await this.stopEntry(entry);
-        await this.registry.update(entry.launchId, { status: "stopped" });
-        await this.log.write({ event: "stopped", launch_id: entry.launchId });
-        return { stopped: true, id: entry.id };
+        // Recorded before the kill: any transaction that sees the processes gone
+        // before the write below lands would otherwise sweep the entry to
+        // `exited`, and a waiter would report a stop as the agent ending itself.
+        // A launch that has already ended is left as it ended: relabelling an
+        // exited task `stopped` would tell its waiter the wrong thing.
+        const requested = await this.registry.requestStop(entry.launchId, (e) => this.ended(e));
+        if (!requested) {
+            // As `wait` and `list` report it, not the raw status: a task's exit
+            // file can land while its worker is still tearing down, and then the
+            // registry still says running though the task has exited.
+            const final = (await this.registry.peek(entry.launchId)) ?? entry;
+            return {
+                stopped: false,
+                id: entry.id,
+                state: (await this.view(final)).state,
+            };
+        }
+        await this.stopEntry(requested);
+        // The agent may have exited on its own between the check and the kill;
+        // an exit recorded before the stop was asked is still an exit.
+        const exit = requested.kind === "task"
+            ? await readExit(requested.exitPath)
+            : undefined;
+        const status = endedByStop(requested, exit) ? "stopped" : "exited";
+        await this.registry.update(entry.launchId, { status });
+        await this.log.write({ event: status, launch_id: entry.launchId });
+        return status === "stopped"
+            ? { stopped: true, id: entry.id }
+            : { stopped: false, id: entry.id, state: status };
     }
     /**
      * Where to look for the agent's own session registry. Two different uses of an
@@ -1200,68 +1316,269 @@ export class Muster {
         for (const original of entries) {
             if (kind && kind !== original.kind)
                 continue;
-            const e = await this.refreshed(original);
-            if (e.mcpConfigPath && !["starting", "running"].includes(e.status))
-                await rm(e.mcpConfigPath, { force: true });
-            if (e.identityPath && !["starting", "running"].includes(e.status))
-                await rm(e.identityPath, { recursive: true, force: true });
-            if (e.kind === "session")
-                result.push({
-                    ...e.peer,
-                    kind: "session",
-                    ...(e.permissions
-                        ? { permissions: e.permissions, sandbox: e.sandbox }
-                        : {}),
-                    // From the entry, not the peer: a listing must report the same fact a
-                    // launch result does, and an entry predating the field has none.
-                    ...(e.enforcement ? { enforcement: e.enforcement } : {}),
-                    ...(e.identity ? { identity: e.identity } : {}),
-                    // Absent on an entry written before the field, which is a different
-                    // fact from an explicit null: that one means Muster resolved nothing
-                    // and the child chose. Both are more useful than a guess.
-                    ...(e.model !== undefined
-                        ? { model: e.model, model_source: e.modelSource ?? null }
-                        : {}),
-                    ...(e.idleTimeout ? { idle_timeout: e.idleTimeout } : {}),
-                    ...(e.ttl ? { ttl: e.ttl } : {}),
-                    id: e.id,
-                    runtime: e.peer?.runtime ?? e.runtime,
-                    state: e.status === "running" ? (e.peer?.state ?? "idle") : e.status,
-                    host: e.host,
-                    ...(e.error ? { error: e.error } : {}),
-                });
-            else {
-                let exit;
-                try {
-                    exit = JSON.parse(await readFile(e.exitPath, "utf8"));
-                }
-                catch { }
-                result.push({
-                    kind: "task",
-                    ...(e.mcp ? { mcp: e.mcp } : {}),
-                    ...(e.mcp_warnings ? { mcp_warnings: e.mcp_warnings } : {}),
-                    ...(e.permissions
-                        ? { permissions: e.permissions, sandbox: e.sandbox }
-                        : {}),
-                    ...(e.enforcement ? { enforcement: e.enforcement } : {}),
-                    ...(e.identity ? { identity: e.identity } : {}),
-                    // Absent on an entry written before the field, which is a different
-                    // fact from an explicit null: that one means Muster resolved nothing
-                    // and the child chose. Both are more useful than a guess.
-                    ...(e.model !== undefined
-                        ? { model: e.model, model_source: e.modelSource ?? null }
-                        : {}),
-                    id: e.id,
-                    runtime: e.runtime,
-                    cwd: e.cwd,
-                    state: exit ? "exited" : e.status,
-                    ...(e.root ? { pid: e.root.pid } : {}),
-                    ...(exit ? { exit_code: exit.code, signal: exit.signal } : {}),
-                    ...(e.error ? { error: e.error } : {}),
+            result.push(await this.view(original));
+        }
+        return result;
+    }
+    /**
+     * One entry as `list` reports it. Shared with `wait`, so the record a waiter
+     * is handed is the record a listing would have shown for the same launch.
+     */
+    async view(original) {
+        const e = await this.refreshed(original);
+        if (e.mcpConfigPath && !["starting", "running"].includes(e.status))
+            await rm(e.mcpConfigPath, { force: true });
+        if (e.identityPath && !["starting", "running"].includes(e.status))
+            await rm(e.identityPath, { recursive: true, force: true });
+        if (e.kind === "session")
+            return {
+                ...e.peer,
+                kind: "session",
+                ...(e.permissions
+                    ? { permissions: e.permissions, sandbox: e.sandbox }
+                    : {}),
+                // From the entry, not the peer: a listing must report the same fact a
+                // launch result does, and an entry predating the field has none.
+                ...(e.enforcement ? { enforcement: e.enforcement } : {}),
+                ...(e.identity ? { identity: e.identity } : {}),
+                // Absent on an entry written before the field, which is a different
+                // fact from an explicit null: that one means Muster resolved nothing
+                // and the child chose. Both are more useful than a guess.
+                ...(e.model !== undefined
+                    ? { model: e.model, model_source: e.modelSource ?? null }
+                    : {}),
+                ...(e.idleTimeout ? { idle_timeout: e.idleTimeout } : {}),
+                ...(e.ttl ? { ttl: e.ttl } : {}),
+                id: e.id,
+                runtime: e.peer?.runtime ?? e.runtime,
+                state: e.status === "running" ? (e.peer?.state ?? "idle") : e.status,
+                host: e.host,
+                ...(e.error ? { error: e.error } : {}),
+                ...onExitView(e),
+            };
+        else {
+            const exit = await readExit(e.exitPath);
+            return {
+                kind: "task",
+                ...(e.mcp ? { mcp: e.mcp } : {}),
+                ...(e.mcp_warnings ? { mcp_warnings: e.mcp_warnings } : {}),
+                ...(e.permissions
+                    ? { permissions: e.permissions, sandbox: e.sandbox }
+                    : {}),
+                ...(e.enforcement ? { enforcement: e.enforcement } : {}),
+                ...(e.identity ? { identity: e.identity } : {}),
+                // Absent on an entry written before the field, which is a different
+                // fact from an explicit null: that one means Muster resolved nothing
+                // and the child chose. Both are more useful than a guess.
+                ...(e.model !== undefined
+                    ? { model: e.model, model_source: e.modelSource ?? null }
+                    : {}),
+                id: e.id,
+                runtime: e.runtime,
+                cwd: e.cwd,
+                // A stop wins over an exit file written after it: the worker writes
+                // that file on the SIGTERM a stop sends, so its presence says the task
+                // ended, not why. One written before the stop was asked is an exit.
+                state: (exit || !["starting", "running"].includes(e.status)) &&
+                    endedByStop(e, exit)
+                    ? "stopped"
+                    : exit
+                        ? "exited"
+                        : e.status,
+                ...(e.root ? { pid: e.root.pid } : {}),
+                ...(exit ? { exit_code: exit.code, signal: exit.signal } : {}),
+                ...(e.error ? { error: e.error } : {}),
+                ...onExitView(e),
+            };
+        }
+    }
+    /**
+     * The entry an id names, by launch id first. A task's id IS its launch id; a
+     * session's durable id replaces it at readiness, and the `--on-exit` watcher
+     * is armed with the launch id, which is the one handle that never changes.
+     */
+    async locate(id) {
+        const byLaunch = (await this.registry.all()).find((e) => e.launchId === id);
+        if (byLaunch)
+            return byLaunch;
+        try {
+            return await this.find(id);
+        }
+        catch (e) {
+            const message = e.message;
+            // The resolver's "unknown: <every live name>" reads as a list of
+            // matches; the ambiguous case does want its candidates, so it keeps them.
+            throw new UnknownRun(message.startsWith("unknown")
+                ? `no session or task matches ${JSON.stringify(id)}`
+                : message);
+        }
+    }
+    /**
+     * Whether the launch has ended, from cheap evidence about this one entry.
+     * True is provisional for a process that vanished: the caller confirms it
+     * through a transaction, whose sweep is what records `exited`.
+     */
+    async ended(entry) {
+        if (!["starting", "running"].includes(entry.status))
+            return true;
+        // A task's worker writes this the moment the agent exits, before it tears
+        // down its group: the earliest signal there is, and the only exact one.
+        if (entry.kind === "task" && (await readExit(entry.exitPath)))
+            return true;
+        if (!entry.root)
+            return !(await isSame(entry.owner));
+        if (await isSame(entry.root))
+            return false;
+        if ((await Promise.all((entry.descendants ?? []).map(isSame))).some(Boolean))
+            return false;
+        return !(entry.group && (await groupAlive(entry.group)));
+    }
+    /**
+     * Blocks until the launch `id` names is no longer running, or `timeoutMs`
+     * passes. An id that names nothing throws `UnknownRun` at once rather than
+     * waiting for something that will never appear.
+     *
+     * Polls one entry, lock-free, backing off to two seconds: a watcher may wait
+     * days on a long session, and a transaction per poll would take the registry
+     * lock and run `ps` for every live entry, every time.
+     */
+    async wait(id, opts = {}) {
+        const { launchId } = await this.locate(id);
+        const deadline = opts.timeoutMs === undefined ? Infinity : Date.now() + opts.timeoutMs;
+        let interval = opts.pollMs ?? 250;
+        while (true) {
+            const peeked = await this.registry.peek(launchId);
+            if (!peeked)
+                throw new UnknownRun(`Unknown launch ${id}`);
+            if (await this.ended(peeked)) {
+                // The transaction's sweep settles what the cheap check guessed.
+                const settled = (await this.registry.all()).find((e) => e.launchId === launchId);
+                if (await this.ended(settled))
+                    return { done: true, record: finalRecord(await this.view(settled)) };
+            }
+            const left = deadline - Date.now();
+            if (left <= 0)
+                return { done: false, record: await this.view(peeked) };
+            await delay(Math.min(interval, left));
+            interval = Math.min(interval * 1.5, 2000);
+        }
+    }
+    /**
+     * Records `command` against the launch and starts the detached watcher that
+     * runs it once the agent exits. The watcher is muster, outside any sandbox
+     * the agent runs in, so a sealed agent that can write nothing still produces
+     * the signal — and it fires on a crash or a kill, which a report the agent
+     * makes about itself never would.
+     *
+     * CLI only, deliberately absent from the MCP `run` schema: the command runs
+     * unsandboxed as the operator, which no MCP caller is granted.
+     */
+    async armOnExit(id, command, cwd) {
+        if (!command.trim())
+            throw new Error("--on-exit needs a command");
+        const entry = await this.locate(id);
+        await this.registry.armOnExit(entry.launchId, { command, cwd });
+        const watcher = spawn(process.execPath, [
+            // <root>/dist from src under vitest and from dist in the package
+            // alike. Same idiom as the reap-check job.
+            fileURLToPath(new URL("../dist/muster.js", import.meta.url)),
+            "on-exit-watch",
+            "--home",
+            this.home,
+            entry.launchId,
+        ], 
+        // In muster's home, not the caller's directory: the command's directory
+        // is recorded on the entry, and a watcher started inside one that is
+        // later removed cannot even start Node there.
+        { detached: true, stdio: "ignore", env: this.sourceEnv, cwd: this.home });
+        await new Promise((resolve, reject) => {
+            watcher.once("spawn", resolve);
+            watcher.once("error", reject);
+        });
+        watcher.unref();
+        await this.log.write({
+            event: "on_exit_armed",
+            launch_id: entry.launchId,
+            watcher_pid: watcher.pid,
+        });
+    }
+    /**
+     * The watcher's whole job: wait for the launch to end, claim the command so
+     * at most one watcher runs it, and run it. Its output goes to
+     * `<home>/on-exit/<launch id>.log`, its status to the registry entry (shown
+     * as `on_exit` by `wait` and `list`) and then to the launch log.
+     *
+     * Everything that can fail is settled before the claim — where it runs and
+     * where its output goes — because a claim is spent whether or not the
+     * command then starts. After the claim the command is spawned before any
+     * launch-log write, so a log that cannot be written cannot stop it, and its
+     * result lands in the registry first: a claim with no result there is a
+     * watcher that died mid-command, which `on_exit` shows as `result: null`.
+     */
+    async runOnExit(launchId) {
+        const { record } = await this.wait(launchId);
+        // The hook is told how the launch ended, not about itself.
+        const { on_exit: _, ...hookRecord } = record;
+        const armed = (await this.registry.peek(launchId))?.onExit;
+        if (!armed)
+            return null;
+        const prepared = await prepareOnExit({
+            cwd: armed.cwd,
+            fallbackCwd: this.sourceEnv.HOME ?? homedir(),
+            logPath: join(this.home, "on-exit", `${launchId}.log`),
+        });
+        try {
+            const entry = await this.registry.claimOnExit(launchId);
+            if (!entry)
+                return null;
+            let result;
+            try {
+                const outcome = await runOnExitCommand({
+                    command: entry.onExit.command,
+                    cwd: prepared.cwd,
+                    env: this.sourceEnv,
+                    entry,
+                    record: hookRecord,
+                    log: prepared.log,
+                    onSpawn: () => void this.log
+                        .write({
+                        event: "on_exit_claimed",
+                        launch_id: launchId,
+                        state: record.state,
+                        ...(prepared.warnings.length
+                            ? { warnings: prepared.warnings }
+                            : {}),
+                    })
+                        .catch(() => { }),
                 });
+                result = { finished_at: new Date().toISOString(), ...outcome };
+            }
+            catch (e) {
+                result = {
+                    finished_at: new Date().toISOString(),
+                    exit_code: null,
+                    signal: null,
+                    error: e.message,
+                };
             }
+            if (prepared.warnings.length)
+                result.warnings = prepared.warnings;
+            await this.registry.update(launchId, { onExitResult: result });
+            await this.log
+                .write({
+                event: "on_exit",
+                launch_id: launchId,
+                state: record.state,
+                exit_code: result.exit_code,
+                ...(result.signal ? { signal: result.signal } : {}),
+                ...(result.error ? { error: result.error } : {}),
+            })
+                .catch(() => { });
+            return result.exit_code;
+        }
+        finally {
+            await prepared.log?.close();
         }
-        return result;
     }
     async output(id, requester = LOCAL) {
         const entry = await this.find(id);

package/dist/task-worker.js

@@ -1,5 +1,5 @@
 // A per-task owner captures output and the exit result; no pools, retries or daemon.
-import { readFile, writeFile, rm, open } from "node:fs/promises";
+import { readFile, writeFile, rm, open, rename } from "node:fs/promises";
 import { spawn } from "node:child_process";
 import { delay, finishOwnedGroup, processRef, stopTree, } from "./identity/processes.js";
 import { LaunchLog } from "./log.js";
@@ -45,9 +45,14 @@ async function finish(code, signal, error) {
     }
     finally {
         try {
-            await writeFile(spec.exitPath, JSON.stringify({ code, signal, error }), {
+            // Published by rename: `wait` treats this file as the task's end, and a
+            // reader that found it created but not yet written would report a
+            // finished task with no exit status.
+            const tmp = `${spec.exitPath}.tmp`;
+            await writeFile(tmp, JSON.stringify({ code, signal, error }), {
                 mode: 0o600,
             });
+            await rename(tmp, spec.exitPath);
         }
         finally {
             await finishOwnedGroup(code ?? 1);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@brutalsystems/muster",
-  "version": "3.0.2",
+  "version": "3.1.0",
   "description": "Launch instructed, reachable local agent sessions",
   "type": "module",
   "engines": {