muse-crew 0.17.2 → 0.17.3

package/lib/reap-spawns.js

@@ -0,0 +1,407 @@
+#!/usr/bin/env node
+// lib/reap-spawns.js — the Step-0 reaper (sandbox exit, chore pilot Piece 2,
+// 2026-09-26).
+//
+// Shebang CLI. --help answers with usage on stdout, exit 0.
+//
+// Invoked by the tick worker at Step 0, before any new spawns. Reaps
+// worker_runs rows left 'running' by a dead or wedged owner: a row is
+// reapable when it is past timeout_ms + REAP_MARGIN_MS. (The design's
+// owner_ended_at fast path needs a ticks table that does not exist; the
+// timeout clause alone is correct and fail-closed — reaping is deferred,
+// never early.)
+//
+// Mechanics (the honest shape): a Node process cannot close another tick's
+// subagent handle — only the tick worker's runtime can. So the reaper is
+// the decider and recorder, and the tick worker is the hands:
+//
+//   reap-spawns.js --crew-home <home> [--reap-margin-ms <n>]
+//     For each reapable row:
+//     (1) read the session file — a terminal outcome means the driver died
+//         after the outcome: finish the close via `spawn-step.js post`
+//         (never kill-and-retry work already done);
+//     (2) otherwise, if the child is an OS pid with a verified
+//         pid+start-time identity, attempt the kill directly and record
+//         killed_by_reaper | orphaned_unkillable | orphaned_unverified;
+//     (3) otherwise (subagent child, or no child identity), emit a kill
+//         instruction for the tick worker.
+//     Prints one JSON line:
+//       {closed:[...], kill_instructions:[...], errors:[]}
+//     Exit 0 — reaping is normal operation, not a failure.
+//
+//   reap-spawns.js --crew-home <home> --record-close --id <row_id>
+//                  --result <killed|not_found|refused> [--detail ...]
+//     The tick worker calls this after attempting the runtime close from
+//     a kill instruction. Re-reads the session file first (race: the child
+//     may have finished between assess and close — then the close is
+//     finished via post semantics instead). Records the kill with machine
+//     evidence: killed → killed_by_reaper; not_found → killed_by_reaper
+//     ('not found at reap'); refused → orphaned_unkillable. A row with no
+//     child identity at all closes as orphaned_unverified (liveness
+//     unverifiable; no kill attempted).
+//     Prints {recorded:true,row_id,outcome} or {recorded:false,reason}.
+//
+// The reaper MAY: read worker_runs, finish closes, attempt kills with
+// identity match, update rows. It MAY NOT: promote any attempt's output,
+// touch tasks/notes, author verdicts. Unknown kill states keep the row
+// running — a row may only leave 'running' with a closed-vocabulary
+// outcome (enforced by crew-api record-spawn-close).
+//
+// Merge-lock discipline (§7 step 4): the reaper never breaks a merge-lock
+// lease. Lease-break is permitted only from orphaned_unkillable /
+// orphaned_unverified / killed_by_reaper — enforced by the driver
+// (chore-run.js), which reads these outcomes from the ledger.
+
+import { execFileSync } from "node:child_process";
+import { readFileSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import { REAP_MARGIN_MS, isClosedOutcome } from "./chore/spawn-vocab.js";
+
+const LIB_DIR = dirname(fileURLToPath(import.meta.url));
+const CREW_API = join(LIB_DIR, "crew-api.js");
+const SPAWN_STEP = join(LIB_DIR, "spawn-step.js");
+
+const USAGE = [
+  "usage: reap-spawns.js --crew-home <path> [--reap-margin-ms <n>]",
+  "       reap-spawns.js --crew-home <path> --record-close --id <row_id> --result <killed|not_found|refused> [--detail <text>]",
+  "       reap-spawns.js --help",
+  "",
+  "Step-0 reaper: finish-closes reapable spawn rows whose session file",
+  "already holds a terminal outcome, kills verifiable OS-pid children",
+  "directly, and emits kill instructions for subagent children (the tick",
+  "worker performs the runtime close, then --record-close records it).",
+].join("\n");
+
+function log(msg) {
+  process.stderr.write("[reap-spawns] " + msg + "\n");
+}
+
+function printJson(obj) {
+  process.stdout.write(JSON.stringify(obj) + "\n");
+}
+
+function callCrewApi(crewHome, command, argsObj) {
+  const argv = ["--crew-home", crewHome, command];
+  if (argsObj !== undefined) argv.push("--json", JSON.stringify(argsObj));
+  let stdout;
+  try {
+    stdout = execFileSync("node", [CREW_API, ...argv], {
+      encoding: "utf8",
+      maxBuffer: 16 * 1024 * 1024,
+    });
+  } catch (e) {
+    const detail = (e.stderr && e.stderr.toString()) || e.message;
+    throw new Error("crew-api " + command + " failed: " + detail.slice(0, 500));
+  }
+  return JSON.parse(stdout);
+}
+
+function callSpawnPost(sessionPath, key) {
+  // spawn-step post implements the close semantics (classify → closed
+  // vocabulary). The reaper delegates finish-closes to it.
+  let stdout;
+  try {
+    stdout = execFileSync("node", [SPAWN_STEP, "post", "--session", sessionPath, "--key", key], {
+      encoding: "utf8",
+      maxBuffer: 16 * 1024 * 1024,
+    });
+  } catch (e) {
+    const detail = (e.stderr && e.stderr.toString()) || e.message;
+    throw new Error("spawn-step post failed: " + detail.slice(0, 500));
+  }
+  return JSON.parse(stdout);
+}
+
+function sessionPathFor(crewHome, spawnKey) {
+  return join(crewHome, ".spawn-sessions", spawnKey + ".json");
+}
+
+function readSession(path) {
+  try {
+    return { ok: true, session: JSON.parse(readFileSync(path, "utf8")) };
+  } catch (e) {
+    return { ok: false, error: e.message };
+  }
+}
+
+const TERMINAL_SESSION_OUTCOMES = ["returned", "spawn_error", "synthetic", "timeout"];
+
+function sessionIsTerminal(session) {
+  if (!session || !TERMINAL_SESSION_OUTCOMES.includes(session.outcome)) return false;
+  if (session.outcome === "timeout") {
+    // A timeout is terminal only with machine-verified kill evidence;
+    // otherwise the row stays running and the reaper verifies.
+    const ke = session.kill_evidence || {};
+    return ke.verified_dead === true;
+  }
+  return true;
+}
+
+// ── OS-pid kill path ────────────────────────────────────────────────
+// Identity: pid + start-time. /proc/<pid>/stat field 22 (starttime) is the
+// boot-relative tick count — a pid reuse never collides on it. Mismatch →
+// orphaned_unverified, no kill, never.
+
+function procStartTime(pid) {
+  try {
+    const stat = readFileSync("/proc/" + pid + "/stat", "utf8");
+    const afterComm = stat.slice(stat.lastIndexOf(")") + 1).trim().split(/\s+/);
+    // afterComm[0] is state (field 3); starttime (field 22) is index 19.
+    return afterComm[19] || null;
+  } catch (e) {
+    return null;
+  }
+}
+
+function pidAlive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (e) {
+    return e.code !== "ESRCH";
+  }
+}
+
+// /proc/<pid>/stat state field (field 3). A zombie (Z) is dead — kill(pid,0)
+// still succeeds on zombies, so the state field is the authority after a kill,
+// not the liveness probe.
+function procState(pid) {
+  try {
+    const stat = readFileSync("/proc/" + pid + "/stat", "utf8");
+    const afterComm = stat.slice(stat.lastIndexOf(")") + 1).trim().split(/\s+/);
+    return afterComm[0] || null;
+  } catch (e) {
+    return null;
+  }
+}
+
+function attemptOsPidKill(crewHome, row, child) {
+  const pid = child.pid;
+  const recordedStart = String(child.start_time || "");
+  const result = (outcome, evidence) =>
+    ({ outcome, evidence: Object.assign({ method: "os_pid_kill", pid }, evidence || {}) });
+
+  if (!Number.isInteger(pid) || pid <= 0) {
+    return result("orphaned_unverified", { detail: "no usable pid in session child record" });
+  }
+  const currentStart = procStartTime(pid);
+  if (currentStart === null) {
+    // No /proc entry: the process is gone.
+    return result("killed_by_reaper", { detail: "not found at reap" });
+  }
+  if (recordedStart && currentStart !== recordedStart) {
+    return result("orphaned_unverified", {
+      detail: "pid+start-time identity mismatch (pid reused); no kill attempted",
+    });
+  }
+  if (!recordedStart) {
+    return result("orphaned_unverified", {
+      detail: "no recorded start-time; identity unverifiable, no kill attempted",
+    });
+  }
+  try {
+    process.kill(pid, "SIGKILL");
+  } catch (e) {
+    if (e.code === "ESRCH") {
+      return result("killed_by_reaper", { detail: "not found at reap" });
+    }
+    return result("orphaned_unkillable", { detail: "SIGKILL refused: " + e.code });
+  }
+  // kill(2) delivers SIGKILL synchronously, but the target needs a scheduling
+  // slice to tear down: immediately after the syscall it can still read as
+  // state R with kill(pid,0) succeeding. Poll briefly for the terminal state
+  // (gone, or zombie Z = dead awaiting parent reap) before judging.
+  const deadline = Date.now() + 2000;
+  const sleeper = new Int32Array(new SharedArrayBuffer(4));
+  let state = procState(pid);
+  while (state !== null && state !== "Z" && Date.now() < deadline) {
+    Atomics.wait(sleeper, 0, 0, 50);
+    state = procState(pid);
+  }
+  // Gone entirely (no /proc entry), or a zombie (Z — dead, awaiting parent
+  // reap): the kill worked. kill(pid,0) still succeeds on zombies, so the
+  // state field is the authority after a kill, not the liveness probe.
+  if (state === null || state === "Z" || !pidAlive(pid)) {
+    return result("killed_by_reaper", {
+      detail:
+        state === "Z"
+          ? "SIGKILL delivered; process is a zombie (dead, awaiting parent reap)"
+          : "SIGKILL delivered; liveness check confirms dead",
+    });
+  }
+  return result("orphaned_unkillable", { detail: "SIGKILL delivered but process still alive" });
+}
+
+function recordCloseOutcome(crewHome, rowId, outcome, evidence, taskCtx) {
+  if (!isClosedOutcome(outcome)) {
+    throw new Error("refusing to record outcome outside the closed vocabulary: " + outcome);
+  }
+  const closed = callCrewApi(crewHome, "record-spawn-close", {
+    id: rowId,
+    status: "failed",
+    result: Object.assign(
+      {
+        outcome,
+        kind: taskCtx.kind,
+        task_id: taskCtx.task_id,
+        phase: taskCtx.phase,
+        spawn_key: taskCtx.spawn_key,
+      },
+      evidence ? { kill_evidence: evidence } : {}
+    ),
+  });
+  return closed;
+}
+
+// ── assess ──────────────────────────────────────────────────────────
+
+function cmdAssess(crewHome, reapMarginMs) {
+  const listed = callCrewApi(crewHome, "list-reapable-spawns", { reap_margin_ms: reapMarginMs });
+  const closed = [];
+  const killInstructions = [];
+  const errors = [];
+
+  for (const row of listed.spawns || []) {
+    const sp = sessionPathFor(crewHome, row.spawn_key);
+    try {
+      const rs = readSession(sp);
+      if (rs.ok && sessionIsTerminal(rs.session)) {
+        // The driver died after the outcome: finish the close, never
+        // kill-and-retry work already done.
+        const postOut = callSpawnPost(sp, row.spawn_key);
+        closed.push({
+          row_id: row.id,
+          spawn_key: row.spawn_key,
+          via: "session",
+          post: postOut,
+        });
+        log("reaped row " + row.id + " (" + row.spawn_key + ") via session finish-close");
+        continue;
+      }
+      const child = rs.ok && rs.session ? rs.session.child : null;
+      if (child && child.kind === "os_pid") {
+        const killRes = attemptOsPidKill(crewHome, row, child);
+        const rec = recordCloseOutcome(crewHome, row.id, killRes.outcome, killRes.evidence, row);
+        closed.push({
+          row_id: row.id,
+          spawn_key: row.spawn_key,
+          via: "os_pid_kill",
+          outcome: killRes.outcome,
+          noop: !!(rec && rec.noop),
+        });
+        log("reaped row " + row.id + " (" + row.spawn_key + ") via os_pid_kill: " + killRes.outcome);
+        continue;
+      }
+      // Subagent child (or no child identity): Node cannot close it. Emit
+      // the instruction; the tick worker performs the runtime close and
+      // calls --record-close.
+      killInstructions.push({
+        row_id: row.id,
+        spawn_key: row.spawn_key,
+        kind: row.kind,
+        task_id: row.task_id,
+        phase: row.phase,
+        child: child || null,
+        reason: rs.ok ? "no terminal session outcome" : "no session file",
+      });
+      log("kill instruction for row " + row.id + " (" + row.spawn_key + "): child=" + (child ? child.kind : "unknown"));
+    } catch (e) {
+      errors.push({ row_id: row.id, spawn_key: row.spawn_key, error: e.message });
+      log("ERROR reaping row " + row.id + ": " + e.message);
+    }
+  }
+
+  printJson({ closed, kill_instructions: killInstructions, errors });
+  process.exit(0);
+}
+
+// ── record-close ────────────────────────────────────────────────────
+
+function cmdRecordClose(crewHome, rowId, result, detail) {
+  if (!["killed", "not_found", "refused"].includes(result)) {
+    printJson({ recorded: false, reason: "bad_result", detail: "result must be killed|not_found|refused" });
+    process.exit(1);
+  }
+  let rowRes;
+  try {
+    rowRes = callCrewApi(crewHome, "get-spawn-row", { id: rowId });
+  } catch (e) {
+    printJson({ recorded: false, reason: "row_not_found", detail: e.message });
+    process.exit(1);
+  }
+  const row = rowRes.row;
+  if (row.status !== "running") {
+    printJson({ recorded: false, reason: "already_closed", row_id: row.id, status: row.status });
+    process.exit(0);
+  }
+  // Race: the child may have finished between assess and the close. If the
+  // session now holds a terminal outcome, finish that close instead.
+  const rs = readSession(sessionPathFor(crewHome, row.spawn_key));
+  if (rs.ok && sessionIsTerminal(rs.session)) {
+    const postOut = callSpawnPost(sessionPathFor(crewHome, row.spawn_key), row.spawn_key);
+    printJson({ recorded: true, row_id: row.id, via: "session_race", post: postOut });
+    process.exit(0);
+  }
+  const evidence = {
+    method: "runtime_close",
+    result,
+    detail: detail || (result === "not_found" ? "not found at reap" : result === "killed" ? "runtime close confirmed" : "runtime close refused"),
+  };
+  const outcome = result === "refused" ? "orphaned_unkillable" : "killed_by_reaper";
+  let rec;
+  try {
+    rec = recordCloseOutcome(crewHome, row.id, outcome, evidence, row);
+  } catch (e) {
+    printJson({ recorded: false, reason: "record_failed", detail: e.message });
+    process.exit(1);
+  }
+  if (rec && rec.noop) {
+    printJson({ recorded: false, reason: "already_closed", row_id: row.id });
+    process.exit(0);
+  }
+  log("recorded close for row " + row.id + ": " + outcome);
+  printJson({ recorded: true, row_id: row.id, outcome });
+  process.exit(0);
+}
+
+// ── main ────────────────────────────────────────────────────────────
+
+function main() {
+  const argv = process.argv.slice(2);
+  if (argv.length === 0 || argv.includes("--help") || argv.includes("-h")) {
+    process.stdout.write(USAGE + "\n");
+    process.exit(0);
+  }
+  const hi = argv.indexOf("--crew-home");
+  const crewHome = hi === -1 ? null : argv[hi + 1];
+  if (!crewHome) {
+    process.stderr.write("reap-spawns.js: --crew-home <path> is required\n");
+    process.exit(1);
+  }
+  if (argv.includes("--record-close")) {
+    const ii = argv.indexOf("--id");
+    const ri = argv.indexOf("--result");
+    const di = argv.indexOf("--detail");
+    const rowId = ii === -1 ? NaN : parseInt(argv[ii + 1], 10);
+    if (!Number.isInteger(rowId) || rowId <= 0) {
+      process.stderr.write("reap-spawns.js: --record-close requires --id <row_id>\n");
+      process.exit(1);
+    }
+    if (ri === -1 || !argv[ri + 1]) {
+      process.stderr.write("reap-spawns.js: --record-close requires --result <killed|not_found|refused>\n");
+      process.exit(1);
+    }
+    cmdRecordClose(crewHome, rowId, argv[ri + 1], di === -1 ? null : argv[di + 1]);
+    return;
+  }
+  const mi = argv.indexOf("--reap-margin-ms");
+  const margin = mi === -1 ? REAP_MARGIN_MS : parseInt(argv[mi + 1], 10);
+  if (!Number.isInteger(margin) || margin < 0) {
+    process.stderr.write("reap-spawns.js: --reap-margin-ms must be a non-negative integer\n");
+    process.exit(1);
+  }
+  cmdAssess(crewHome, margin);
+}
+
+main();

package/lib/schema.sql

@@ -296,7 +296,18 @@ CREATE TABLE IF NOT EXISTS worker_runs (
   status     TEXT NOT NULL CHECK (status IN ('running', 'completed', 'failed')),
   error      TEXT,
   started_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
-  ended_at   TEXT
+  ended_at   TEXT,
+  -- Spawn ledger (sandbox exit, chore pilot Piece 2, 2026-09-26): one row
+  -- per creative spawn, opened by spawn-step pre and closed by spawn-step
+  -- post / the Step-0 reaper. NULL for non-spawn rows (e.g. dispatcher).
+  -- Kill states live in the result JSON's closed outcome vocabulary, never
+  -- as new statuses. attempt_n is derived by count, never stored.
+  spawn_key     TEXT NULL,
+  owner_tick_seq INTEGER NULL,
+  kind          TEXT NULL CHECK (kind IS NULL OR kind IN ('work-agent','verdict-reask','evidence-gatherer')),
+  timeout_ms    INTEGER NULL,
+  result        TEXT NULL
 );
 CREATE INDEX IF NOT EXISTS worker_runs_task_id_idx ON worker_runs(task_id);
 CREATE INDEX IF NOT EXISTS worker_runs_phase_idx ON worker_runs(phase);
+CREATE INDEX IF NOT EXISTS worker_runs_spawn_key_idx ON worker_runs(spawn_key);

package/lib/spawn-keys.js

@@ -0,0 +1,47 @@
+// lib/spawn-keys.js — pure spawn-key builders for the worker-layer
+// phase modules (sandbox exit, Phase C Piece 2).
+//
+// Generalized from lib/chore/spawn-keys.js (which stays live and untouched):
+// every function here is workflow-neutral.
+//
+// Import-safe: pure functions, no I/O, no clock, no randomness.
+// Key logic from workflows/chore.js:
+//   attemptKey(base, reworkCount)
+//   workRetryKey(stepName, reworkSuffix, attempt)
+//   verdictReaskKey(stepName, reworkSuffix, attempt)
+// plus workKeyBase (the first-attempt work key, inline in chore.js as
+// `"work-" + step.name + (reworkCount > 0 ? "-r" + reworkCount : "")`).
+//
+// Worker-layer difference (2026-09-26): the source's keys were implicitly
+// scoped per workflow process — the runtime keyed agent() calls by explicit
+// key per process, so `work-Triage` could never collide across tasks. The
+// worker-layer ledger (worker_runs) is global per crew home, so every key
+// carries the task id as a suffix. Without it, task B's driver would read
+// task A's completed `work-Triage` row as its own, and the global
+// duplicate-running guard in record-spawn-open would serialize unrelated
+// tasks. The task suffix is the mechanical equivalent of the runtime's
+// per-process scoping.
+
+// attemptKey — replay-key scoping: on rework a re-executed phase mints fresh
+// keys with the same -r<N> suffix as the phase loop. Pure function of inputs,
+// no clock, no randomness (determinism contract).
+export function attemptKey(base, reworkCount) {
+  return base + (reworkCount > 0 ? "-r" + reworkCount : "");
+}
+
+// workKeyBase — the first-attempt work-agent key for a phase visit.
+export function workKeyBase(taskId, stepName, reworkCount) {
+  return "work-" + stepName + (reworkCount > 0 ? "-r" + reworkCount : "") + "-" + taskId;
+}
+
+// workRetryKey — transport-retry work-agent keys: work-<Step>[-r<N>]-t<attempt>-<taskId>
+// (attempt is the 1-based retry number: first retry is -t1).
+export function workRetryKey(taskId, stepName, reworkSuffix, attempt) {
+  return "work-" + stepName + reworkSuffix + "-t" + attempt + "-" + taskId;
+}
+
+// verdictReaskKey — verdict re-ask keys: verdict-reask-<Step>[-r<N>]-a<attempt>-<taskId>
+// (note the -a- infix, not -t-).
+export function verdictReaskKey(taskId, stepName, reworkSuffix, attempt) {
+  return "verdict-reask-" + stepName + reworkSuffix + "-a" + attempt + "-" + taskId;
+}