flowviant 0.43.0 → 0.44.1

package/bin/lib/fleet.mjs

@@ -5,7 +5,13 @@
  * MCP token, and only spawns Claude when the server says an agent has work.
  */
 
-import { mkdirSync, existsSync, rmSync, readdirSync, statSync } from 'node:fs';
+import {
+  mkdirSync,
+  existsSync,
+  rmSync,
+  readdirSync,
+  statSync,
+} from 'node:fs';
 import { execFileSync } from 'node:child_process';
 import { createHash } from 'node:crypto';
 import { homedir } from 'node:os';
@@ -59,8 +65,6 @@ import {
   REGROUND_KICKOFF,
   SYSTEM_PLAN,
   PLAN_TURN_KICKOFF,
-  SYSTEM_WORK,
-  WORK_TURN_KICKOFF,
   SYSTEM_QUICK_EDIT,
   QUICK_EDIT_KICKOFF,
 } from './claude.mjs';
@@ -79,6 +83,7 @@ import {
 import { processDeployJobs, reportDeployConfig } from './deploy.mjs';
 import { machineSnapshot } from './resources.mjs';
 import { detectRuntimes, pickRuntimeFor, RUNTIMES } from './runtimes.mjs';
+import { createWorkManager } from './work.mjs';
 
 async function fetchRoster(haveIds) {
   const url = new URL(FLEET_URL);
@@ -453,6 +458,7 @@ export async function runFleetDaemon() {
   const workers = new Map(); // agentId -> { state, promise, wt, label }
   let daemonAlive = true; // flipped false on shutdown so the stream stops reconnecting
   let stream = null; // push channel handle (set once the loop is set up)
+  let workShutdown = null; // kills live session-turn CLIs (set with the work manager below)
 
   // Shutdown KEEPS the worktrees: in-flight local work survives Ctrl+C and
   // resumes in place on the next run (the task marker matches). Worktrees are
@@ -473,6 +479,14 @@ export async function runFleetDaemon() {
     } catch {
       /* best-effort */
     }
+    // Session-turn CLIs die with the daemon too: an orphan keeps editing the
+    // session worktree and burning quota, and its live-pid lock would make the
+    // restarted daemon skip that tab's turns for as long as it survived.
+    try {
+      workShutdown?.();
+    } catch {
+      /* best-effort */
+    }
     for (const [, w] of workers) {
       w.state.alive = false;
       try {
@@ -496,6 +510,15 @@ export async function runFleetDaemon() {
     teardown();
     process.exit(130);
   });
+  // A service manager stops the daemon with SIGTERM, not Ctrl+C. Without this
+  // handler every child survived a `systemctl stop` — the exact orphaning the
+  // teardown exists to prevent.
+  process.on('SIGTERM', () => {
+    console.log('');
+    note('shutting down (SIGTERM) — stopping workers. Worktrees are kept: in-flight work resumes next run.');
+    teardown();
+    process.exit(143);
+  });
   // Keep the daemon alive on a stray rejection. Many loops here are fire-and-
   // forget (`void drainWiki()`, dispatch, sync) and rely on their callees never
   // rejecting; Node ≥15 terminates the process on an unhandled rejection, which
@@ -1091,7 +1114,9 @@ export async function runFleetDaemon() {
           await reportMergeOutcome(CONSULT_DONE_URL, {
             consultId: job.id,
             ok: false,
-            answer: e?.message ?? 'the planning turn failed',
+            // Scrub, like the success path: an exception routinely quotes
+            // command output, and command output can quote a synced secret.
+            answer: envScrub(String(e?.message ?? 'the planning turn failed')).slice(0, 2000),
           });
           warn(`planning turn failed: ${e?.message ?? e}`);
         } finally {
@@ -1103,312 +1128,23 @@ export async function runFleetDaemon() {
 
   // ── Work sessions — the Workbench tabs ─────────────────────────────────────
   //
-  // A tab is a held Claude session with BUILD permissions in a PERSISTENT
-  // worktree on its own branch. The opposite of a plan directory on both
-  // counts: nothing here is detached and nothing is ever reset — uncommitted
-  // state between turns IS the session, and blowing it away would be closing
-  // the human's editor mid-thought.
-  const WORK_TOKEN_URL = FLEET_URL.replace(/\/agents\/?$/, '/work-token');
-  const WORK_DONE_URL = FLEET_URL.replace(/\/agents\/?$/, '/work-turn-done');
-  const workAnswering = new Set();
-  const workAttempts = new Map(); // turn id -> tries
-  const MAX_WORK_TRIES = 3;
-  /** Per-SESSION serialization, parallel ACROSS sessions: turns within one tab
-   *  must land in order (they share a directory and a context), but two tabs
-   *  are two terminals — the human opened both on purpose. */
-  const workChains = new Map(); // sessionId -> Promise
-
-  let workToken = null;
-  const mintWorkToken = async (force = false) => {
-    if (workToken && !force) return workToken;
-    try {
-      const res = await fetch(WORK_TOKEN_URL, {
-        method: 'POST',
-        headers: { Authorization: `Bearer ${FLEET_TOKEN}`, 'User-Agent': USER_AGENT },
-      });
-      if (!res.ok) return null;
-      const data = await res.json().catch(() => null);
-      workToken = data?.data?.token ?? null;
-      return workToken;
-    } catch {
-      return null;
-    }
-  };
-
-  /**
-   * This tab's worktree — its held context, expressed as a place, ON A BRANCH.
-   *
-   * Fresh: branch `session/<id>` off the current base. Existing: touched not at
-   * all — no fetch-reset-clean like a plan directory, because the dirty state
-   * is the point. If the directory was retired but the branch survives, the
-   * worktree re-attaches to the branch and the committed work is still there.
-   */
-  const sessionWtFor = (sessionId) => {
-    if (!isSafePathSegment(sessionId)) return null;
-    const wt = join(baseDir, 'sessions', sessionId);
-    const fresh = !existsSync(wt);
-    if (fresh) {
-      const branch = `session/${sessionId}`;
-      try {
-        git(['worktree', 'add', '-b', branch, wt, baseRef], repoRoot);
-      } catch {
-        git(['worktree', 'prune'], repoRoot);
-        try {
-          // The branch may already exist (a retired directory's work) — attach.
-          git(['worktree', 'add', wt, branch], repoRoot);
-        } catch {
-          try {
-            git(['worktree', 'add', '-b', branch, wt, baseRef], repoRoot);
-          } catch {
-            return null;
-          }
-        }
-      }
-    }
-    return { wt, fresh };
-  };
-
-  /**
-   * Retire the least-recently-touched CLEAN session directories past the cap.
-   * A dirty worktree is never touched — uncommitted work is the human's, and a
-   * resource bound does not outrank it. Committed work survives retirement on
-   * the session branch either way.
-   */
-  const MAX_WORK_DIRS = 12;
-  const workTouched = new Map(); // sessionId -> ms
-  const retireIdleWorkSessions = () => {
-    const dir = join(baseDir, 'sessions');
-    if (!existsSync(dir)) return;
-    let ids;
-    try {
-      ids = readdirSync(dir);
-    } catch {
-      return;
-    }
-    if (ids.length <= MAX_WORK_DIRS) return;
-    const oldestFirst = ids.sort(
-      (a, b) => (workTouched.get(a) ?? 0) - (workTouched.get(b) ?? 0)
-    );
-    let excess = ids.length - MAX_WORK_DIRS;
-    for (const id of oldestFirst) {
-      if (excess <= 0) break;
-      const wt = join(dir, id);
-      try {
-        if (git(['status', '--porcelain'], wt).trim() !== '') continue; // dirty — skip
-        git(['worktree', 'remove', wt], repoRoot);
-        workTouched.delete(id);
-        excess--;
-      } catch {
-        /* leave it; a directory we can't cleanly remove is not worth a turn */
-      }
-    }
-    try {
-      git(['worktree', 'prune'], repoRoot);
-    } catch {
-      /* best effort */
-    }
-  };
-
-  const processWorkTurns = (jobs) => {
-    for (const job of jobs ?? []) {
-      if (!job || typeof job.id !== 'string' || !job.body || !job.sessionId) continue;
-      if (workAnswering.has(job.id)) continue;
-      const tries = (workAttempts.get(job.id) ?? 0) + 1;
-      if (tries > MAX_WORK_TRIES) continue;
-      workAttempts.set(job.id, tries);
-      workAnswering.add(job.id);
-      const chain = workChains.get(job.sessionId) ?? Promise.resolve();
-      workChains.set(
-        job.sessionId,
-        chain.then(async () => {
-          try {
-            note(
-              `${c.cyan('tab')} ${c.dim(`— ${job.askedByName || 'the owner'} in "${job.sessionName || 'a session'}"`)}`
-            );
-            const workRt = pickRuntimeFor('build');
-            if (!workRt) {
-              warn('a session turn is waiting, but no installed CLI can build here');
-              return;
-            }
-            const token = await mintWorkToken();
-            if (!token) {
-              warn('a session turn is waiting, but the work credential could not be minted');
-              return;
-            }
-            const dir = sessionWtFor(job.sessionId);
-            if (!dir) {
-              warn('a session turn is waiting, but its worktree could not be opened');
-              return;
-            }
-            workTouched.set(job.sessionId, Date.now());
-            const resume = !dir.fresh && Boolean(job.sessionRef);
-            const mcp = mcpFor(workRt, token, mcpUrl);
-            let out;
-            try {
-              out = await runTurn({
-                prompt: WORK_TURN_KICKOFF({
-                  sessionId: job.sessionId,
-                  sessionName: job.sessionName,
-                  message: job.body,
-                  askedByName: job.askedByName,
-                }),
-                resume,
-                system: SYSTEM_WORK,
-                cwd: dir.wt,
-                mcpArgs: mcp.args,
-                mcpEnv: mcp.env,
-                runtime: workRt,
-                label: c.cyan('[tab]'),
-              });
-            } finally {
-              if (mcp.dir) rmSync(mcp.dir, { recursive: true, force: true });
-            }
-            const answer = (out || '').trim();
-            const posted = await reportMergeOutcome(WORK_DONE_URL, {
-              turnId: job.id,
-              ok: answer.length > 0,
-              // Scrub: a reply can quote config or env-adjacent code.
-              answer: envScrub(answer).slice(0, 16000),
-              sessionRef: dir.wt,
-            });
-            if (posted) workAttempts.delete(job.id);
-            ok(`${c.cyan('tab')} ${c.dim('— replied in the session')}`);
-            retireIdleWorkSessions();
-          } catch (e) {
-            await reportMergeOutcome(WORK_DONE_URL, {
-              turnId: job.id,
-              ok: false,
-              answer: e?.message ?? 'the session turn failed',
-            });
-            warn(`session turn failed: ${e?.message ?? e}`);
-          } finally {
-            workAnswering.delete(job.id);
-          }
-        })
-      );
-    }
-  };
-
-  // Ship — a session's branch merging to main, on the human's word.
-  //
-  // --no-ff, NEVER squash: every delivered card carries commit shas as its
-  // receipts, and a squash would point them all at commits that no longer
-  // exist on main. Sequence: refuse a dirty worktree (auto-committing someone's
-  // mid-thought state is not shipping, it is guessing), fold main INTO the
-  // branch first so conflicts surface in the worktree where the session can
-  // resolve them, collect the branch's own commits (the server's
-  // reconciliation input), then merge outward through a throwaway worktree so
-  // nobody's checkout moves. Failures report INTO the tab — a ship that failed
-  // silently leaves the human believing their work is on main.
-  const SHIP_DONE_URL = FLEET_URL.replace(/\/agents\/?$/, '/ship-done');
-  const shipping = new Set();
-  let shipChain = Promise.resolve();
-
-  const processShipJobs = (jobs) => {
-    for (const job of jobs ?? []) {
-      if (!job || typeof job.sessionId !== 'string') continue;
-      if (shipping.has(job.sessionId)) continue;
-      shipping.add(job.sessionId);
-      shipChain = shipChain.then(async () => {
-        const done = (payload) =>
-          reportMergeOutcome(SHIP_DONE_URL, { sessionId: job.sessionId, ...payload }).catch(
-            () => {}
-          );
-        try {
-          if (!isSafePathSegment(job.sessionId)) {
-            await done({ ok: false, error: 'invalid session id' });
-            return;
-          }
-          note(`${c.cyan('ship')} ${c.dim(`— "${job.sessionName || job.sessionId}"`)}`);
-          const wt = join(baseDir, 'sessions', job.sessionId);
-          if (!existsSync(wt)) {
-            await done({ ok: false, error: 'no session worktree on this machine' });
-            return;
-          }
-          if (git(['status', '--porcelain'], wt) !== '') {
-            await done({
-              ok: false,
-              error:
-                'the session has uncommitted changes — ask it to commit or discard them first',
-            });
-            return;
-          }
-          try {
-            git(['fetch', 'origin', '--quiet'], repoRoot);
-          } catch {
-            /* offline fetch — merge against what we have */
-          }
-          // Fold main into the branch FIRST: conflicts land here, in the
-          // session's own worktree, where the next turn can resolve them.
-          try {
-            git(['merge', '--no-edit', baseRef], wt);
-          } catch {
-            try {
-              git(['merge', '--abort'], wt);
-            } catch {
-              /* nothing in progress */
-            }
-            await done({
-              ok: false,
-              error: 'conflicts with main — ask the session to resolve them, then ship again',
-            });
-            return;
-          }
-          // The branch's own commits — the server's reconciliation input.
-          // --no-merges: fold-commits describe plumbing, not work.
-          const commits = git([
-            'log',
-            `${baseRef}..HEAD`,
-            '--no-merges',
-            '--format=%H%x09%s',
-          ], wt)
-            .split('\n')
-            .filter(Boolean)
-            .map((l) => {
-              const [sha, ...rest] = l.split('\t');
-              return { sha, subject: envScrub(rest.join('\t')).slice(0, 200) };
-            });
-          if (commits.length === 0) {
-            await done({ ok: false, error: 'nothing to ship — no commits on the session branch' });
-            return;
-          }
-          // Merge outward through a throwaway worktree so no checkout moves.
-          const branch = `session/${job.sessionId}`;
-          const tmp = join(baseDir, 'ship', job.sessionId);
-          try {
-            try {
-              git(['worktree', 'remove', '--force', tmp], repoRoot);
-            } catch {
-              /* not there — fine */
-            }
-            git(['worktree', 'add', '--detach', tmp, baseRef], repoRoot);
-            git([
-              'merge',
-              '--no-ff',
-              branch,
-              '-m',
-              `ship(${job.sessionName || job.sessionId.slice(0, 8)}): ${commits.length} commit${commits.length === 1 ? '' : 's'}`,
-            ], tmp);
-            git(['push', 'origin', `HEAD:${baseBranchName(baseRef)}`], tmp);
-          } finally {
-            try {
-              git(['worktree', 'remove', '--force', tmp], repoRoot);
-              git(['worktree', 'prune'], repoRoot);
-            } catch {
-              /* best effort */
-            }
-          }
-          await done({ ok: true, commits });
-          ok(`${c.cyan('ship')} ${c.dim(`— ${commits.length} commit${commits.length === 1 ? '' : 's'} on main`)}`);
-        } catch (e) {
-          await done({ ok: false, error: envScrub(e?.message ?? 'the merge failed').slice(0, 500) });
-          warn(`ship failed: ${e?.message ?? e}`);
-        } finally {
-          shipping.delete(job.sessionId);
-        }
-      });
-    }
-  };
+  // The whole machinery — per-session turn/ship chains, per-session work
+  // tokens, the settle-every-turn contract, the ship executor, worktree
+  // retirement — lives in work.mjs; this hands it the loop's mutable state.
+  const {
+    flushWorkReports,
+    processWorkTurns,
+    processShipJobs,
+    retireWorkSessions,
+    shutdownWork,
+  } = createWorkManager({
+    repoRoot,
+    baseDir,
+    baseRef,
+    getMcpUrl: () => mcpUrl,
+    getLeaseTtl: () => leaseTtlSeconds,
+  });
+  workShutdown = shutdownWork; // teardown can now reach the live session CLIs
 
   const processMergeJobs = (jobs) => {
     for (const job of jobs ?? []) {
@@ -2002,12 +1738,22 @@ export async function runFleetDaemon() {
       });
       if (updating) return;
     }
+    // Settle any turn/ship answers whose earlier report POST failed BEFORE
+    // taking new work — the skip-if-pending guards make the ordering safe, but
+    // delivering first keeps the tab honest a poll sooner.
+    void flushWorkReports();
     processMergeJobs(roster.mergeJobs);
     processPatchRevertJobs(roster.patchRevertJobs);
     processPlanCheckJobs(roster.planCheckJobs);
     processConsultJobs(roster.consultJobs);
     processWorkTurns(roster.workTurnJobs);
-    processShipJobs(roster.shipJobs);
+    // The roster's live-session list rides along: an ENDED session's ship
+    // must not be refused by checks whose remedies need a live tab.
+    processShipJobs(roster.shipJobs, roster.activeWorkSessions);
+    // AFTER the work/ship intake: retirement is the server saying which
+    // sessions are LIVE, and the guards above (chains, shipping) are populated
+    // by the intake this same tick.
+    retireWorkSessions(roster.activeWorkSessions);
     processJoinJobs(roster.joinJobs);
     processCleanupJobs(roster.cleanupJobs);
     const rosterIds = new Set(roster.agents.map((a) => a.agentId));