@cluesmith/codev 3.2.2 → 3.2.4

package/dist/agent-farm/servers/tower-routes.js

@@ -42,6 +42,9 @@ import { fetchIssue, searchIssues, fetchPRList, fetchCurrentUser, parseLinkedIss
 import { computeAnalytics } from './analytics.js';
 import { getAllTasks, executeTask, getTaskId } from './tower-cron.js';
 import { getGlobalDb } from '../db/index.js';
+import { listProcessCensus } from './process-census.js';
+import { SHELLPER_MARKER, computeRegisteredShellperPids, findHuskShellpers, sweepShellperHusks, resolveHuskGraceMs, } from './shellper-husk-sweep.js';
+import { getProcessStartTime } from '../../terminal/session-manager.js';
 import { getWorkspaceTerminals, getTerminalManager, getWorkspaceTerminalsEntry, getNextShellId, saveTerminalSession, isSessionPersistent, deleteTerminalSession, removeTerminalFromRegistry, deleteWorkspaceTerminalSessions, deleteFileTabsForWorkspace, saveFileTab, deleteFileTab, getRehydratedTerminalsEntry, isStartupReconcileSettled, getTerminalSessionById, getActiveShellLabels, updateTerminalLabel, } from './tower-terminals.js';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -65,7 +68,9 @@ export function stopSendBuffer() {
     sendBuffer.stop();
 }
 const ROUTES = {
-    'GET /health': (_req, res) => handleHealthCheck(res),
+    'GET /health': (_req, res, _url, ctx) => handleHealthCheck(res, ctx),
+    'GET /api/shellpers/husks': (_req, res, _url, ctx) => handleHuskPreview(res, ctx),
+    'POST /api/shellpers/husks/sweep': (_req, res, _url, ctx) => handleHuskSweep(res, ctx),
     'GET /api/workspaces': (_req, res) => handleListWorkspaces(res),
     'POST /api/terminals': (req, res, _url, ctx) => handleTerminalCreate(req, res, ctx),
     'GET /api/terminals': (_req, res) => handleTerminalList(res),
@@ -177,9 +182,56 @@ export async function handleRequest(req, res, ctx) {
 // ============================================================================
 // Global route handlers
 // ============================================================================
-async function handleHealthCheck(res) {
+/**
+ * Issue #1227: fleet RSS + unregistered-shellper count, scoped to this Tower
+ * instance's socketDir. `fleetRssKb` sums every in-scope shellper AND its
+ * direct children (the full process-group tree Tower manages) regardless of
+ * DB-registration state, so a husk not yet swept still counts toward the
+ * visible cost. `unregisteredShellperCount` is the lighter, ungated signal
+ * ("N shellpers Tower doesn't currently track") — distinct from the stricter
+ * childless+aged reap predicate in shellper-husk-sweep.ts.
+ *
+ * Best-effort: returns undefined fields (never throws) if `ps` or the DB read
+ * fails, so a fleet-accounting hiccup never takes down /health itself.
+ */
+async function computeFleetHealthFields(ctx) {
+    const shellperManager = ctx.getShellperManager();
+    if (!shellperManager)
+        return {};
+    try {
+        const census = await listProcessCensus();
+        const scopeMarker = shellperManager.socketDir.endsWith('/')
+            ? shellperManager.socketDir
+            : `${shellperManager.socketDir}/`;
+        const inScopePids = new Set();
+        let fleetRssKb = 0;
+        for (const entry of census) {
+            if (entry.cmdline.includes(SHELLPER_MARKER) && entry.cmdline.includes(scopeMarker)) {
+                inScopePids.add(entry.pid);
+                fleetRssKb += entry.rssKb;
+            }
+        }
+        for (const entry of census) {
+            if (inScopePids.has(entry.ppid)) {
+                fleetRssKb += entry.rssKb;
+            }
+        }
+        const registered = await computeRegisteredShellperPids(getGlobalDb());
+        let unregisteredShellperCount = 0;
+        for (const pid of inScopePids) {
+            if (!registered.has(pid))
+                unregisteredShellperCount++;
+        }
+        return { fleetRssKb, unregisteredShellperCount };
+    }
+    catch {
+        return {};
+    }
+}
+async function handleHealthCheck(res, ctx) {
     const instances = await getInstances();
     const activeCount = instances.filter((i) => i.running).length;
+    const fleetFields = await computeFleetHealthFields(ctx);
     res.writeHead(200, { 'Content-Type': 'application/json' });
     res.end(JSON.stringify({
         status: 'healthy',
@@ -191,9 +243,82 @@ async function handleHealthCheck(res) {
         activeWorkspaces: activeCount,
         totalWorkspaces: instances.length,
         memoryUsage: process.memoryUsage().heapUsed,
+        ...fleetFields,
         timestamp: new Date().toISOString(),
     }));
 }
+/**
+ * Issue #1227: GET /api/shellpers/husks — preview, read-only. Returns the
+ * husk candidates the next sweep (periodic or `--apply`) would reap, without
+ * touching anything.
+ */
+async function handleHuskPreview(res, ctx) {
+    const shellperManager = ctx.getShellperManager();
+    if (!shellperManager) {
+        res.writeHead(503, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify({ error: 'Shellper manager not initialized' }));
+        return;
+    }
+    try {
+        const graceMs = resolveHuskGraceMs();
+        // One census snapshot, shared between the candidate decision and the
+        // displayed RSS — passing it via the `census` seam avoids a second `ps`
+        // scan and guarantees the RSS shown is for the exact snapshot that decided
+        // candidacy, not a later, possibly-different one (codex #1227 PR review).
+        const census = await listProcessCensus();
+        const rssByPid = new Map(census.map((entry) => [entry.pid, entry.rssKb]));
+        const registered = await computeRegisteredShellperPids(getGlobalDb());
+        const pids = await findHuskShellpers({
+            socketDir: shellperManager.socketDir,
+            registeredShellperPids: registered,
+            graceMs,
+            census: () => census,
+        });
+        const now = Date.now();
+        const candidates = await Promise.all(pids.map(async (pid) => {
+            const startTime = await getProcessStartTime(pid);
+            return {
+                pid,
+                rssKb: rssByPid.get(pid) ?? 0,
+                ageMs: startTime !== null ? now - startTime : null,
+            };
+        }));
+        res.writeHead(200, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify({ candidates, graceMs }));
+    }
+    catch (err) {
+        res.writeHead(500, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify({ error: err.message }));
+    }
+}
+/**
+ * Issue #1227: POST /api/shellpers/husks/sweep — apply. Actually reaps the
+ * current husk candidates. Destructive; the `afx tower sweep-husks` CLI is
+ * the only caller expected in practice, and it gates this behind `--apply`
+ * plus a confirmation prompt (unless `--yes`).
+ */
+async function handleHuskSweep(res, ctx) {
+    const shellperManager = ctx.getShellperManager();
+    if (!shellperManager) {
+        res.writeHead(503, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify({ error: 'Shellper manager not initialized' }));
+        return;
+    }
+    try {
+        const result = await sweepShellperHusks({
+            socketDir: shellperManager.socketDir,
+            db: getGlobalDb(),
+            graceMs: resolveHuskGraceMs(),
+            log: (msg) => ctx.log('INFO', msg),
+        });
+        res.writeHead(200, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify(result));
+    }
+    catch (err) {
+        res.writeHead(500, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify({ error: err.message }));
+    }
+}
 /**
  * GET /api/version (#983) — report the version of the *running* Tower process.
  *
@@ -448,8 +573,11 @@ async function handleTerminalCreate(req, res, ctx) {
                     ...defaultSessionOptions(),
                     cols: cols || DEFAULT_COLS,
                 });
-                const replayData = client.getReplayData() ?? Buffer.alloc(0);
+                // Read session info BEFORE awaiting replay: an instantly-exiting
+                // child's EXIT frame can remove the session from the manager during
+                // the await, and this lookup must not miss (#1198).
                 const shellperInfo = shellperManager.getSessionInfo(sessionId);
+                const replayData = await client.waitForReplay(); // #1198: fresh shellpers always send REPLAY (possibly empty); awaiting avoids racing early child output
                 const session = manager.createSessionRaw({
                     label: label || `terminal-${sessionId.slice(0, 8)}`,
                     cwd,
@@ -1083,6 +1211,18 @@ async function handleSend(req, res, ctx) {
         }));
         return;
     }
+    // #1198: a session whose shellper connection died still reports status
+    // 'running', but every write to it is dropped. Fail the send loudly
+    // instead of logging "Message sent" for a message that went nowhere.
+    if (!session.writable) {
+        ctx.log('ERROR', `Message DROPPED: ${from ?? 'unknown'} → ${result.agent} (terminal ${result.terminalId.slice(0, 8)}...): terminal not writable (shellper connection down)`);
+        res.writeHead(503, { 'Content-Type': 'application/json' });
+        res.end(JSON.stringify({
+            error: 'TERMINAL_NOT_WRITABLE',
+            message: `Terminal for '${result.agent}' is not accepting input (its process connection is down). Retry shortly; if this persists, check Tower logs.`,
+        }));
+        return;
+    }
     // Format the message based on sender/target
     const isArchitectTarget = result.agent === 'architect';
     let formattedMessage;
@@ -1706,8 +1846,11 @@ async function handleWorkspaceShellCreate(res, ctx, workspacePath) {
                     env: shellEnv,
                     ...defaultSessionOptions(),
                 });
-                const replayData = client.getReplayData() ?? Buffer.alloc(0);
+                // Read session info BEFORE awaiting replay: an instantly-exiting
+                // child's EXIT frame can remove the session from the manager during
+                // the await, and this lookup must not miss (#1198).
                 const shellperInfo = shellperManager.getSessionInfo(sessionId);
+                const replayData = await client.waitForReplay(); // #1198: fresh shellpers always send REPLAY (possibly empty); awaiting avoids racing early child output
                 const session = manager.createSessionRaw({
                     label: `Shell ${shellId.replace('shell-', '')}`,
                     cwd: workspacePath,