@worca/ui 0.21.0 → 0.23.0

package/server/worktrees-routes.js

@@ -16,7 +16,7 @@
  * the discrepancy with `du -sh`.
  */
 
-import { existsSync, readdirSync, readFileSync } from 'node:fs';
+import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
 import * as fsp from 'node:fs/promises';
 import { join } from 'node:path';
 import { Router } from 'express';
@@ -192,11 +192,95 @@ function _readWorktreeStatus(worktreePath) {
   return null;
 }
 
+/**
+ * Run a pre-validated cleanup batch in the background. Each task stamps
+ * `cleanup_state: 'cleaning'`, calls `removeWorktree` (which deletes the
+ * registry entry on success), and on failure stamps `cleanup_error` while
+ * clearing `cleanup_state` so the UI can render the error and let the user
+ * retry. Concurrency is bounded by `CLEANUP_CONCURRENCY`.
+ */
+async function _runCleanupBatch(worcaDir, accepted) {
+  const tasks = accepted.map(({ run_id, reg }) => ({
+    run_id,
+    fn: async () => {
+      _patchRegistry(worcaDir, run_id, { cleanup_state: 'cleaning' });
+      try {
+        await removeWorktree(worcaDir, run_id, { skipPrune: true });
+        if (reg.worktree_path) _diskCache.delete(reg.worktree_path);
+        return { run_id, ok: true };
+      } catch (err) {
+        _patchRegistry(worcaDir, run_id, {
+          cleanup_state: undefined,
+          cleanup_error: err?.message || String(err),
+        });
+        return { run_id, ok: false, error: err?.message || String(err) };
+      }
+    },
+  }));
+
+  try {
+    await runWithConcurrencyLimit(tasks, CLEANUP_CONCURRENCY);
+  } catch {
+    /* per-task failures already persisted into the registry */
+  }
+
+  try {
+    await pruneWorktrees(worcaDir);
+  } catch {
+    /* non-fatal */
+  }
+}
+
+/**
+ * Atomically patch fields on a pipelines.d/<run>.json entry.
+ * Set a field to `undefined` to delete it. Returns `false` if the file is
+ * gone (the worktree was already cleaned up) or unreadable.
+ *
+ * Note: write is not strictly atomic — for a single-writer-per-id model
+ * (the cleanup background task owns its registry entry for the lifetime
+ * of the cleanup), read-modify-write is fine. A multi-writer scenario
+ * would need rename-into-place; we don't have that here.
+ */
+function _patchRegistry(worcaDir, runId, patch) {
+  const regFile = join(worcaDir, 'multi', 'pipelines.d', `${runId}.json`);
+  if (!existsSync(regFile)) return false;
+  let reg;
+  try {
+    reg = JSON.parse(readFileSync(regFile, 'utf8'));
+  } catch {
+    return false;
+  }
+  for (const [k, v] of Object.entries(patch)) {
+    if (v === undefined) delete reg[k];
+    else reg[k] = v;
+  }
+  try {
+    writeFileSync(regFile, JSON.stringify(reg, null, 2), 'utf8');
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function _isPidAlive(pid) {
+  if (!pid || typeof pid !== 'number') return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    if (err.code === 'EPERM') return true;
+    return false;
+  }
+}
+
 async function _listWorktrees(worcaDir) {
   const pipelinesDir = join(worcaDir, 'multi', 'pipelines.d');
   if (!existsSync(pipelinesDir)) return [];
 
-  const entries = [];
+  // Phase 1: cheap synchronous metadata (registry parse, status read).
+  // Phase 2: disk walks in parallel — without this, 13 worktrees serialize
+  // ~3s of awaits even when most results would have been disk-cache hits.
+  const metas = [];
   for (const file of readdirSync(pipelinesDir)) {
     if (!file.endsWith('.json')) continue;
 
@@ -211,13 +295,35 @@ async function _listWorktrees(worcaDir) {
     const worktreePath = reg.worktree_path;
     const worktreeExists = existsSync(worktreePath);
 
-    // Prefer actual status.json; fall back to registry field
     let status = reg.status || 'unknown';
     if (worktreeExists) {
       const actual = _readWorktreeStatus(worktreePath);
       if (actual) status = actual;
     }
 
+    // Stale-registry reconciliation: a child can die before ever writing
+    // status.json (e.g. fleet halt right after dispatch, preflight crash,
+    // SIGKILL). In that case the worktree exists but .worca/runs/ doesn't,
+    // _readWorktreeStatus returns null, and we'd fall back to reg.status
+    // which may still say "running" with a dead pid. Treat that as
+    // "interrupted" and patch the registry so this only happens once.
+    //
+    // Only reconcile when reg.pid is present — a missing pid means the
+    // entry is either from a non-standard registration path (e.g. test
+    // fixtures) or pre-dates the pid-on-registration contract, so we
+    // can't make liveness claims about it.
+    if (
+      status === 'running' &&
+      typeof reg.pid === 'number' &&
+      !_isPidAlive(reg.pid)
+    ) {
+      status = 'interrupted';
+      _patchRegistry(worcaDir, reg.run_id, {
+        status: 'interrupted',
+        interrupted_reason: 'stale_pid',
+      });
+    }
+
     let ageSeconds = 0;
     if (reg.started_at) {
       const started = new Date(reg.started_at).getTime();
@@ -226,32 +332,44 @@ async function _listWorktrees(worcaDir) {
       }
     }
 
-    let diskInfo = { bytes: 0, truncated: false };
-    if (worktreeExists) {
-      diskInfo = await _getDiskBytes(worktreePath);
-    }
-
-    entries.push({
-      run_id: reg.run_id || '',
-      title: reg.title || '',
-      branch: reg.branch || '',
-      worktree_path: worktreePath,
-      disk_bytes: diskInfo.bytes,
-      truncated: diskInfo.truncated,
-      age_seconds: ageSeconds,
-      // started_at lets the client sort with the same sortByStartDesc helper
-      // used by run-list, keeping ordering consistent across views.
-      started_at: reg.started_at || null,
+    metas.push({
+      reg,
+      worktreePath,
+      worktreeExists,
       status,
-      removable: status !== 'running',
-      fleet_id: reg.fleet_id || null,
-      workspace_id: reg.workspace_id || null,
-      group_type: reg.group_type || null,
-      group_status: null, // populated by W-040 / W-047
-      resumable: RESUMABLE_STATUSES.has(status),
+      ageSeconds,
+      cleanup_state: reg.cleanup_state || null,
+      cleanup_error: reg.cleanup_error || null,
     });
   }
-  return entries;
+
+  const disks = await Promise.all(
+    metas.map((m) =>
+      m.worktreeExists
+        ? _getDiskBytes(m.worktreePath)
+        : Promise.resolve({ bytes: 0, truncated: false }),
+    ),
+  );
+
+  return metas.map((m, i) => ({
+    run_id: m.reg.run_id || '',
+    title: m.reg.title || '',
+    branch: m.reg.branch || '',
+    worktree_path: m.worktreePath,
+    disk_bytes: disks[i].bytes,
+    truncated: disks[i].truncated,
+    age_seconds: m.ageSeconds,
+    started_at: m.reg.started_at || null,
+    status: m.status,
+    removable: m.status !== 'running',
+    fleet_id: m.reg.fleet_id || null,
+    workspace_id: m.reg.workspace_id || null,
+    group_type: m.reg.group_type || null,
+    group_status: null,
+    resumable: RESUMABLE_STATUSES.has(m.status),
+    cleanup_state: m.cleanup_state,
+    cleanup_error: m.cleanup_error,
+  }));
 }
 
 const RUN_ID_RE = /^[a-zA-Z0-9_-]+$/;
@@ -367,13 +485,19 @@ export function createWorktreesRouter() {
 
   // POST /worktrees/cleanup
   //
-  // Batch worktree removal. Always responds with HTTP 200 and a JSON body of
-  // shape `{ ok, results, failed_count }`, where `ok` is the AND of per-id
-  // outcomes and `failed_count` is the number of entries with `ok: false`.
-  // Per-entry errors carry a `code` field (`running`, `resumable_or_grouped`)
-  // when actionable. Clients must inspect `results[]` — a single bad id never
-  // aborts the batch, and partial failures are not signalled via HTTP status.
-  router.post('/cleanup', async (req, res) => {
+  // Batch worktree removal — async. Synchronously validates each id and
+  // stamps `cleanup_state: 'pending'` on the registry entries that pass
+  // pre-flight checks, then returns 202. The actual removal happens in
+  // the background with bounded concurrency. Clients poll GET /worktrees
+  // and observe `cleanup_state` per entry; on success the entry vanishes,
+  // on failure `cleanup_error` is set and `cleanup_state` is cleared.
+  //
+  // Response shape `{ ok, accepted, rejected }` where `rejected[]` carries
+  // entries that failed pre-flight (running, resumable without force, etc).
+  // A single bad id never blocks the rest of the batch; this stays
+  // compatible with the legacy synchronous shape's promise that partial
+  // failures are not signalled via HTTP status.
+  router.post('/cleanup', (req, res) => {
     const worcaDir = req.project?.worcaDir;
     if (!worcaDir) {
       return res
@@ -395,86 +519,82 @@ export function createWorktreesRouter() {
       }
     }
 
-    const tasks = run_ids.map((run_id) => ({
-      run_id,
-      fn: async () => {
-        const regFile = join(
-          worcaDir,
-          'multi',
-          'pipelines.d',
-          `${run_id}.json`,
-        );
-        if (!existsSync(regFile)) {
-          return {
-            run_id,
-            ok: false,
-            error: `Worktree "${run_id}" not found`,
-          };
-        }
-
-        let reg;
-        try {
-          reg = JSON.parse(readFileSync(regFile, 'utf8'));
-        } catch {
-          return {
-            run_id,
-            ok: false,
-            error: 'Failed to read registry entry',
-          };
-        }
-
-        let status = reg.status || 'unknown';
-        if (reg.worktree_path && existsSync(reg.worktree_path)) {
-          const actual = _readWorktreeStatus(reg.worktree_path);
-          if (actual) status = actual;
-        }
+    // Pre-flight: read each registry entry, decide pending vs reject. We do
+    // this synchronously so the HTTP response can carry the rejection list
+    // — clients shouldn't have to poll to learn that a 'running' worktree
+    // was refused.
+    const accepted = [];
+    const rejected = [];
+    for (const run_id of run_ids) {
+      const regFile = join(worcaDir, 'multi', 'pipelines.d', `${run_id}.json`);
+      if (!existsSync(regFile)) {
+        rejected.push({
+          run_id,
+          ok: false,
+          error: `Worktree "${run_id}" not found`,
+        });
+        continue;
+      }
+      let reg;
+      try {
+        reg = JSON.parse(readFileSync(regFile, 'utf8'));
+      } catch {
+        rejected.push({
+          run_id,
+          ok: false,
+          error: 'Failed to read registry entry',
+        });
+        continue;
+      }
 
-        if (status === 'running') {
-          return {
-            run_id,
-            ok: false,
-            error: 'Cannot remove a running worktree',
-            code: 'running',
-          };
-        }
+      let status = reg.status || 'unknown';
+      if (reg.worktree_path && existsSync(reg.worktree_path)) {
+        const actual = _readWorktreeStatus(reg.worktree_path);
+        if (actual) status = actual;
+      }
 
-        const isResumable = RESUMABLE_STATUSES.has(status);
-        const isGrouped = !!(reg.fleet_id || reg.workspace_id);
-        if (!force && (isResumable || isGrouped)) {
-          return {
-            run_id,
-            ok: false,
-            error:
-              'Removing this worktree prevents resuming the run. Pass force=true to confirm.',
-            code: 'resumable_or_grouped',
-          };
-        }
+      if (status === 'running') {
+        rejected.push({
+          run_id,
+          ok: false,
+          error: 'Cannot remove a running worktree',
+          code: 'running',
+        });
+        continue;
+      }
 
-        try {
-          await removeWorktree(worcaDir, run_id, { skipPrune: true });
-          if (reg.worktree_path) _diskCache.delete(reg.worktree_path);
-          return { run_id, ok: true };
-        } catch (err) {
-          return { run_id, ok: false, error: err.message };
-        }
-      },
-    }));
+      const isResumable = RESUMABLE_STATUSES.has(status);
+      const isGrouped = !!(reg.fleet_id || reg.workspace_id);
+      if (!force && (isResumable || isGrouped)) {
+        rejected.push({
+          run_id,
+          ok: false,
+          error:
+            'Removing this worktree prevents resuming the run. Pass force=true to confirm.',
+          code: 'resumable_or_grouped',
+        });
+        continue;
+      }
 
-    let results;
-    try {
-      results = await runWithConcurrencyLimit(tasks, CLEANUP_CONCURRENCY);
-    } catch (err) {
-      return res.status(500).json({ ok: false, error: err.message });
+      // Stamp pending so a reload mid-cleanup shows the same state.
+      _patchRegistry(worcaDir, run_id, {
+        cleanup_state: 'pending',
+        cleanup_error: undefined,
+      });
+      accepted.push({ run_id, reg });
     }
 
-    try {
-      await pruneWorktrees(worcaDir);
-    } catch {
-      /* non-fatal */
-    }
+    // Respond immediately — the client polls GET /worktrees to observe progress.
+    res.status(202).json({
+      ok: rejected.length === 0,
+      accepted: accepted.map((a) => a.run_id),
+      rejected,
+    });
 
-    const failed_count = results.reduce((n, r) => (r.ok ? n : n + 1), 0);
-    res.json({ ok: failed_count === 0, failed_count, results });
+    // Fire-and-forget background removal. Errors are persisted into the
+    // registry so the client can render them; nothing here is awaited by
+    // the HTTP request.
+    void _runCleanupBatch(worcaDir, accepted);
   });
 
   return router;

package/server/ws-fleet-manifest-watcher.js

@@ -0,0 +1,130 @@
+/**
+ * Fleet manifest watcher — monitors ~/.worca/fleet-runs/<fleet_id>.json for changes.
+ * Emits fleet-update WS events when a fleet manifest is written (§13.5).
+ */
+
+import { existsSync, readFileSync, watch } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { effectiveFleetStatus } from './fleet-routes.js';
+
+const FLEET_DEBOUNCE_MS = 200;
+const DEFAULT_FLEET_RUNS_DIR = join(homedir(), '.worca', 'fleet-runs');
+
+const FAILURE_STATES = new Set(['failed', 'setup_failed', 'unrecoverable']);
+
+function readJson(path) {
+  try {
+    return JSON.parse(readFileSync(path, 'utf8'));
+  } catch {
+    return null;
+  }
+}
+
+function resolveChildStatus(child) {
+  const { project_path, run_id } = child;
+  if (!project_path || !run_id) return 'running';
+  const registryPath = join(
+    project_path,
+    '.worca',
+    'multi',
+    'pipelines.d',
+    `${run_id}.json`,
+  );
+  const entry = readJson(registryPath);
+  return entry?.status ?? 'running';
+}
+
+/**
+ * @param {{ broadcaster: { broadcast: Function }, fleetRunsDir?: string }} deps
+ */
+export function createFleetManifestWatcher({
+  broadcaster,
+  fleetRunsDir = DEFAULT_FLEET_RUNS_DIR,
+}) {
+  let fsWatcher = null;
+  /** @type {Map<string, ReturnType<typeof setTimeout>>} */
+  const debounceTimers = new Map();
+
+  function broadcastFleetUpdate(fleetId, manifestPath) {
+    const manifest = readJson(manifestPath);
+    if (!manifest) return;
+
+    const rawChildren = Array.isArray(manifest.children)
+      ? manifest.children
+      : [];
+    const children = rawChildren.map((child) => ({
+      run_id: child.run_id,
+      project_path: child.project_path,
+      status: resolveChildStatus(child),
+    }));
+
+    const completed_children = children.filter(
+      (c) => c.status === 'completed',
+    ).length;
+    const failed_children = children.filter((c) =>
+      FAILURE_STATES.has(c.status),
+    ).length;
+
+    // Derive the effective status (same rules as REST) instead of broadcasting
+    // raw manifest.status — otherwise cards stay "running" forever, because
+    // run_fleet.py never writes a terminal status after it exits. Pure
+    // function: persists nothing, so we don't trigger a watch→write→watch loop.
+    const { status, halt_reason } = effectiveFleetStatus(
+      manifest,
+      children.map((c) => c.status),
+    );
+
+    broadcaster.broadcast('fleet-update', {
+      fleet_id: fleetId,
+      status,
+      halt_reason,
+      completed_children,
+      failed_children,
+      children,
+    });
+  }
+
+  function scheduleUpdate(fleetId, manifestPath) {
+    const existing = debounceTimers.get(fleetId);
+    if (existing) clearTimeout(existing);
+    debounceTimers.set(
+      fleetId,
+      setTimeout(() => {
+        debounceTimers.delete(fleetId);
+        broadcastFleetUpdate(fleetId, manifestPath);
+      }, FLEET_DEBOUNCE_MS),
+    );
+  }
+
+  try {
+    if (existsSync(fleetRunsDir)) {
+      fsWatcher = watch(
+        fleetRunsDir,
+        { persistent: false },
+        (_event, filename) => {
+          if (!filename?.endsWith('.json')) return;
+          const fleetId = filename.slice(0, -5);
+          scheduleUpdate(fleetId, join(fleetRunsDir, filename));
+        },
+      );
+    }
+  } catch {
+    // fs.watch unsupported or dir unavailable — skip silently
+  }
+
+  function destroy() {
+    if (fsWatcher) {
+      try {
+        fsWatcher.close();
+      } catch {
+        /* ignore */
+      }
+      fsWatcher = null;
+    }
+    for (const timer of debounceTimers.values()) clearTimeout(timer);
+    debounceTimers.clear();
+  }
+
+  return { destroy };
+}

package/server/ws-message-router.js

@@ -80,6 +80,22 @@ export function createMessageRouter({
     };
   }
 
+  // When a run-scoped subscribe arrives with a `payload.projectId` that
+  // differs from the client's currently-bound subs.projectId, re-bind the
+  // WS to that project. Without this, the targeted project's WatcherSet
+  // stays in POLLING tier (no logWatcher / no live updates), and the
+  // backfill / live stream silently never arrive — symptoms reported by
+  // the user as "no logs / no agent prompts" on the run-detail page in
+  // global mode. We keep the previous projectId reference so demotion
+  // still happens via the normal client-count mechanism.
+  function _adoptProjectFromPayload(ws, payload) {
+    const requested = payload?.projectId;
+    if (!requested || !watcherSets.has(requested)) return;
+    const subs = clientManager.getSubs(ws);
+    if (subs?.projectId === requested) return;
+    clientManager.setProtocol(ws, subs?.protocolVersion ?? 1, requested);
+  }
+
   async function handleMessage(ws, data) {
     let json;
     try {
@@ -147,6 +163,7 @@ export function createMessageRouter({
         );
         return;
       }
+      _adoptProjectFromPayload(ws, req.payload);
       const proj = resolveProject(ws, req.payload);
       const runs = discoverRuns(proj.worcaDir);
       const run = runs.find((r) => r.id === runId);
@@ -294,6 +311,7 @@ export function createMessageRouter({
         );
         return;
       }
+      _adoptProjectFromPayload(ws, req.payload);
       const proj = resolveProject(ws, req.payload);
       if (!proj) {
         ws.send(
@@ -336,6 +354,7 @@ export function createMessageRouter({
     // subscribe-log
     if (req.type === 'subscribe-log') {
       const { stage, runId, iteration } = req.payload || {};
+      _adoptProjectFromPayload(ws, req.payload);
       const proj = resolveProject(ws, req.payload);
       const s = clientManager.ensureSubs(ws);
       s.logStage = stage || '*';
@@ -652,6 +671,7 @@ export function createMessageRouter({
         );
         return;
       }
+      _adoptProjectFromPayload(ws, req.payload);
       const proj = resolveProject(ws, req.payload);
       if (!proj.wset.beadsWatcher) {
         ws.send(JSON.stringify(makeOk(req, { issues: [], runId })));

package/server/ws-modular.js

@@ -7,6 +7,7 @@
  */
 
 import { existsSync, watch } from 'node:fs';
+import { homedir } from 'node:os';
 import { join } from 'node:path';
 import { WebSocketServer } from 'ws';
 import { readProjects, synthesizeDefaultProject } from './project-registry.js';
@@ -14,6 +15,7 @@ import { TIER_FULL, TIER_POLLING, WatcherSet } from './watcher-set.js';
 import { readProjectWorcaVersion } from './worca-setup.js';
 import { createBroadcaster } from './ws-broadcaster.js';
 import { createClientManager } from './ws-client-manager.js';
+import { createFleetManifestWatcher } from './ws-fleet-manifest-watcher.js';
 import { createMessageRouter } from './ws-message-router.js';
 import { resolveLatestRunDir } from './ws-status-watcher.js';
 
@@ -45,7 +47,13 @@ export function attachWsServer(httpServer, config) {
     getSubs: clientManager.getSubs,
   });
 
-  // 3. Create WatcherSet(s) — one per project
+  // 3a. Fleet manifest watcher — global, not per-project (§13.5)
+  const fleetManifestWatcher = createFleetManifestWatcher({
+    broadcaster,
+    fleetRunsDir: join(homedir(), '.worca', 'fleet-runs'),
+  });
+
+  // 3b. Create WatcherSet(s) — one per project
   /** @type {Map<string, WatcherSet>} */
   const watcherSets = new Map();
 
@@ -269,6 +277,7 @@ export function attachWsServer(httpServer, config) {
 
   wss.on('close', () => {
     clientManager.destroy();
+    fleetManifestWatcher.destroy();
     if (dirWatcher) {
       try {
         dirWatcher.close();