@lumenflow/cli 2.2.2 → 2.3.1

package/dist/orchestrate-monitor.js

@@ -1,20 +1,109 @@
 #!/usr/bin/env node
+/* eslint-disable no-console -- CLI tool requires console output */
 /**
- * Orchestrate Monitor CLI
+ * Orchestrate Monitor CLI (WU-1241)
  *
- * Monitors spawned agent progress using mem:inbox signals.
- * Designed to prevent context exhaustion by using compact signal output.
+ * Monitors spawned agent progress and spawn health.
+ * Wires CLI to spawn-monitor APIs in @lumenflow/core.
+ *
+ * Features:
+ * - Stuck detection: identifies pending spawns older than threshold
+ * - Zombie lock detection: identifies abandoned WU locks (dead PIDs)
+ * - Recovery actions: signal agent, restart spawn, escalate to human
+ * - Status reporting: active spawns, stuck spawns, zombie locks, suggestions
  *
  * Usage:
- *   pnpm orchestrate:monitor --since 30m
+ *   pnpm orchestrate:monitor                    # Show spawn status
+ *   pnpm orchestrate:monitor --threshold 15    # Custom threshold (15 min)
+ *   pnpm orchestrate:monitor --recover          # Run recovery actions
+ *   pnpm orchestrate:monitor --recover --dry-run # Show what would be done
+ *   pnpm orchestrate:monitor --since 30m        # Show signals since time
  */
 import { Command } from 'commander';
 import { existsSync, readdirSync, readFileSync } from 'node:fs';
 import { join } from 'node:path';
-import { EXIT_CODES, LUMENFLOW_PATHS } from '@lumenflow/core/dist/wu-constants.js';
+import { EXIT_CODES, LUMENFLOW_PATHS, SpawnRegistryStore, analyzeSpawns, detectStuckSpawns, checkZombieLocks, generateSuggestions, formatMonitorOutput, formatRecoveryResults, runRecovery, processSpawnFailureSignals, formatSignalHandlerOutput, DEFAULT_THRESHOLD_MINUTES, calculateBackoff, } from '@lumenflow/core';
 import chalk from 'chalk';
 import ms from 'ms';
+// ============================================================================
+// WU-1242: Watch Mode Constants
+// ============================================================================
+/**
+ * Default watch interval (5 minutes in milliseconds)
+ */
+export const DEFAULT_WATCH_INTERVAL_MS = 5 * 60 * 1000;
+/**
+ * Minimum watch interval (1 minute in milliseconds)
+ */
+export const MIN_WATCH_INTERVAL_MS = 60 * 1000;
+/**
+ * Maximum backoff interval (1 hour in milliseconds)
+ */
+export const MAX_BACKOFF_MS = 60 * 60 * 1000;
 const LOG_PREFIX = '[orchestrate:monitor]';
+/**
+ * Runs the spawn monitor.
+ *
+ * @param options - Monitor options
+ * @returns MonitorResult with analysis data
+ */
+export async function runMonitor(options = {}) {
+    const { baseDir = process.cwd(), thresholdMinutes = DEFAULT_THRESHOLD_MINUTES, recover = false, dryRun = false, } = options;
+    // Load spawn registry
+    // WU-1278: Use full LUMENFLOW_PATHS.STATE_DIR without stripping .lumenflow/ prefix
+    const stateDir = join(baseDir, LUMENFLOW_PATHS.STATE_DIR);
+    const store = new SpawnRegistryStore(stateDir);
+    let spawns = [];
+    try {
+        await store.load();
+        spawns = store.getAllSpawns();
+    }
+    catch {
+        // Registry doesn't exist or is invalid - continue with empty spawns
+    }
+    // Run core analysis
+    const analysis = analyzeSpawns(spawns);
+    const stuckSpawns = detectStuckSpawns(spawns, thresholdMinutes);
+    const zombieLocks = await checkZombieLocks({ baseDir });
+    const suggestions = generateSuggestions(stuckSpawns, zombieLocks);
+    const result = {
+        analysis,
+        stuckSpawns,
+        zombieLocks,
+        suggestions,
+        dryRun,
+    };
+    // Run recovery if requested
+    if (recover) {
+        const recoveryResults = await runRecovery(stuckSpawns, { baseDir, dryRun });
+        result.recoveryResults = recoveryResults;
+    }
+    return result;
+}
+/**
+ * Formats monitor result for display.
+ *
+ * @param result - Monitor result to format
+ * @returns Formatted output string
+ */
+export function formatOutput(result) {
+    const lines = [];
+    // Use core formatMonitorOutput for base formatting
+    const baseOutput = formatMonitorOutput(result);
+    lines.push(baseOutput);
+    // Add recovery results if present
+    if (result.recoveryResults && result.recoveryResults.length > 0) {
+        lines.push('');
+        lines.push(formatRecoveryResults(result.recoveryResults));
+    }
+    // Add dry-run notice if applicable
+    if (result.dryRun) {
+        lines.push('');
+        lines.push(chalk.yellow('=== DRY RUN MODE ==='));
+        lines.push(chalk.yellow('No actions were taken. Remove --dry-run to execute.'));
+    }
+    return lines.join('\n');
+}
 function parseTimeString(timeStr) {
     const msValue = ms(timeStr);
     if (typeof msValue === 'number') {
@@ -26,14 +115,16 @@ function parseTimeString(timeStr) {
     }
     return date;
 }
-function loadRecentSignals(since) {
+function loadRecentSignals(since, baseDir = process.cwd()) {
     const signals = [];
-    if (!existsSync(LUMENFLOW_PATHS.MEMORY_DIR)) {
+    // WU-1278: Use full LUMENFLOW_PATHS.MEMORY_DIR without stripping .lumenflow/ prefix
+    const memoryDir = join(baseDir, LUMENFLOW_PATHS.MEMORY_DIR);
+    if (!existsSync(memoryDir)) {
         return signals;
     }
-    const files = readdirSync(LUMENFLOW_PATHS.MEMORY_DIR).filter((f) => f.endsWith('.ndjson'));
+    const files = readdirSync(memoryDir).filter((f) => f.endsWith('.ndjson'));
     for (const file of files) {
-        const filePath = join(LUMENFLOW_PATHS.MEMORY_DIR, file);
+        const filePath = join(memoryDir, file);
         const content = readFileSync(filePath, 'utf-8');
         const lines = content.trim().split('\n').filter(Boolean);
         for (const line of lines) {
@@ -51,38 +142,295 @@ function loadRecentSignals(since) {
     }
     return signals.sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime());
 }
+/**
+ * Format signal type with appropriate color
+ */
+function formatSignalType(signalType) {
+    if (signalType === 'complete') {
+        return chalk.green(signalType);
+    }
+    if (signalType === 'error') {
+        return chalk.red(signalType);
+    }
+    return chalk.yellow(signalType);
+}
+/**
+ * Display signals in --signals-only mode
+ */
+async function displaySignals(opts) {
+    const baseDir = process.cwd();
+    const since = parseTimeString(opts.since);
+    console.log(chalk.cyan(`${LOG_PREFIX} Loading signals since ${since.toISOString()}...`));
+    const signals = loadRecentSignals(since, baseDir);
+    if (signals.length === 0) {
+        console.log(chalk.yellow(`${LOG_PREFIX} No signals found.`));
+        console.log(chalk.gray('Agents may still be starting up, or memory layer not initialized.'));
+        return;
+    }
+    const filtered = opts.wu ? signals.filter((s) => s.wuId === opts.wu) : signals;
+    console.log(chalk.bold(`\nRecent Signals (${filtered.length}):\n`));
+    for (const signal of filtered) {
+        const time = new Date(signal.timestamp).toLocaleTimeString();
+        const wu = signal.wuId ? chalk.cyan(signal.wuId) : chalk.gray('system');
+        const type = formatSignalType(signal.type);
+        console.log(`  ${chalk.gray(time)} [${wu}] ${type}: ${signal.message || ''}`);
+    }
+    console.log('');
+    console.log(chalk.gray(`Use: pnpm mem:inbox --since ${opts.since} for more details`));
+}
+/**
+ * Run main spawn monitoring mode
+ */
+async function runSpawnMonitoring(opts) {
+    const baseDir = process.cwd();
+    const thresholdMinutes = parseInt(opts.threshold, 10);
+    if (isNaN(thresholdMinutes) || thresholdMinutes <= 0) {
+        console.error(chalk.red(`${LOG_PREFIX} Invalid threshold: ${opts.threshold}`));
+        process.exit(EXIT_CODES.FAILURE);
+    }
+    console.log(chalk.cyan(`${LOG_PREFIX} Analyzing spawn health...`));
+    console.log(chalk.gray(`  Threshold: ${thresholdMinutes} minutes`));
+    console.log(chalk.gray(`  Recovery: ${opts.recover ? 'enabled' : 'disabled'}`));
+    console.log(chalk.gray(`  Dry-run: ${opts.dryRun ? 'yes' : 'no'}`));
+    console.log('');
+    const result = await runMonitor({
+        baseDir,
+        thresholdMinutes,
+        recover: opts.recover,
+        dryRun: opts.dryRun,
+    });
+    console.log(formatOutput(result));
+    if (opts.recover) {
+        console.log('');
+        console.log(chalk.cyan(`${LOG_PREFIX} Processing spawn failure signals...`));
+        const signalResult = await processSpawnFailureSignals({ baseDir, dryRun: opts.dryRun });
+        console.log(formatSignalHandlerOutput(signalResult));
+    }
+    if (result.stuckSpawns.length > 0 || result.zombieLocks.length > 0) {
+        process.exit(EXIT_CODES.ERROR);
+    }
+}
+/**
+ * Parses CLI arguments into watch mode options.
+ *
+ * @param opts - Raw CLI options
+ * @returns Parsed watch mode options
+ */
+export function parseWatchOptions(opts) {
+    let intervalMs = DEFAULT_WATCH_INTERVAL_MS;
+    if (opts.interval) {
+        // Check if it's a plain number (no unit suffix) - treat as minutes
+        if (/^\d+$/.test(opts.interval)) {
+            const minutes = parseInt(opts.interval, 10);
+            intervalMs = minutes * 60 * 1000;
+        }
+        else {
+            // Parse with ms library (handles units like "5m", "1h", "30s")
+            const parsed = ms(opts.interval);
+            if (typeof parsed === 'number') {
+                intervalMs = parsed;
+            }
+        }
+    }
+    // Enforce minimum interval
+    if (intervalMs < MIN_WATCH_INTERVAL_MS) {
+        intervalMs = MIN_WATCH_INTERVAL_MS;
+    }
+    return { intervalMs };
+}
+/**
+ * Formats watch cycle output for display.
+ *
+ * @param result - Monitor result from the cycle
+ * @param cycleNumber - The cycle number (1-indexed)
+ * @param timestamp - Timestamp of the cycle
+ * @returns Formatted output string
+ */
+export function formatWatchCycleOutput(result, cycleNumber, timestamp) {
+    const lines = [];
+    // Cycle header with timestamp
+    const timeStr = timestamp.toISOString().replace('T', ' ').substring(0, 19);
+    lines.push(chalk.cyan(`=== Patrol Cycle #${cycleNumber} [${timeStr}] ===`));
+    lines.push('');
+    // Quick summary
+    const { analysis, stuckSpawns, zombieLocks } = result;
+    const statusLine = [
+        `Pending: ${analysis.pending}`,
+        `Completed: ${analysis.completed}`,
+        `Stuck: ${stuckSpawns.length}`,
+        `Zombies: ${zombieLocks.length}`,
+    ].join(' | ');
+    if (stuckSpawns.length === 0 && zombieLocks.length === 0) {
+        lines.push(chalk.green(`  ${statusLine}`));
+        lines.push(chalk.green('  All spawns healthy.'));
+    }
+    else {
+        lines.push(chalk.yellow(`  ${statusLine}`));
+        // Show stuck spawns
+        if (stuckSpawns.length > 0) {
+            lines.push('');
+            lines.push(chalk.yellow('  Stuck spawns:'));
+            for (const info of stuckSpawns) {
+                lines.push(chalk.yellow(`    - ${info.spawn.targetWuId} (${info.ageMinutes}min)`));
+            }
+        }
+        // Show zombie locks
+        if (zombieLocks.length > 0) {
+            lines.push('');
+            lines.push(chalk.yellow('  Zombie locks:'));
+            for (const lock of zombieLocks) {
+                lines.push(chalk.yellow(`    - ${lock.lane} (PID ${lock.pid})`));
+            }
+        }
+    }
+    lines.push('');
+    return lines.join('\n');
+}
+/**
+ * Creates a watch mode runner for continuous spawn monitoring.
+ *
+ * @param options - Configuration options
+ * @returns WatchModeRunner instance
+ */
+export function createWatchModeRunner(options) {
+    const { checkFn, intervalMs, onOutput = console.log } = options;
+    let currentIntervalMs = intervalMs;
+    let consecutiveFailures = 0;
+    let running = false;
+    let timer = null;
+    let cycleCount = 0;
+    function scheduleNext() {
+        if (!running)
+            return;
+        timer = setTimeout(() => {
+            void runCycle().then(() => scheduleNext());
+        }, currentIntervalMs);
+    }
+    async function runCycle() {
+        if (!running)
+            return;
+        cycleCount++;
+        const cycleNumber = cycleCount;
+        try {
+            const result = await checkFn();
+            // Success - reset backoff
+            consecutiveFailures = 0;
+            currentIntervalMs = intervalMs;
+            // Output cycle result
+            const output = formatWatchCycleOutput(result, cycleNumber, new Date());
+            onOutput(output);
+        }
+        catch (error) {
+            // Failure - apply backoff
+            consecutiveFailures++;
+            currentIntervalMs = calculateBackoff(consecutiveFailures, intervalMs);
+            if (currentIntervalMs > MAX_BACKOFF_MS) {
+                currentIntervalMs = MAX_BACKOFF_MS;
+            }
+            const errorMsg = error instanceof Error ? error.message : String(error);
+            onOutput(chalk.red(`=== Patrol Cycle #${cycleNumber} ERROR ===`));
+            onOutput(chalk.red(`  ${errorMsg}`));
+            onOutput(chalk.yellow(`  Next check in ${Math.round(currentIntervalMs / 1000)}s (backoff)`));
+            onOutput('');
+        }
+    }
+    return {
+        start() {
+            if (running)
+                return;
+            running = true;
+            onOutput(chalk.cyan(`${LOG_PREFIX} Starting watch mode (interval: ${intervalMs / 1000}s)`));
+            onOutput('');
+            scheduleNext();
+        },
+        stop() {
+            running = false;
+            if (timer) {
+                clearTimeout(timer);
+                timer = null;
+            }
+            onOutput(chalk.cyan(`${LOG_PREFIX} Stopping watch mode. Exiting gracefully.`));
+        },
+        get isRunning() {
+            return running;
+        },
+        get currentIntervalMs() {
+            return currentIntervalMs;
+        },
+        get consecutiveFailures() {
+            return consecutiveFailures;
+        },
+    };
+}
+/**
+ * Run watch mode (continuous patrol)
+ */
+async function runWatchMode(opts) {
+    const baseDir = process.cwd();
+    const thresholdMinutes = parseInt(opts.threshold, 10);
+    if (isNaN(thresholdMinutes) || thresholdMinutes <= 0) {
+        console.error(chalk.red(`${LOG_PREFIX} Invalid threshold: ${opts.threshold}`));
+        process.exit(EXIT_CODES.FAILURE);
+    }
+    const watchOptions = parseWatchOptions(opts);
+    console.log(chalk.cyan(`${LOG_PREFIX} Starting continuous patrol mode...`));
+    console.log(chalk.gray(`  Threshold: ${thresholdMinutes} minutes`));
+    console.log(chalk.gray(`  Interval: ${watchOptions.intervalMs / 1000} seconds`));
+    console.log(chalk.gray(`  Recovery: ${opts.recover ? 'enabled' : 'disabled'}`));
+    console.log(chalk.gray(`  Press Ctrl+C to stop`));
+    console.log('');
+    const checkFn = async () => {
+        return runMonitor({
+            baseDir,
+            thresholdMinutes,
+            recover: opts.recover,
+            dryRun: opts.dryRun,
+        });
+    };
+    const runner = createWatchModeRunner({
+        checkFn,
+        intervalMs: watchOptions.intervalMs,
+    });
+    // Handle graceful shutdown
+    const shutdown = () => {
+        runner.stop();
+        process.exit(0);
+    };
+    process.on('SIGINT', shutdown);
+    process.on('SIGTERM', shutdown);
+    runner.start();
+    // Keep process alive
+    await new Promise(() => {
+        // Never resolves - waits for signal
+    });
+}
+// CLI program
 const program = new Command()
     .name('orchestrate:monitor')
-    .description('Monitor spawned agent progress')
+    .description('Monitor spawned agent progress and spawn health (WU-1241, WU-1242)')
+    .option('--threshold <minutes>', 'Stuck detection threshold in minutes (default: 30)', '30')
+    .option('--recover', 'Run recovery actions for stuck spawns', false)
+    .option('--dry-run', 'Show what would be done without taking action', false)
     .option('--since <time>', 'Show signals since (e.g., 30m, 1h)', '30m')
     .option('--wu <id>', 'Filter by WU ID')
-    .action((opts) => {
+    .option('--signals-only', 'Only show signals (skip spawn analysis)', false)
+    .option('--watch', 'Continuous patrol mode (WU-1242)', false)
+    .option('--interval <time>', 'Patrol interval for watch mode (e.g., 5m, 10m, 1h)', '5m')
+    .action(async (opts) => {
     try {
-        const since = parseTimeString(opts.since);
-        console.log(chalk.cyan(`${LOG_PREFIX} Loading signals since ${since.toISOString()}...`));
-        const signals = loadRecentSignals(since);
-        if (signals.length === 0) {
-            console.log(chalk.yellow(`${LOG_PREFIX} No signals found.`));
-            console.log(chalk.gray('Agents may still be starting up, or memory layer not initialized.'));
+        if (opts.signalsOnly) {
+            await displaySignals(opts);
             return;
         }
-        const filtered = opts.wu ? signals.filter((s) => s.wuId === opts.wu) : signals;
-        console.log(chalk.bold(`\nRecent Signals (${filtered.length}):\n`));
-        for (const signal of filtered) {
-            const time = new Date(signal.timestamp).toLocaleTimeString();
-            const wu = signal.wuId ? chalk.cyan(signal.wuId) : chalk.gray('system');
-            const type = signal.type === 'complete'
-                ? chalk.green(signal.type)
-                : signal.type === 'error'
-                    ? chalk.red(signal.type)
-                    : chalk.yellow(signal.type);
-            console.log(`  ${chalk.gray(time)} [${wu}] ${type}: ${signal.message || ''}`);
+        if (opts.watch) {
+            await runWatchMode(opts);
+            return;
         }
-        console.log('');
-        console.log(chalk.gray(`Use: pnpm mem:inbox --since ${opts.since} for more details`));
+        await runSpawnMonitoring(opts);
     }
     catch (err) {
-        console.error(chalk.red(`${LOG_PREFIX} Error: ${err.message}`));
+        const message = err instanceof Error ? err.message : String(err);
+        console.error(chalk.red(`${LOG_PREFIX} Error: ${message}`));
         process.exit(EXIT_CODES.ERROR);
     }
 });