@lumenflow/core 1.0.0

package/dist/spawn-recovery.d.ts

@@ -0,0 +1,82 @@
+/**
+ * Spawn Recovery Module (WU-1951)
+ *
+ * Auto-recovery heuristics for stuck spawns and zombie locks.
+ * Used by orchestrate:monitor for automatic spawn health management.
+ *
+ * Recovery Heuristics:
+ * 1. Zombie lock (PID not running) -> auto-release, mark spawn crashed
+ * 2. Stale lock (>2h) -> auto-release, mark spawn timeout
+ * 3. Active lock + no checkpoint in 1h -> mark stuck, escalate
+ *
+ * All recovery actions are logged to .beacon/recovery/ for audit.
+ *
+ * Library-First Note: This is project-specific spawn recovery code for
+ * PatientPath's custom spawn-registry.jsonl, lane-lock, and memory-store.
+ * No external library exists for this domain-specific agent lifecycle management.
+ *
+ * @see {@link tools/lib/__tests__/spawn-recovery.test.mjs} - Tests
+ * @see {@link tools/lib/spawn-monitor.mjs} - Monitoring logic
+ * @see {@link tools/lib/spawn-registry-store.mjs} - Spawn state
+ */
+/**
+ * Recovery action constants
+ */
+export declare const RecoveryAction: Readonly<{
+    /** No recovery needed */
+    NONE: "none";
+    /** Zombie lock released (PID not running) */
+    RELEASED_ZOMBIE: "released_zombie";
+    /** Stale lock released (>2h old) */
+    RELEASED_STALE: "released_stale";
+    /** Stuck spawn escalated (active but no checkpoint in 1h) */
+    ESCALATED_STUCK: "escalated_stuck";
+}>;
+/**
+ * Recovery directory name
+ */
+export declare const RECOVERY_DIR_NAME = "recovery";
+/**
+ * Threshold for "no checkpoint" detection (1 hour in milliseconds)
+ */
+export declare const NO_CHECKPOINT_THRESHOLD_MS: number;
+/**
+ * Recovers a stuck spawn by applying appropriate heuristics.
+ *
+ * Recovery order (first match wins):
+ * 1. Zombie lock (PID not running) -> release lock, mark crashed
+ * 2. Stale lock (>2h) -> release lock, mark timeout
+ * 3. Active lock + no checkpoint in 1h -> escalate (no auto-release)
+ * 4. Healthy spawn -> no action
+ *
+ * @param {string} spawnId - ID of the spawn to recover
+ * @param {RecoverStuckSpawnOptions} options - Options
+ * @returns {Promise<RecoveryResult>} Recovery result
+ *
+ * @example
+ * const result = await recoverStuckSpawn('spawn-1234', { baseDir: '/path/to/project' });
+ * if (result.recovered) {
+ *   console.log(`Recovered: ${result.action} - ${result.reason}`);
+ * }
+ */
+export interface RecoverStuckSpawnOptions {
+    /** Base directory for .beacon/ */
+    baseDir?: string;
+}
+export declare function recoverStuckSpawn(spawnId: any, options?: RecoverStuckSpawnOptions): Promise<{
+    recovered: boolean;
+    action: "none";
+    reason: string;
+} | {
+    recovered: boolean;
+    action: "released_zombie";
+    reason: string;
+} | {
+    recovered: boolean;
+    action: "released_stale";
+    reason: string;
+} | {
+    recovered: boolean;
+    action: "escalated_stuck";
+    reason: string;
+}>;

package/dist/spawn-recovery.js

@@ -0,0 +1,298 @@
+/**
+ * Spawn Recovery Module (WU-1951)
+ *
+ * Auto-recovery heuristics for stuck spawns and zombie locks.
+ * Used by orchestrate:monitor for automatic spawn health management.
+ *
+ * Recovery Heuristics:
+ * 1. Zombie lock (PID not running) -> auto-release, mark spawn crashed
+ * 2. Stale lock (>2h) -> auto-release, mark spawn timeout
+ * 3. Active lock + no checkpoint in 1h -> mark stuck, escalate
+ *
+ * All recovery actions are logged to .beacon/recovery/ for audit.
+ *
+ * Library-First Note: This is project-specific spawn recovery code for
+ * PatientPath's custom spawn-registry.jsonl, lane-lock, and memory-store.
+ * No external library exists for this domain-specific agent lifecycle management.
+ *
+ * @see {@link tools/lib/__tests__/spawn-recovery.test.mjs} - Tests
+ * @see {@link tools/lib/spawn-monitor.mjs} - Monitoring logic
+ * @see {@link tools/lib/spawn-registry-store.mjs} - Spawn state
+ */
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { SpawnRegistryStore } from './spawn-registry-store.js';
+import { SpawnStatus } from './spawn-registry-schema.js';
+import { isZombieLock, isLockStale, readLockMetadata, getLockFilePath, releaseLaneLock, } from './lane-lock.js';
+import { toKebab } from './wu-constants.js';
+// Optional import from @lumenflow/memory
+let loadMemory = null;
+try {
+    const mod = await import('@lumenflow/memory/store');
+    loadMemory = mod.loadMemory;
+}
+catch {
+    // @lumenflow/memory not available - memory features disabled
+}
+/**
+ * Recovery action constants
+ */
+export const RecoveryAction = Object.freeze({
+    /** No recovery needed */
+    NONE: 'none',
+    /** Zombie lock released (PID not running) */
+    RELEASED_ZOMBIE: 'released_zombie',
+    /** Stale lock released (>2h old) */
+    RELEASED_STALE: 'released_stale',
+    /** Stuck spawn escalated (active but no checkpoint in 1h) */
+    ESCALATED_STUCK: 'escalated_stuck',
+});
+/**
+ * Recovery directory name
+ */
+export const RECOVERY_DIR_NAME = 'recovery';
+/**
+ * Threshold for "no checkpoint" detection (1 hour in milliseconds)
+ */
+export const NO_CHECKPOINT_THRESHOLD_MS = 60 * 60 * 1000;
+/**
+ * Log prefix for spawn-recovery messages
+ */
+const LOG_PREFIX = '[spawn-recovery]';
+/**
+ * @typedef {Object} RecoveryResult
+ * @property {boolean} recovered - Whether a recovery action was taken
+ * @property {string} action - The recovery action taken (from RecoveryAction)
+ * @property {string} reason - Human-readable explanation of the result
+ */
+/**
+ * @typedef {Object} AuditLogEntry
+ * @property {string} timestamp - ISO timestamp of recovery action
+ * @property {string} spawnId - ID of the spawn being recovered
+ * @property {string} action - Recovery action taken
+ * @property {string} reason - Explanation of why action was taken
+ * @property {Object} context - Additional context
+ * @property {string} context.targetWuId - Target WU ID
+ * @property {string} context.lane - Lane name
+ * @property {Object|null} context.lockMetadata - Lock metadata if present
+ * @property {string|null} context.lastCheckpoint - Last checkpoint timestamp
+ */
+/**
+ * Converts lane name to lock file path (kebab-case)
+ *
+ * @param {string} lane - Lane name (e.g., "Operations: Tooling")
+ * @returns {string} Kebab-case lane name (e.g., "operations-tooling")
+ */
+function laneToKebab(lane) {
+    return toKebab(lane);
+}
+/**
+ * Gets the recovery directory path
+ *
+ * @param {string} baseDir - Base directory
+ * @returns {string} Path to .beacon/recovery/
+ */
+function getRecoveryDir(baseDir) {
+    return path.join(baseDir, '.beacon', RECOVERY_DIR_NAME);
+}
+/**
+ * Creates an audit log entry
+ *
+ * @param {string} baseDir - Base directory
+ * @param {AuditLogEntry} entry - Audit log entry
+ * @returns {Promise<void>}
+ */
+async function createAuditLog(baseDir, entry) {
+    const recoveryDir = getRecoveryDir(baseDir);
+    await fs.mkdir(recoveryDir, { recursive: true });
+    const timestamp = entry.timestamp.replace(/[:.]/g, '-');
+    const fileName = `${entry.spawnId}-${timestamp}.json`;
+    const filePath = path.join(recoveryDir, fileName);
+    await fs.writeFile(filePath, JSON.stringify(entry, null, 2), 'utf-8');
+    console.log(`${LOG_PREFIX} Audit log created: ${fileName}`);
+}
+/**
+ * Gets the most recent checkpoint for a WU from the memory store
+ *
+ * @param {string} baseDir - Base directory
+ * @param {string} wuId - WU ID to find checkpoints for
+ * @returns {Promise<{timestamp: string, content: string}|null>} Most recent checkpoint or null
+ */
+async function getLastCheckpoint(baseDir, wuId) {
+    // If memory module not available, return null
+    if (!loadMemory) {
+        return null;
+    }
+    const memoryDir = path.join(baseDir, '.beacon', 'state');
+    try {
+        const memory = await loadMemory(memoryDir, wuId);
+        if (!memory) {
+            return null;
+        }
+        const checkpoints = memory.checkpoints ?? [];
+        if (checkpoints.length === 0) {
+            return null;
+        }
+        // Sort by timestamp descending, get most recent
+        checkpoints.sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime());
+        const latest = checkpoints[0];
+        return {
+            timestamp: latest.timestamp,
+            content: '',
+        };
+    }
+    catch {
+        // Memory store doesn't exist or is invalid
+        return null;
+    }
+}
+/**
+ * Checks if a checkpoint is recent enough (within 1 hour)
+ *
+ * @param {string|null} checkpointTimestamp - ISO timestamp of last checkpoint
+ * @returns {boolean} True if checkpoint is recent (within 1 hour)
+ */
+function isCheckpointRecent(checkpointTimestamp) {
+    if (!checkpointTimestamp) {
+        return false;
+    }
+    const checkpointTime = new Date(checkpointTimestamp).getTime();
+    const now = Date.now();
+    return now - checkpointTime <= NO_CHECKPOINT_THRESHOLD_MS;
+}
+export async function recoverStuckSpawn(spawnId, options = {}) {
+    const { baseDir = process.cwd() } = options;
+    const registryDir = path.join(baseDir, '.beacon', 'state');
+    // Load spawn registry
+    const store = new SpawnRegistryStore(registryDir);
+    try {
+        await store.load();
+    }
+    catch {
+        // Registry doesn't exist or is invalid
+        return {
+            recovered: false,
+            action: RecoveryAction.NONE,
+            reason: `Spawn ${spawnId} not found: registry unavailable`,
+        };
+    }
+    // Find the spawn
+    const spawn = store.getById(spawnId);
+    if (!spawn) {
+        return {
+            recovered: false,
+            action: RecoveryAction.NONE,
+            reason: `Spawn ${spawnId} not found in registry`,
+        };
+    }
+    // Check if already completed
+    if (spawn.status !== SpawnStatus.PENDING) {
+        return {
+            recovered: false,
+            action: RecoveryAction.NONE,
+            reason: `Spawn ${spawnId} already ${spawn.status}`,
+        };
+    }
+    // Get lock for this spawn's lane
+    const laneKebab = laneToKebab(spawn.lane);
+    const lockPath = getLockFilePath(spawn.lane, baseDir);
+    const lockMetadata = readLockMetadata(lockPath);
+    // If no lock, nothing to recover
+    if (!lockMetadata) {
+        return {
+            recovered: false,
+            action: RecoveryAction.NONE,
+            reason: `No lock found for spawn ${spawnId} (lane: ${spawn.lane})`,
+        };
+    }
+    // Check if lock belongs to this WU
+    if (lockMetadata.wuId !== spawn.targetWuId) {
+        return {
+            recovered: false,
+            action: RecoveryAction.NONE,
+            reason: `Lock belongs to ${lockMetadata.wuId}, not spawn target ${spawn.targetWuId}`,
+        };
+    }
+    // Get last checkpoint for context
+    const lastCheckpoint = await getLastCheckpoint(baseDir, spawn.targetWuId);
+    const lastCheckpointTs = lastCheckpoint?.timestamp ?? null;
+    // Build common audit context
+    const auditContext = {
+        targetWuId: spawn.targetWuId,
+        parentWuId: spawn.parentWuId,
+        lane: spawn.lane,
+        spawnedAt: spawn.spawnedAt,
+        lockMetadata,
+        lastCheckpoint: lastCheckpointTs,
+    };
+    // Heuristic 1: Zombie lock (PID not running)
+    if (isZombieLock(lockMetadata)) {
+        console.log(`${LOG_PREFIX} Detected zombie lock for ${spawnId} (PID ${lockMetadata.pid} not running)`);
+        // Release the lock
+        releaseLaneLock(spawn.lane, { baseDir, force: true });
+        // Mark spawn as crashed
+        await store.updateStatus(spawnId, SpawnStatus.CRASHED);
+        const reason = `Zombie lock detected: PID ${lockMetadata.pid} not running`;
+        // Create audit log
+        await createAuditLog(baseDir, {
+            timestamp: new Date().toISOString(),
+            spawnId,
+            action: RecoveryAction.RELEASED_ZOMBIE,
+            reason,
+            context: auditContext,
+        });
+        return {
+            recovered: true,
+            action: RecoveryAction.RELEASED_ZOMBIE,
+            reason,
+        };
+    }
+    // Heuristic 2: Stale lock (>2h old)
+    if (isLockStale(lockMetadata)) {
+        console.log(`${LOG_PREFIX} Detected stale lock for ${spawnId} (acquired ${lockMetadata.timestamp})`);
+        // Release the lock
+        releaseLaneLock(spawn.lane, { baseDir, force: true });
+        // Mark spawn as timeout
+        await store.updateStatus(spawnId, SpawnStatus.TIMEOUT);
+        const reason = `Stale lock detected: acquired ${lockMetadata.timestamp} (>2h threshold)`;
+        // Create audit log
+        await createAuditLog(baseDir, {
+            timestamp: new Date().toISOString(),
+            spawnId,
+            action: RecoveryAction.RELEASED_STALE,
+            reason,
+            context: auditContext,
+        });
+        return {
+            recovered: true,
+            action: RecoveryAction.RELEASED_STALE,
+            reason,
+        };
+    }
+    // Heuristic 3: Active lock + no recent checkpoint -> escalate
+    if (!isCheckpointRecent(lastCheckpointTs)) {
+        const reason = lastCheckpointTs
+            ? `No checkpoint in last hour (last: ${lastCheckpointTs})`
+            : 'No checkpoints recorded for this spawn';
+        console.log(`${LOG_PREFIX} Escalating stuck spawn ${spawnId}: ${reason}`);
+        // Create audit log (escalation, not recovery)
+        await createAuditLog(baseDir, {
+            timestamp: new Date().toISOString(),
+            spawnId,
+            action: RecoveryAction.ESCALATED_STUCK,
+            reason,
+            context: auditContext,
+        });
+        return {
+            recovered: false, // No auto-recovery, just escalation
+            action: RecoveryAction.ESCALATED_STUCK,
+            reason: `Stuck spawn: ${reason}`,
+        };
+    }
+    // Healthy spawn with recent checkpoint
+    return {
+        recovered: false,
+        action: RecoveryAction.NONE,
+        reason: `Spawn ${spawnId} healthy (recent checkpoint at ${lastCheckpointTs})`,
+    };
+}

package/dist/spawn-registry-schema.d.ts

@@ -0,0 +1,98 @@
+/**
+ * Spawn Registry Schema (WU-1944)
+ *
+ * Zod schemas for spawn event validation.
+ * Defines schema for tracking sub-agent spawns by orchestrators.
+ *
+ * @see {@link tools/lib/__tests__/spawn-registry-store.test.mjs} - Tests
+ * @see {@link tools/lib/spawn-registry-store.mjs} - Store implementation
+ */
+import { z } from 'zod';
+/**
+ * Spawn status values
+ */
+export declare const SpawnStatus: {
+    readonly PENDING: "pending";
+    readonly COMPLETED: "completed";
+    readonly TIMEOUT: "timeout";
+    readonly CRASHED: "crashed";
+    /** WU-1967: Spawn escalated to orchestrator (signal sent, prevents duplicates) */
+    readonly ESCALATED: "escalated";
+};
+/** Type for spawn status values */
+export type SpawnStatusValue = (typeof SpawnStatus)[keyof typeof SpawnStatus];
+/**
+ * Array of valid spawn statuses
+ */
+export declare const SPAWN_STATUSES: readonly ["pending", "completed", "timeout", "crashed", "escalated"];
+/**
+ * Regex patterns for spawn validation
+ */
+export declare const SPAWN_PATTERNS: {
+    /** Spawn ID format: spawn-{4 hex chars} */
+    SPAWN_ID: RegExp;
+    /** WU ID format: WU-{digits} */
+    WU_ID: RegExp;
+};
+/**
+ * Spawn Event Schema
+ *
+ * Defines the structure for spawn registry events.
+ * Uses append-only JSONL storage with event replay for state reconstruction.
+ */
+export declare const SpawnEventSchema: z.ZodObject<{
+    id: z.ZodString;
+    parentWuId: z.ZodString;
+    targetWuId: z.ZodString;
+    lane: z.ZodString;
+    spawnedAt: z.ZodString;
+    status: z.ZodEnum<{
+        completed: "completed";
+        timeout: "timeout";
+        pending: "pending";
+        crashed: "crashed";
+        escalated: "escalated";
+    }>;
+    completedAt: z.ZodNullable<z.ZodString>;
+}, z.core.$strip>;
+/**
+ * TypeScript type inferred from schema
+ */
+export type SpawnEvent = z.infer<typeof SpawnEventSchema>;
+/**
+ * Validates spawn event data against schema
+ *
+ * @param {unknown} data - Data to validate
+ * @returns Validation result
+ *
+ * @example
+ * const result = validateSpawnEvent(eventData);
+ * if (!result.success) {
+ *   result.error.issues.forEach(issue => {
+ *     console.error(`${issue.path.join('.')}: ${issue.message}`);
+ *   });
+ * }
+ */
+export declare function validateSpawnEvent(data: unknown): z.ZodSafeParseResult<{
+    id: string;
+    parentWuId: string;
+    targetWuId: string;
+    lane: string;
+    spawnedAt: string;
+    status: "completed" | "timeout" | "pending" | "crashed" | "escalated";
+    completedAt?: string;
+}>;
+/**
+ * Generates a unique spawn ID from parent WU, target WU, and timestamp
+ *
+ * Format: spawn-XXXX (4 hex characters from SHA-256 hash)
+ *
+ * @param {string} parentWuId - Parent WU ID
+ * @param {string} targetWuId - Target WU ID
+ * @returns {string} Spawn ID in format spawn-XXXX
+ *
+ * @example
+ * const id = generateSpawnId('WU-1000', 'WU-1001');
+ * // Returns: 'spawn-a1b2'
+ */
+export declare function generateSpawnId(parentWuId: string, targetWuId: string): string;

package/dist/spawn-registry-schema.js

@@ -0,0 +1,108 @@
+/**
+ * Spawn Registry Schema (WU-1944)
+ *
+ * Zod schemas for spawn event validation.
+ * Defines schema for tracking sub-agent spawns by orchestrators.
+ *
+ * @see {@link tools/lib/__tests__/spawn-registry-store.test.mjs} - Tests
+ * @see {@link tools/lib/spawn-registry-store.mjs} - Store implementation
+ */
+import { z } from 'zod';
+import crypto from 'node:crypto';
+/**
+ * Spawn status values
+ */
+export const SpawnStatus = {
+    PENDING: 'pending',
+    COMPLETED: 'completed',
+    TIMEOUT: 'timeout',
+    CRASHED: 'crashed',
+    /** WU-1967: Spawn escalated to orchestrator (signal sent, prevents duplicates) */
+    ESCALATED: 'escalated',
+};
+/**
+ * Array of valid spawn statuses
+ */
+export const SPAWN_STATUSES = ['pending', 'completed', 'timeout', 'crashed', 'escalated'];
+/**
+ * Regex patterns for spawn validation
+ */
+export const SPAWN_PATTERNS = {
+    /** Spawn ID format: spawn-{4 hex chars} */
+    SPAWN_ID: /^spawn-[0-9a-f]{4}$/,
+    /** WU ID format: WU-{digits} */
+    WU_ID: /^WU-\d+$/,
+};
+/**
+ * Error messages for schema validation
+ */
+const ERROR_MESSAGES = {
+    SPAWN_ID: 'Spawn ID must match pattern spawn-XXXX (e.g., spawn-a1b2)',
+    WU_ID: 'WU ID must match pattern WU-XXX (e.g., WU-1000)',
+    LANE_REQUIRED: 'Lane is required',
+    STATUS: `Status must be one of: ${SPAWN_STATUSES.join(', ')}`,
+    TIMESTAMP_REQUIRED: 'Timestamp is required',
+};
+/**
+ * Spawn Event Schema
+ *
+ * Defines the structure for spawn registry events.
+ * Uses append-only JSONL storage with event replay for state reconstruction.
+ */
+export const SpawnEventSchema = z.object({
+    /** Unique spawn ID in format spawn-XXXX (4 hex chars from SHA hash) */
+    id: z.string().regex(SPAWN_PATTERNS.SPAWN_ID, { message: ERROR_MESSAGES.SPAWN_ID }),
+    /** Parent WU ID (the orchestrator that spawned this agent) */
+    parentWuId: z.string().regex(SPAWN_PATTERNS.WU_ID, { message: ERROR_MESSAGES.WU_ID }),
+    /** Target WU ID (the WU being executed by the spawned agent) */
+    targetWuId: z.string().regex(SPAWN_PATTERNS.WU_ID, { message: ERROR_MESSAGES.WU_ID }),
+    /** Lane for the spawned work */
+    lane: z.string().min(1, { message: ERROR_MESSAGES.LANE_REQUIRED }),
+    /** ISO 8601 timestamp when spawn was recorded */
+    spawnedAt: z.string().datetime({ message: ERROR_MESSAGES.TIMESTAMP_REQUIRED }),
+    /** Current status of the spawned agent */
+    status: z.enum(SPAWN_STATUSES, {
+        error: ERROR_MESSAGES.STATUS,
+    }),
+    /** ISO 8601 timestamp when spawn completed (null if pending) */
+    completedAt: z.string().datetime().nullable(),
+});
+/**
+ * Validates spawn event data against schema
+ *
+ * @param {unknown} data - Data to validate
+ * @returns Validation result
+ *
+ * @example
+ * const result = validateSpawnEvent(eventData);
+ * if (!result.success) {
+ *   result.error.issues.forEach(issue => {
+ *     console.error(`${issue.path.join('.')}: ${issue.message}`);
+ *   });
+ * }
+ */
+export function validateSpawnEvent(data) {
+    return SpawnEventSchema.safeParse(data);
+}
+/**
+ * Generates a unique spawn ID from parent WU, target WU, and timestamp
+ *
+ * Format: spawn-XXXX (4 hex characters from SHA-256 hash)
+ *
+ * @param {string} parentWuId - Parent WU ID
+ * @param {string} targetWuId - Target WU ID
+ * @returns {string} Spawn ID in format spawn-XXXX
+ *
+ * @example
+ * const id = generateSpawnId('WU-1000', 'WU-1001');
+ * // Returns: 'spawn-a1b2'
+ */
+export function generateSpawnId(parentWuId, targetWuId) {
+    // Include timestamp and random bytes for uniqueness
+    const timestamp = Date.now().toString();
+    const randomBytes = crypto.randomBytes(4).toString('hex');
+    const input = `${parentWuId}:${targetWuId}:${timestamp}:${randomBytes}`;
+    const hash = crypto.createHash('sha256').update(input).digest('hex');
+    // Take first 4 hex chars
+    return `spawn-${hash.slice(0, 4)}`;
+}