@tenex-chat/backend 0.9.1

package/src/daemon/RestartState.ts

@@ -0,0 +1,152 @@
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import * as os from "node:os";
+import { logger } from "@/utils/logger";
+
+/**
+ * State persisted during graceful restart
+ */
+export interface RestartStateData {
+    /** Timestamp when restart was requested */
+    requestedAt: number;
+    /** List of project IDs that were booted at the time of restart */
+    bootedProjects: string[];
+    /** PID of the process that requested the restart */
+    pid: number;
+    /** Hostname for debugging */
+    hostname: string;
+}
+
+/**
+ * Manages restart state persistence for graceful restarts.
+ *
+ * When the daemon receives SIGHUP in supervised mode, it:
+ * 1. Persists the list of currently booted projects
+ * 2. Waits for all active RALs to complete
+ * 3. Exits cleanly (exit code 0)
+ *
+ * On startup, the daemon:
+ * 1. Checks for restart state file
+ * 2. If found, auto-boots the projects listed
+ * 3. Clears the restart state file
+ */
+export class RestartState {
+    private static readonly RESTART_STATE_FILE = "restart-state.json";
+    private daemonDir: string;
+
+    constructor(daemonDir: string) {
+        this.daemonDir = daemonDir;
+    }
+
+    /**
+     * Get the path to the restart state file
+     */
+    private getStatePath(): string {
+        return path.join(this.daemonDir, RestartState.RESTART_STATE_FILE);
+    }
+
+    /**
+     * Save restart state before graceful shutdown.
+     * Uses atomic write (write to temp file, then rename) to prevent corruption.
+     */
+    async save(bootedProjects: string[]): Promise<void> {
+        const statePath = this.getStatePath();
+        const tempPath = `${statePath}.tmp.${process.pid}`;
+
+        const state: RestartStateData = {
+            requestedAt: Date.now(),
+            bootedProjects,
+            pid: process.pid,
+            hostname: os.hostname(),
+        };
+
+        try {
+            // Write to temp file first
+            await fs.writeFile(tempPath, JSON.stringify(state, null, 2), "utf-8");
+
+            // Atomic rename
+            await fs.rename(tempPath, statePath);
+
+            logger.info("[RestartState] Saved restart state", {
+                bootedProjectCount: bootedProjects.length,
+                statePath,
+            });
+        } catch (error) {
+            // Clean up temp file on failure
+            try {
+                await fs.unlink(tempPath);
+            } catch {
+                // Ignore cleanup errors
+            }
+
+            logger.error("[RestartState] Failed to save restart state", {
+                error: error instanceof Error ? error.message : String(error),
+            });
+            throw error;
+        }
+    }
+
+    /**
+     * Load restart state if it exists.
+     * Returns null if no restart state file is present.
+     */
+    async load(): Promise<RestartStateData | null> {
+        const statePath = this.getStatePath();
+
+        try {
+            const content = await fs.readFile(statePath, "utf-8");
+            const state: RestartStateData = JSON.parse(content);
+
+            logger.info("[RestartState] Loaded restart state", {
+                requestedAt: new Date(state.requestedAt).toISOString(),
+                bootedProjectCount: state.bootedProjects.length,
+                originalPid: state.pid,
+            });
+
+            return state;
+        } catch (error) {
+            const err = error as NodeJS.ErrnoException;
+            if (err.code === "ENOENT") {
+                // No restart state file - normal startup
+                return null;
+            }
+
+            logger.error("[RestartState] Failed to load restart state", {
+                error: error instanceof Error ? error.message : String(error),
+            });
+            return null;
+        }
+    }
+
+    /**
+     * Clear restart state after successful processing.
+     */
+    async clear(): Promise<void> {
+        const statePath = this.getStatePath();
+
+        try {
+            await fs.unlink(statePath);
+            logger.debug("[RestartState] Cleared restart state file");
+        } catch (error) {
+            const err = error as NodeJS.ErrnoException;
+            if (err.code !== "ENOENT") {
+                logger.warn("[RestartState] Failed to clear restart state", {
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            }
+        }
+    }
+
+    /**
+     * Check if a restart state file exists
+     */
+    async exists(): Promise<boolean> {
+        const statePath = this.getStatePath();
+        try {
+            await fs.stat(statePath);
+            return true;
+        } catch {
+            return false;
+        }
+    }
+}

package/src/daemon/RuntimeLifecycle.ts

@@ -0,0 +1,268 @@
+import { logger } from "@/utils/logger";
+import type { NDKProject } from "@nostr-dev-kit/ndk";
+import { trace } from "@opentelemetry/api";
+import { ProjectAlreadyRunningError } from "@/services/scheduling/errors";
+import { ProjectRuntime } from "./ProjectRuntime";
+
+/**
+ * Manages the lifecycle of project runtimes.
+ * Handles starting, stopping, restarting, and crash recovery for ProjectRuntime instances.
+ *
+ * This class extracts runtime management logic from the Daemon class,
+ * eliminating duplicate code and providing a focused responsibility.
+ */
+export class RuntimeLifecycle {
+    private activeRuntimes = new Map<string, ProjectRuntime>();
+    private startingRuntimes = new Map<string, Promise<ProjectRuntime>>();
+
+    constructor(private projectsBase: string) {}
+
+    /**
+     * Get all active runtimes
+     */
+    getActiveRuntimes(): Map<string, ProjectRuntime> {
+        return new Map(this.activeRuntimes);
+    }
+
+    /**
+     * Get a specific runtime by project ID
+     */
+    getRuntime(projectId: string): ProjectRuntime | undefined {
+        return this.activeRuntimes.get(projectId);
+    }
+
+    /**
+     * Check if a runtime is active
+     */
+    isRuntimeActive(projectId: string): boolean {
+        return this.activeRuntimes.has(projectId);
+    }
+
+    /**
+     * Start a new runtime for a project.
+     * This is the single source of truth for runtime startup logic.
+     *
+     * @param projectId - The project ID
+     * @param project - The NDKProject instance
+     * @returns Promise resolving to the started ProjectRuntime
+     * @throws Error if startup fails
+     */
+    async startRuntime(projectId: string, project: NDKProject): Promise<ProjectRuntime> {
+        // Check if already running
+        const existingRuntime = this.activeRuntimes.get(projectId);
+        if (existingRuntime) {
+            const status = existingRuntime.getStatus();
+            if (status.isRunning) {
+                throw new ProjectAlreadyRunningError(projectId);
+            }
+            // Runtime exists but is not running - remove it and allow restart
+            trace.getActiveSpan()?.addEvent("runtime_lifecycle.removing_stale", {
+                "project.id": projectId,
+            });
+            this.activeRuntimes.delete(projectId);
+        }
+
+        // Check if already starting
+        const existingPromise = this.startingRuntimes.get(projectId);
+        if (existingPromise) {
+            return existingPromise;
+        }
+
+        const projectTitle = project.tagValue("title") || "Untitled";
+        trace.getActiveSpan()?.addEvent("runtime_lifecycle.starting", {
+            "project.id": projectId,
+            "project.title": projectTitle,
+        });
+
+        // Create startup promise to prevent concurrent startups
+        const startupPromise = this.performStartup(project);
+        this.startingRuntimes.set(projectId, startupPromise);
+
+        try {
+            const runtime = await startupPromise;
+            this.activeRuntimes.set(projectId, runtime);
+
+            trace.getActiveSpan()?.addEvent("runtime_lifecycle.started", {
+                "project.id": projectId,
+                "project.title": projectTitle,
+            });
+
+            return runtime;
+        } catch (error) {
+            logger.error(`Failed to start project runtime: ${projectId}`, {
+                error: error instanceof Error ? error.message : String(error),
+            });
+            throw error;
+        } finally {
+            // Always clean up the starting promise
+            this.startingRuntimes.delete(projectId);
+        }
+    }
+
+    /**
+     * Perform the actual runtime startup
+     * @param project - The NDKProject instance
+     * @returns Promise resolving to the started ProjectRuntime
+     */
+    private async performStartup(project: NDKProject): Promise<ProjectRuntime> {
+        const newRuntime = new ProjectRuntime(project, this.projectsBase);
+        await newRuntime.start();
+        return newRuntime;
+    }
+
+    /**
+     * Stop a runtime gracefully
+     * @param projectId - The project ID to stop
+     * @throws Error if the runtime is not found
+     */
+    async stopRuntime(projectId: string): Promise<void> {
+        const runtime = this.activeRuntimes.get(projectId);
+        if (!runtime) {
+            throw new Error(`Runtime not found: ${projectId}`);
+        }
+
+        trace.getActiveSpan()?.addEvent("runtime_lifecycle.stopping", {
+            "project.id": projectId,
+        });
+
+        try {
+            await runtime.stop();
+            this.activeRuntimes.delete(projectId);
+            trace.getActiveSpan()?.addEvent("runtime_lifecycle.stopped", {
+                "project.id": projectId,
+            });
+        } catch (error) {
+            logger.error(`Error stopping project runtime: ${projectId}`, {
+                error: error instanceof Error ? error.message : String(error),
+            });
+            // Still remove from active runtimes even if stop fails
+            this.activeRuntimes.delete(projectId);
+            throw error;
+        }
+    }
+
+    /**
+     * Restart a runtime (stop and start again)
+     * @param projectId - The project ID to restart
+     * @param project - The NDKProject instance (needed for restart)
+     * @throws Error if the runtime is not found or restart fails
+     */
+    async restartRuntime(projectId: string, project: NDKProject): Promise<ProjectRuntime> {
+        const runtime = this.activeRuntimes.get(projectId);
+        if (!runtime) {
+            throw new Error(`Runtime not found: ${projectId}`);
+        }
+
+        trace.getActiveSpan()?.addEvent("runtime_lifecycle.restarting", {
+            "project.id": projectId,
+        });
+
+        try {
+            // Stop the runtime
+            await runtime.stop();
+            this.activeRuntimes.delete(projectId);
+
+            // Start it again
+            return await this.startRuntime(projectId, project);
+        } catch (error) {
+            logger.error(`Failed to restart project runtime: ${projectId}`, {
+                error: error instanceof Error ? error.message : String(error),
+            });
+            throw error;
+        }
+    }
+
+    /**
+     * Handle a crashed runtime by cleaning it up
+     * @param projectId - The project ID that crashed
+     * @param runtime - The crashed runtime instance
+     */
+    async handleRuntimeCrash(projectId: string, runtime: ProjectRuntime): Promise<void> {
+        logger.error(`Handling crashed runtime: ${projectId}`);
+
+        // Remove from active runtimes
+        this.activeRuntimes.delete(projectId);
+
+        // Best effort cleanup
+        try {
+            await runtime.stop();
+        } catch (cleanupError) {
+            logger.warn(`Failed to clean up crashed runtime: ${projectId}`, {
+                error: cleanupError instanceof Error ? cleanupError.message : String(cleanupError),
+            });
+        }
+    }
+
+    /**
+     * Stop all active runtimes (for shutdown)
+     */
+    async stopAllRuntimes(): Promise<void> {
+        trace.getActiveSpan()?.addEvent("runtime_lifecycle.stopping_all", {
+            "runtimes.active_count": this.activeRuntimes.size,
+        });
+
+        const stopPromises = Array.from(this.activeRuntimes.entries()).map(
+            async ([projectId, runtime]) => {
+                try {
+                    await runtime.stop();
+                } catch (error) {
+                    logger.error(`Error stopping runtime during shutdown: ${projectId}`, {
+                        error: error instanceof Error ? error.message : String(error),
+                    });
+                }
+            }
+        );
+
+        await Promise.all(stopPromises);
+        this.activeRuntimes.clear();
+
+        trace.getActiveSpan()?.addEvent("runtime_lifecycle.all_stopped");
+    }
+
+    /**
+     * Get statistics about managed runtimes
+     */
+    getStats(): {
+        activeCount: number;
+        startingCount: number;
+        projectIds: string[];
+    } {
+        return {
+            activeCount: this.activeRuntimes.size,
+            startingCount: this.startingRuntimes.size,
+            projectIds: Array.from(this.activeRuntimes.keys()),
+        };
+    }
+
+    /**
+     * Check if any runtimes are currently starting
+     */
+    hasStartingRuntimes(): boolean {
+        return this.startingRuntimes.size > 0;
+    }
+
+    /**
+     * Wait for all starting runtimes to complete
+     * Useful for graceful shutdown
+     */
+    async waitForStartingRuntimes(): Promise<void> {
+        if (this.startingRuntimes.size === 0) {
+            return;
+        }
+
+        trace.getActiveSpan()?.addEvent("runtime_lifecycle.waiting_for_starting", {
+            "runtimes.starting_count": this.startingRuntimes.size,
+        });
+
+        const promises = Array.from(this.startingRuntimes.values());
+        await Promise.allSettled(promises);
+    }
+
+    /**
+     * Get list of currently active project IDs.
+     * Used by graceful restart to persist which projects to auto-boot after restart.
+     */
+    getActiveProjectIds(): string[] {
+        return Array.from(this.activeRuntimes.keys());
+    }
+}