@spfn/core 0.2.0-beta.15 → 0.2.0-beta.17

package/dist/server/index.js

@@ -580,8 +580,231 @@ var SSETokenManager = class {
     }
   }
 };
+var serverLogger = logger.child("@spfn/core:server");
+
+// src/server/shutdown-manager.ts
+var DEFAULT_HOOK_TIMEOUT = 1e4;
+var DEFAULT_HOOK_ORDER = 100;
+var DRAIN_POLL_INTERVAL = 500;
+var ShutdownManager = class {
+  state = "running";
+  hooks = [];
+  operations = /* @__PURE__ */ new Map();
+  operationCounter = 0;
+  /**
+   * Register a shutdown hook
+   *
+   * Hooks run in order during shutdown, after all tracked operations drain.
+   * Each hook has its own timeout — failure does not block subsequent hooks.
+   *
+   * @example
+   * shutdown.onShutdown('ai-service', async () => {
+   *     await aiService.cancelPending();
+   * }, { timeout: 30000, order: 10 });
+   */
+  onShutdown(name, handler, options) {
+    this.hooks.push({
+      name,
+      handler,
+      timeout: options?.timeout ?? DEFAULT_HOOK_TIMEOUT,
+      order: options?.order ?? DEFAULT_HOOK_ORDER
+    });
+    this.hooks.sort((a, b) => a.order - b.order);
+    serverLogger.debug(`Shutdown hook registered: ${name}`, {
+      order: options?.order ?? DEFAULT_HOOK_ORDER,
+      timeout: `${options?.timeout ?? DEFAULT_HOOK_TIMEOUT}ms`
+    });
+  }
+  /**
+   * Track a long-running operation
+   *
+   * During shutdown (drain phase), the process waits for ALL tracked
+   * operations to complete before proceeding with cleanup.
+   *
+   * If shutdown has already started, the operation is rejected immediately.
+   *
+   * @returns The operation result (pass-through)
+   *
+   * @example
+   * const result = await shutdown.trackOperation(
+   *     'ai-generate',
+   *     aiService.generate(prompt)
+   * );
+   */
+  async trackOperation(name, operation) {
+    if (this.state !== "running") {
+      throw new Error(`Cannot start operation '${name}': server is shutting down`);
+    }
+    const id = `${name}-${++this.operationCounter}`;
+    this.operations.set(id, {
+      name,
+      startedAt: Date.now()
+    });
+    serverLogger.debug(`Operation tracked: ${id}`, {
+      activeOperations: this.operations.size
+    });
+    try {
+      return await operation;
+    } finally {
+      this.operations.delete(id);
+      serverLogger.debug(`Operation completed: ${id}`, {
+        activeOperations: this.operations.size
+      });
+    }
+  }
+  /**
+   * Whether the server is shutting down
+   *
+   * Use this to reject new work early (e.g., return 503 in route handlers).
+   */
+  isShuttingDown() {
+    return this.state !== "running";
+  }
+  /**
+   * Number of currently active tracked operations
+   */
+  getActiveOperationCount() {
+    return this.operations.size;
+  }
+  /**
+   * Mark shutdown as started immediately
+   *
+   * Call this at the very beginning of the shutdown sequence so that:
+   * - Health check returns 503 right away
+   * - trackOperation() rejects new work
+   * - isShuttingDown() returns true
+   */
+  beginShutdown() {
+    if (this.state !== "running") {
+      return;
+    }
+    this.state = "draining";
+    serverLogger.info("Shutdown manager: state set to draining");
+  }
+  /**
+   * Execute the full shutdown sequence
+   *
+   * 1. State → draining (reject new operations)
+   * 2. Wait for all tracked operations to complete (drain)
+   * 3. Run shutdown hooks in order
+   * 4. State → closed
+   *
+   * @param drainTimeout - Max time to wait for operations to drain (ms)
+   */
+  async execute(drainTimeout) {
+    if (this.state === "closed") {
+      serverLogger.warn("ShutdownManager.execute() called but already closed");
+      return;
+    }
+    this.state = "draining";
+    serverLogger.info("Shutdown manager: draining started", {
+      activeOperations: this.operations.size,
+      registeredHooks: this.hooks.length,
+      drainTimeout: `${drainTimeout}ms`
+    });
+    await this.drain(drainTimeout);
+    await this.executeHooks();
+    this.state = "closed";
+    serverLogger.info("Shutdown manager: all hooks executed");
+  }
+  // ========================================================================
+  // Private
+  // ========================================================================
+  /**
+   * Wait for all tracked operations to complete, up to drainTimeout
+   */
+  async drain(drainTimeout) {
+    if (this.operations.size === 0) {
+      serverLogger.info("Shutdown manager: no active operations, drain skipped");
+      return;
+    }
+    serverLogger.info(`Shutdown manager: waiting for ${this.operations.size} operations to drain...`);
+    const deadline = Date.now() + drainTimeout;
+    while (this.operations.size > 0 && Date.now() < deadline) {
+      const remaining = deadline - Date.now();
+      const ops = Array.from(this.operations.values()).map((op) => ({
+        name: op.name,
+        elapsed: `${Math.round((Date.now() - op.startedAt) / 1e3)}s`
+      }));
+      serverLogger.info("Shutdown manager: drain in progress", {
+        activeOperations: this.operations.size,
+        remainingTimeout: `${Math.round(remaining / 1e3)}s`,
+        operations: ops
+      });
+      await sleep(Math.min(DRAIN_POLL_INTERVAL, remaining));
+    }
+    if (this.operations.size > 0) {
+      const abandoned = Array.from(this.operations.values()).map((op) => op.name);
+      serverLogger.warn("Shutdown manager: drain timeout \u2014 abandoning operations", {
+        abandoned
+      });
+    } else {
+      serverLogger.info("Shutdown manager: all operations drained successfully");
+    }
+  }
+  /**
+   * Execute registered shutdown hooks in order
+   */
+  async executeHooks() {
+    if (this.hooks.length === 0) {
+      return;
+    }
+    serverLogger.info(`Shutdown manager: executing ${this.hooks.length} hooks...`);
+    for (const hook of this.hooks) {
+      serverLogger.debug(`Shutdown hook [${hook.name}] starting (timeout: ${hook.timeout}ms)`);
+      try {
+        await withTimeout(
+          hook.handler(),
+          hook.timeout,
+          `Shutdown hook '${hook.name}' timeout after ${hook.timeout}ms`
+        );
+        serverLogger.info(`Shutdown hook [${hook.name}] completed`);
+      } catch (error) {
+        serverLogger.error(
+          `Shutdown hook [${hook.name}] failed`,
+          error
+        );
+      }
+    }
+  }
+};
+var instance = null;
+function getShutdownManager() {
+  if (!instance) {
+    instance = new ShutdownManager();
+  }
+  return instance;
+}
+function resetShutdownManager() {
+  instance = null;
+}
+function sleep(ms) {
+  return new Promise((resolve2) => setTimeout(resolve2, ms));
+}
+async function withTimeout(promise, timeout, message) {
+  let timeoutId;
+  return Promise.race([
+    promise.finally(() => {
+      if (timeoutId) clearTimeout(timeoutId);
+    }),
+    new Promise((_, reject) => {
+      timeoutId = setTimeout(() => {
+        reject(new Error(message));
+      }, timeout);
+    })
+  ]);
+}
+
+// src/server/helpers.ts
 function createHealthCheckHandler(detailed) {
   return async (c) => {
+    const shutdownManager = getShutdownManager();
+    if (shutdownManager.isShuttingDown()) {
+      return c.json({
+        status: "shutting_down",
+        timestamp: (/* @__PURE__ */ new Date()).toISOString()
+      }, 503);
+    }
     const response = {
       status: "ok",
       timestamp: (/* @__PURE__ */ new Date()).toISOString()
@@ -695,7 +918,6 @@ function buildStartupConfig(config, timeouts) {
     }
   };
 }
-var serverLogger = logger.child("@spfn/core:server");
 
 // src/server/create-server.ts
 async function createServer(config) {
@@ -743,7 +965,7 @@ async function createAutoConfiguredApp(config) {
   registerSSEEndpoint(app, config);
   await executeAfterRoutesHook(app, config);
   if (enableErrorHandler) {
-    app.onError(ErrorHandler());
+    app.onError(ErrorHandler({ onError: config?.middleware?.onError }));
   }
   return app;
 }
@@ -1162,11 +1384,6 @@ async function startServer(config) {
       config: finalConfig,
       close: async () => {
         serverLogger.info("Manual server shutdown requested");
-        if (shutdownState.isShuttingDown) {
-          serverLogger.warn("Shutdown already in progress, ignoring manual close request");
-          return;
-        }
-        shutdownState.isShuttingDown = true;
         await shutdownServer();
       }
     };
@@ -1310,58 +1527,67 @@ function createShutdownHandler(server, config, shutdownState) {
       return;
     }
     shutdownState.isShuttingDown = true;
-    serverLogger.debug("Closing HTTP server...");
-    let timeoutId;
-    await Promise.race([
-      new Promise((resolve2, reject) => {
-        server.close((err) => {
-          if (timeoutId) clearTimeout(timeoutId);
-          if (err) {
-            serverLogger.error("HTTP server close error", err);
-            reject(err);
-          } else {
-            serverLogger.info("HTTP server closed");
-            resolve2();
-          }
-        });
-      }),
-      new Promise((_, reject) => {
-        timeoutId = setTimeout(() => {
-          reject(new Error(`HTTP server close timeout after ${TIMEOUTS.SERVER_CLOSE}ms`));
-        }, TIMEOUTS.SERVER_CLOSE);
-      })
-    ]).catch((error) => {
-      if (timeoutId) clearTimeout(timeoutId);
-      serverLogger.warn("HTTP server close timeout, forcing shutdown", error);
-    });
+    const shutdownTimeout = getShutdownTimeout(config.shutdown);
+    const shutdownManager = getShutdownManager();
+    shutdownManager.beginShutdown();
+    serverLogger.info("Phase 1: Closing HTTP server (stop accepting new connections)...");
+    await closeHttpServer(server);
     if (config.jobs) {
-      serverLogger.debug("Stopping pg-boss...");
+      serverLogger.info("Phase 2: Stopping pg-boss...");
       try {
         await stopBoss();
+        serverLogger.info("pg-boss stopped");
       } catch (error) {
         serverLogger.error("pg-boss stop failed", error);
       }
     }
+    const drainTimeout = Math.floor(shutdownTimeout * 0.8);
+    serverLogger.info(`Phase 3: Draining tracked operations (timeout: ${drainTimeout}ms)...`);
+    await shutdownManager.execute(drainTimeout);
     if (config.lifecycle?.beforeShutdown) {
-      serverLogger.debug("Executing beforeShutdown hook...");
+      serverLogger.info("Phase 4: Executing beforeShutdown lifecycle hook...");
       try {
         await config.lifecycle.beforeShutdown();
       } catch (error) {
-        serverLogger.error("beforeShutdown hook failed", error);
+        serverLogger.error("beforeShutdown lifecycle hook failed", error);
       }
     }
+    serverLogger.info("Phase 5: Closing infrastructure...");
     const infraConfig = getInfrastructureConfig(config);
     if (infraConfig.database) {
-      serverLogger.debug("Closing database connections...");
       await closeInfrastructure(closeDatabase, "Database", TIMEOUTS.DATABASE_CLOSE);
     }
     if (infraConfig.redis) {
-      serverLogger.debug("Closing Redis connections...");
       await closeInfrastructure(closeCache, "Redis", TIMEOUTS.REDIS_CLOSE);
     }
     serverLogger.info("Server shutdown completed");
   };
 }
+async function closeHttpServer(server) {
+  let timeoutId;
+  await Promise.race([
+    new Promise((resolve2, reject) => {
+      server.close((err) => {
+        if (timeoutId) clearTimeout(timeoutId);
+        if (err) {
+          serverLogger.error("HTTP server close error", err);
+          reject(err);
+        } else {
+          serverLogger.info("HTTP server closed");
+          resolve2();
+        }
+      });
+    }),
+    new Promise((_, reject) => {
+      timeoutId = setTimeout(() => {
+        reject(new Error(`HTTP server close timeout after ${TIMEOUTS.SERVER_CLOSE}ms`));
+      }, TIMEOUTS.SERVER_CLOSE);
+    })
+  ]).catch((error) => {
+    if (timeoutId) clearTimeout(timeoutId);
+    serverLogger.warn("HTTP server close timeout, forcing shutdown", error);
+  });
+}
 async function closeInfrastructure(closeFn, name, timeout) {
   let timeoutId;
   try {
@@ -1484,6 +1710,7 @@ async function cleanupOnFailure(config) {
     if (infraConfig.redis) {
       await closeInfrastructure(closeCache, "Redis", TIMEOUTS.REDIS_CLOSE);
     }
+    resetShutdownManager();
     serverLogger.debug("Cleanup completed");
   } catch (cleanupError) {
     serverLogger.error("Cleanup failed", cleanupError);
@@ -1767,6 +1994,6 @@ function defineServerConfig() {
   return new ServerConfigBuilder();
 }
 
-export { createServer, defineServerConfig, loadEnv, loadEnvFiles, startServer };
+export { createServer, defineServerConfig, getShutdownManager, loadEnv, loadEnvFiles, startServer };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map