@adaptic/backend-legacy 0.0.916 → 0.0.918

package/prismaClient.cjs

@@ -1,7 +1,18 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.startConnectionHealthMonitor = startConnectionHealthMonitor;
+exports.stopConnectionHealthMonitor = stopConnectionHealthMonitor;
+exports.disconnectWithTimeout = disconnectWithTimeout;
 const client_1 = require("@prisma/client");
 const logger_1 = require("./utils/logger");
+/** Default statement timeout (30s) prevents hung queries from blocking pool slots indefinitely */
+const DEFAULT_STATEMENT_TIMEOUT_MS = 30000;
+/** Heartbeat interval: how often we verify the DB connection is alive */
+const HEARTBEAT_INTERVAL_MS = 30000;
+/** Consecutive heartbeat failures before triggering client reconnection */
+const MAX_HEARTBEAT_FAILURES = 3;
+/** Heartbeat query timeout (5s) — shorter than pool timeout to detect issues early */
+const HEARTBEAT_TIMEOUT_MS = 5000;
 /**
  * Connection pool size defaults per deployment tier.
  * Can be overridden via the DATABASE_POOL_SIZE environment variable.
@@ -78,7 +89,8 @@ function buildDatabaseUrl() {
         poolTimeoutMs: poolTimeout,
         environment: process.env.NODE_ENV || 'development',
     });
-    return `${baseUrl}${separator}connection_limit=${poolSize}&pool_timeout=${Math.floor(poolTimeout / 1000)}`;
+    const statementTimeoutMs = parseInt(process.env.DATABASE_STATEMENT_TIMEOUT_MS || String(DEFAULT_STATEMENT_TIMEOUT_MS), 10);
+    return `${baseUrl}${separator}connection_limit=${poolSize}&pool_timeout=${Math.floor(poolTimeout / 1000)}&statement_timeout=${statementTimeoutMs}&idle_in_transaction_session_timeout=${statementTimeoutMs}`;
 }
 // Create a singleton that works in all environments
 if (!global.prisma) {
@@ -148,5 +160,142 @@ if (!global.prisma) {
 }
 // Initialize a singleton PrismaClient with a connection pool that persists across requests
 const prisma = global.prisma;
+// ---------------------------------------------------------------------------
+// Connection health monitor — detects stale connections before they cause
+// query hangs by periodically running a lightweight heartbeat query.
+// After MAX_HEARTBEAT_FAILURES consecutive failures, the client is
+// disconnected and reconnected to force Prisma to establish fresh connections.
+// ---------------------------------------------------------------------------
+let heartbeatFailures = 0;
+let heartbeatTimer;
+let isReconnecting = false;
+async function heartbeat() {
+    if (isReconnecting || !global.prisma)
+        return;
+    try {
+        const start = Date.now();
+        await Promise.race([
+            global.prisma.$queryRaw `SELECT 1`,
+            new Promise((_, reject) => setTimeout(() => reject(new Error('Heartbeat timeout')), HEARTBEAT_TIMEOUT_MS)),
+        ]);
+        const latencyMs = Date.now() - start;
+        if (heartbeatFailures > 0) {
+            logger_1.logger.info('Database heartbeat recovered', { latencyMs, previousFailures: heartbeatFailures });
+        }
+        heartbeatFailures = 0;
+    }
+    catch (error) {
+        heartbeatFailures++;
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        logger_1.logger.warn('Database heartbeat failed', {
+            consecutiveFailures: heartbeatFailures,
+            error: errorMessage,
+        });
+        if (heartbeatFailures >= MAX_HEARTBEAT_FAILURES) {
+            logger_1.logger.error('Database heartbeat exceeded failure threshold, reconnecting Prisma client', {
+                consecutiveFailures: heartbeatFailures,
+            });
+            await reconnectPrisma();
+        }
+    }
+}
+async function reconnectPrisma() {
+    var _a;
+    if (isReconnecting)
+        return;
+    isReconnecting = true;
+    try {
+        // Disconnect existing client — this drops all pooled connections
+        try {
+            await Promise.race([
+                (_a = global.prisma) === null || _a === void 0 ? void 0 : _a.$disconnect(),
+                new Promise((resolve) => setTimeout(resolve, 5000)),
+            ]);
+        }
+        catch (_b) {
+            // Disconnect may fail if DB is unreachable — that's expected
+        }
+        // Create a fresh client with the same configuration
+        const newClient = new client_1.PrismaClient({
+            log: [
+                { level: 'error', emit: 'event' },
+                { level: 'warn', emit: 'event' },
+            ],
+            datasources: {
+                db: {
+                    url: buildDatabaseUrl(),
+                },
+            },
+        });
+        // Re-register error/warn handlers
+        newClient.$on('error', (e) => {
+            logger_1.logger.error('Prisma client error (reconnected)', { message: e.message, timestamp: e.timestamp });
+        });
+        newClient.$on('warn', (e) => {
+            logger_1.logger.warn('Prisma client warning (reconnected)', { message: e.message });
+        });
+        global.prisma = newClient;
+        heartbeatFailures = 0;
+        logger_1.logger.info('Prisma client reconnected successfully');
+    }
+    catch (error) {
+        logger_1.logger.error('Failed to reconnect Prisma client', {
+            error: error instanceof Error ? error.message : String(error),
+        });
+    }
+    finally {
+        isReconnecting = false;
+    }
+}
+/**
+ * Starts the periodic database heartbeat monitor.
+ * Call once during server startup. The returned cleanup function
+ * must be called during graceful shutdown.
+ */
+function startConnectionHealthMonitor() {
+    if (heartbeatTimer)
+        return () => stopConnectionHealthMonitor();
+    logger_1.logger.info('Starting database connection health monitor', {
+        intervalMs: HEARTBEAT_INTERVAL_MS,
+        failureThreshold: MAX_HEARTBEAT_FAILURES,
+    });
+    heartbeatTimer = setInterval(() => {
+        void heartbeat();
+    }, HEARTBEAT_INTERVAL_MS);
+    // Run an initial heartbeat immediately
+    void heartbeat();
+    return () => stopConnectionHealthMonitor();
+}
+/**
+ * Stops the periodic heartbeat monitor. Safe to call multiple times.
+ */
+function stopConnectionHealthMonitor() {
+    if (heartbeatTimer) {
+        clearInterval(heartbeatTimer);
+        heartbeatTimer = undefined;
+        logger_1.logger.info('Database connection health monitor stopped');
+    }
+}
+/**
+ * Disconnects the Prisma client with a timeout to prevent hanging during shutdown.
+ * @param timeoutMs - Maximum time to wait for disconnect (default: 5000ms)
+ */
+async function disconnectWithTimeout(timeoutMs = 5000) {
+    stopConnectionHealthMonitor();
+    if (!global.prisma)
+        return;
+    try {
+        await Promise.race([
+            global.prisma.$disconnect(),
+            new Promise((_, reject) => setTimeout(() => reject(new Error(`Prisma disconnect timed out after ${timeoutMs}ms`)), timeoutMs)),
+        ]);
+        logger_1.logger.info('Database connections closed successfully');
+    }
+    catch (error) {
+        logger_1.logger.warn('Database disconnect did not complete cleanly', {
+            error: error instanceof Error ? error.message : String(error),
+        });
+    }
+}
 exports.default = prisma;
 //# sourceMappingURL=prismaClient.js.map

package/prismaClient.d.ts

@@ -6,5 +6,20 @@ declare global {
     var prisma: PrismaClient | undefined;
 }
 declare const prisma: PrismaClient;
+/**
+ * Starts the periodic database heartbeat monitor.
+ * Call once during server startup. The returned cleanup function
+ * must be called during graceful shutdown.
+ */
+export declare function startConnectionHealthMonitor(): () => void;
+/**
+ * Stops the periodic heartbeat monitor. Safe to call multiple times.
+ */
+export declare function stopConnectionHealthMonitor(): void;
+/**
+ * Disconnects the Prisma client with a timeout to prevent hanging during shutdown.
+ * @param timeoutMs - Maximum time to wait for disconnect (default: 5000ms)
+ */
+export declare function disconnectWithTimeout(timeoutMs?: number): Promise<void>;
 export default prisma;
 //# sourceMappingURL=prismaClient.d.ts.map

package/server.cjs

@@ -1,4 +1,37 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -22,10 +55,11 @@ const jsonwebtoken_1 = __importDefault(require("jsonwebtoken"));
 const auth_1 = require("./middleware/auth.cjs");
 const audit_logger_1 = require("./middleware/audit-logger.cjs");
 const jwtConfig_1 = require("./config/jwtConfig.cjs");
-const prismaClient_1 = __importDefault(require("./prismaClient.cjs"));
+const prismaClient_1 = __importStar(require("./prismaClient.cjs"));
 const health_1 = require("./health.cjs");
 const child_process_1 = require("child_process");
 const logger_1 = require("./utils/logger.cjs");
+const tracing_1 = require("./config/tracing.cjs");
 let dbUnreachableCount = 0;
 let lastRestartAttempt = 0;
 async function restartDatabase() {
@@ -82,37 +116,14 @@ const startServer = async () => {
                 // If we've hit 3 (for example) attempts
                 if (dbUnreachableCount >= 3) {
                     const now = Date.now();
-                    // Optional: Check if we've tried restarting recently
                     if (now - lastRestartAttempt > 5 * 60 * 1000) {
                         lastRestartAttempt = now;
-                        try {
-                            void restartDatabase();
-                            // Reset the counter after attempting a restart
-                            dbUnreachableCount = 0;
-                        }
-                        catch (restartError) {
+                        dbUnreachableCount = 0;
+                        restartDatabase().catch((restartError) => {
                             logger_1.logger.error('Error trying to restart DB', {
                                 restartError: String(restartError),
                             });
-                            // If the restart fails, we can try again after a delay
-                            const backoffTime = Math.min(30000, 1000 * Math.pow(2, dbUnreachableCount - 3)); // Exponential backoff with a max of 30 seconds
-                            logger_1.logger.info('Waiting before next restart attempt', {
-                                backoffSeconds: backoffTime / 1000,
-                            });
-                            setTimeout(() => {
-                                restartDatabase()
-                                    .then(() => {
-                                    dbUnreachableCount = 0; // Reset the counter after a successful restart
-                                })
-                                    .catch((restartError) => {
-                                    logger_1.logger.error('Error trying to restart DB', {
-                                        restartError: String(restartError),
-                                    });
-                                    // We do not reset the counter here if the restart fails,
-                                    // so it can try again next time.
-                                });
-                            }, backoffTime);
-                        }
+                        });
                     }
                 }
             }
@@ -284,6 +295,8 @@ const startServer = async () => {
             return { prisma: global.prisma, user };
         },
     }, wsServer);
+    // Start the periodic database connection health monitor
+    (0, prismaClient_1.startConnectionHealthMonitor)();
     const PORT = process.env.PORT || 4000;
     httpServer.listen(PORT, () => {
         logger_1.logger.info('Server ready', {
@@ -294,6 +307,56 @@ const startServer = async () => {
             endpoint: `ws://localhost:${PORT}/subscriptions`,
         });
     });
+    // --- Graceful shutdown ---
+    // Sequence: stop accepting connections → drain in-flight requests →
+    // close WebSocket → disconnect DB → flush traces → exit
+    let isShuttingDown = false;
+    const SHUTDOWN_TIMEOUT_MS = 15000;
+    async function gracefulShutdown(signal) {
+        if (isShuttingDown)
+            return;
+        isShuttingDown = true;
+        logger_1.logger.info(`Received ${signal}, starting graceful shutdown`);
+        // Force exit if shutdown takes too long
+        const forceExitTimer = setTimeout(() => {
+            logger_1.logger.error('Graceful shutdown timed out, forcing exit');
+            process.exit(1);
+        }, SHUTDOWN_TIMEOUT_MS);
+        forceExitTimer.unref();
+        try {
+            // 1. Stop accepting new HTTP connections
+            await new Promise((resolve) => {
+                httpServer.close(() => {
+                    logger_1.logger.info('HTTP server closed');
+                    resolve();
+                });
+            });
+            // 2. Close WebSocket server (rejects new WS connections)
+            await new Promise((resolve) => {
+                wsServer.close(() => {
+                    logger_1.logger.info('WebSocket server closed');
+                    resolve();
+                });
+            });
+            // 3. Apollo Server drain is handled by ApolloServerPluginDrainHttpServer
+            // 4. Stop health monitor and disconnect DB with timeout
+            await (0, prismaClient_1.disconnectWithTimeout)(5000);
+            // 5. Flush OpenTelemetry traces
+            await (0, tracing_1.shutdownTracing)();
+            logger_1.logger.info('Graceful shutdown complete');
+        }
+        catch (error) {
+            logger_1.logger.error('Error during graceful shutdown', {
+                error: error instanceof Error ? error.message : String(error),
+            });
+        }
+        finally {
+            clearTimeout(forceExitTimer);
+            process.exit(0);
+        }
+    }
+    process.on('SIGINT', () => { void gracefulShutdown('SIGINT'); });
+    process.on('SIGTERM', () => { void gracefulShutdown('SIGTERM'); });
 };
 startServer().catch((error) => {
     logger_1.logger.error('Error starting the server', {
@@ -312,34 +375,4 @@ process.on('uncaughtException', (error) => {
         stack: error.stack,
     });
 });
-// Only disconnect Prisma when the process is truly shutting down
-process.on('SIGINT', async () => {
-    var _a;
-    logger_1.logger.info('Gracefully shutting down and closing database connections');
-    try {
-        await ((_a = global.prisma) === null || _a === void 0 ? void 0 : _a.$disconnect());
-        logger_1.logger.info('Database connections closed successfully');
-    }
-    catch (e) {
-        logger_1.logger.error('Error disconnecting from database', {
-            error: e instanceof Error ? e.message : String(e),
-        });
-    }
-    process.exit(0);
-});
-// Also handle SIGTERM for containerized environments
-process.on('SIGTERM', async () => {
-    var _a;
-    logger_1.logger.info('Received SIGTERM, gracefully shutting down');
-    try {
-        await ((_a = global.prisma) === null || _a === void 0 ? void 0 : _a.$disconnect());
-        logger_1.logger.info('Database connections closed successfully');
-    }
-    catch (e) {
-        logger_1.logger.error('Error disconnecting from database', {
-            error: e instanceof Error ? e.message : String(e),
-        });
-    }
-    process.exit(0);
-});
 //# sourceMappingURL=server.js.map