@spfn/core 0.2.0-beta.43 → 0.2.0-beta.45

package/dist/db/index.d.ts

@@ -273,6 +273,36 @@ declare function initDatabase(options?: DatabaseOptions): Promise<{
  * ```
  */
 declare function closeDatabase(): Promise<void>;
+/**
+ * Force an immediate database pool rebuild
+ *
+ * Destroys the current postgres.js pool(s) and rebuilds them with the same
+ * configuration passed to the original `initDatabase()` call (or whatever
+ * was detected from environment variables). Uses the same atomic-swap
+ * strategy as the periodic health check: new connections are created and
+ * tested BEFORE the old ones are torn down, so `getDatabase()` callers never
+ * observe a missing instance.
+ *
+ * Use this when application code detects that the pool is stuck and does not
+ * want to wait for the next periodic health check tick. Concurrent calls are
+ * coalesced — if a reconnect is already in progress, this resolves to `false`
+ * without starting a second one.
+ *
+ * @param reason - Short label describing why the rebuild was requested (for logs)
+ * @returns `true` if a reconnection ran, `false` if one was already in-flight.
+ *          Resolves after the rebuild completes (success or max retries exhausted).
+ *
+ * @example
+ * ```typescript
+ * import { forceReconnectDatabase } from '@spfn/core/db';
+ *
+ * app.post('/admin/db/reconnect', async (c) => {
+ *     const ran = await forceReconnectDatabase('admin_request');
+ *     return c.json({ reconnected: ran });
+ * });
+ * ```
+ */
+declare function forceReconnectDatabase(reason?: string): Promise<boolean>;
 /**
  * Get database connection info (for debugging)
  *
@@ -308,6 +338,60 @@ declare function getDatabaseInfo(): {
     isReplica: boolean;
 };
 
+/**
+ * Reconnect Trigger — Query-error driven pool rebuild
+ *
+ * Complements the periodic health check with a fast-path: when application
+ * queries start failing with connection-level errors, we do not wait up to
+ * DB_HEALTH_CHECK_INTERVAL (default 60s) to notice. A sliding-window counter
+ * trips a force-reconnect as soon as the failure rate crosses a threshold.
+ *
+ * Why this exists:
+ *  - postgres.js transparently drops dead sockets and opens new ones on the
+ *    next query. A single `SELECT 1` on the periodic interval can therefore
+ *    false-pass while user-facing queries keep hitting the remaining dead
+ *    sockets in the pool.
+ *  - This module observes real query errors and, when it sees a burst of
+ *    connection-level failures, calls triggerForceReconnect() which performs
+ *    the same atomic-swap rebuild as the health check.
+ *
+ * Configuration (env vars, hardcoded defaults):
+ *  - DB_RECONNECT_ERROR_THRESHOLD (default 3): errors needed in window
+ *  - DB_RECONNECT_ERROR_WINDOW_MS (default 10000): sliding window size
+ */
+/**
+ * Determine whether an error looks like a pool/connection failure
+ *
+ * Returns true when any layer in the error chain exposes a connection-level
+ * code (postgres.js driver code, Node network errno, PG SQLSTATE class 08 etc.)
+ * or is an instance of our own ConnectionError wrapper.
+ *
+ * Returns false for query errors (syntax, constraint violations, etc.) — those
+ * should NOT trigger a pool rebuild.
+ */
+declare function isConnectionLevelError(error: unknown): boolean;
+/**
+ * Reset the internal error counter
+ *
+ * Exposed for tests that need a clean slate between cases. Does not clear
+ * the WeakSet (which is GC-backed and self-cleans with error lifetimes).
+ */
+declare function resetConnectionErrorCounter(): void;
+/**
+ * Report a database error to the reconnect trigger
+ *
+ * Call this from any site that catches a query error before rethrowing.
+ * It is a no-op for non-connection-level errors. When the threshold is
+ * crossed it calls triggerForceReconnect() in the background — callers
+ * should NOT await it.
+ *
+ * Safe to call from any context: catches its own errors so it cannot
+ * disrupt the calling catch block. Deduplicates across error-chain
+ * re-wrapping so one failure counts exactly once regardless of how many
+ * catch layers it passes through.
+ */
+declare function reportDatabaseError(error: unknown): void;
+
 /**
  * Create database connection with exponential backoff retry strategy
  *
@@ -1674,4 +1758,4 @@ declare abstract class BaseRepository<TSchema extends Record<string, unknown> =
     protected _count<T extends PgTable>(table: T, where?: Record<string, any> | SQL | undefined): Promise<number>;
 }
 
-export { type AfterCommitCallback, BaseRepository, type DatabaseClients, type DrizzleConfigOptions, type PoolConfig, RepositoryError, type RetryConfig, type RunInTransactionOptions, type TransactionContext, type TransactionDB, Transactional, type TransactionalOptions, auditFields, checkConnection, closeDatabase, count, create, createDatabaseConnection, createDatabaseFromEnv, createMany, createSchema, deleteMany, deleteOne, detectDialect, enumText, findMany, findOne, foreignKey, fromPostgresError, generateDrizzleConfigFile, getDatabase, getDatabaseInfo, getDrizzleConfig, getSchemaInfo, getTransaction, id, initDatabase, onAfterCommit, optionalForeignKey, packageNameToSchema, publishingFields, runInTransaction, runWithTransaction, setDatabase, softDelete, timestamps, typedJsonb, updateMany, updateOne, upsert, utcTimestamp, uuid, verificationTimestamp };
+export { type AfterCommitCallback, BaseRepository, type DatabaseClients, type DrizzleConfigOptions, type PoolConfig, RepositoryError, type RetryConfig, type RunInTransactionOptions, type TransactionContext, type TransactionDB, Transactional, type TransactionalOptions, auditFields, checkConnection, closeDatabase, count, create, createDatabaseConnection, createDatabaseFromEnv, createMany, createSchema, deleteMany, deleteOne, detectDialect, enumText, findMany, findOne, forceReconnectDatabase, foreignKey, fromPostgresError, generateDrizzleConfigFile, getDatabase, getDatabaseInfo, getDrizzleConfig, getSchemaInfo, getTransaction, id, initDatabase, isConnectionLevelError, onAfterCommit, optionalForeignKey, packageNameToSchema, publishingFields, reportDatabaseError, resetConnectionErrorCounter, runInTransaction, runWithTransaction, setDatabase, softDelete, timestamps, typedJsonb, updateMany, updateOne, upsert, utcTimestamp, uuid, verificationTimestamp };

package/dist/db/index.js

@@ -520,9 +520,20 @@ var setHealthCheckInterval = (interval) => {
 var setMonitoringConfig = (config) => {
   globalThis.__SPFN_DB_MONITORING__ = config;
 };
+var getInitOptions = () => globalThis.__SPFN_DB_INIT_OPTIONS__;
+var setInitOptions = (options) => {
+  globalThis.__SPFN_DB_INIT_OPTIONS__ = options;
+};
+var getIsClosing = () => globalThis.__SPFN_DB_CLOSING__ === true;
+var setIsClosing = (closing) => {
+  globalThis.__SPFN_DB_CLOSING__ = closing;
+};
 var dbLogger3 = logger.child("@spfn/core:database");
 var CLIENT_CLOSE_TIMEOUT = 5;
 var isReconnecting = false;
+function isReconnectingNow() {
+  return isReconnecting;
+}
 async function testDatabaseConnection(db) {
   await db.execute("SELECT 1");
 }
@@ -541,6 +552,10 @@ async function closeClient(client) {
   }
 }
 async function reconnectAndRestore(options) {
+  if (getIsClosing()) {
+    dbLogger3.debug("reconnectAndRestore aborted: database is closing");
+    return false;
+  }
   const result = await createDatabaseFromEnv(options);
   if (!result.write) {
     return false;
@@ -549,6 +564,16 @@ async function reconnectAndRestore(options) {
   if (result.read && result.read !== result.write) {
     await testDatabaseConnection(result.read);
   }
+  if (getIsClosing()) {
+    dbLogger3.warn("reconnectAndRestore: close started mid-rebuild, discarding new pool");
+    if (result.writeClient) {
+      await closeClient(result.writeClient);
+    }
+    if (result.readClient && result.readClient !== result.writeClient) {
+      await closeClient(result.readClient);
+    }
+    return false;
+  }
   const oldWriteClient = getWriteClient();
   const oldReadClient = getReadClient();
   setWriteInstance(result.write);
@@ -586,15 +611,34 @@ function startHealthCheck(config, options, getDatabase2) {
       const message = error instanceof Error ? error.message : "Unknown error";
       dbLogger3.error("Database health check failed", { error: message });
       if (config.reconnect) {
-        await attemptReconnection(config, options);
+        await attemptReconnection(config, options, "health_check_failed");
       }
     }
   }, config.interval);
   setHealthCheckInterval(interval);
 }
-async function attemptReconnection(config, options) {
+async function triggerForceReconnect(reason) {
+  if (!getWriteInstance()) {
+    dbLogger3.warn("Force reconnect skipped: database not initialized", { reason });
+    return false;
+  }
+  if (getIsClosing()) {
+    dbLogger3.debug("Force reconnect skipped: database is closing", { reason });
+    return false;
+  }
+  const options = getInitOptions();
+  const config = buildHealthCheckConfig(options?.healthCheck);
+  dbLogger3.warn("Force reconnect triggered", { reason });
+  return await attemptReconnection(config, options, reason);
+}
+async function attemptReconnection(config, options, reason) {
+  if (isReconnecting) {
+    dbLogger3.debug("Reconnection coalesced: attempt already in progress", { reason });
+    return false;
+  }
   isReconnecting = true;
   dbLogger3.warn("Attempting database reconnection", {
+    reason,
     maxRetries: config.maxRetries,
     retryInterval: `${config.retryInterval}ms`
   });
@@ -608,7 +652,7 @@ async function attemptReconnection(config, options) {
         const success = await reconnectAndRestore(options);
         if (success) {
           dbLogger3.info("Database reconnection successful", { attempt });
-          return;
+          return true;
         } else {
           dbLogger3.error(`Reconnection attempt ${attempt} failed: No write database instance created`);
         }
@@ -627,6 +671,7 @@ async function attemptReconnection(config, options) {
   } finally {
     isReconnecting = false;
   }
+  return true;
 }
 function stopHealthCheck() {
   const healthCheck = getHealthCheckInterval();
@@ -647,7 +692,6 @@ var STACK_TRACE_PATTERNS = {
   withoutParens: /at (.+):(\d+):(\d+)/
 };
 var initPromise = null;
-var isClosing = false;
 async function cleanupDatabaseConnections(writeClient, readClient) {
   const cleanupPromises = [];
   if (writeClient) {
@@ -748,7 +792,7 @@ function setDatabase(write, read) {
   setReadInstance(read ?? write);
 }
 async function initDatabase(options) {
-  if (isClosing) {
+  if (getIsClosing()) {
     throw new Error("Cannot initialize database while closing");
   }
   const writeInst = getWriteInstance();
@@ -770,7 +814,7 @@ async function initDatabase(options) {
         const message = error instanceof Error ? error.message : "Unknown error";
         throw new Error(`Database connection test failed: ${message}`);
       }
-      if (isClosing) {
+      if (getIsClosing()) {
         dbLogger4.warn("Database closed during initialization, cleaning up...");
         await cleanupDatabaseConnections(result.writeClient, result.readClient);
         throw new Error("Database closed during initialization");
@@ -779,6 +823,7 @@ async function initDatabase(options) {
       setReadInstance(result.read);
       setWriteClient(result.writeClient);
       setReadClient(result.readClient);
+      setInitOptions(options);
       const hasReplica = result.read && result.read !== result.write;
       dbLogger4.info(
         hasReplica ? "Database connected (Primary + Replica)" : "Database connected"
@@ -803,11 +848,11 @@ async function initDatabase(options) {
   return await initPromise;
 }
 async function closeDatabase() {
-  if (isClosing) {
+  if (getIsClosing()) {
     dbLogger4.debug("Database close already in progress");
     return;
   }
-  isClosing = true;
+  setIsClosing(true);
   if (initPromise) {
     dbLogger4.debug("Waiting for database initialization to complete before closing...");
     try {
@@ -820,7 +865,7 @@ async function closeDatabase() {
   const readInst = getReadInstance();
   if (!writeInst && !readInst) {
     dbLogger4.debug("No database connections to close");
-    isClosing = false;
+    setIsClosing(false);
     return;
   }
   try {
@@ -842,9 +887,13 @@ async function closeDatabase() {
     setWriteClient(void 0);
     setReadClient(void 0);
     setMonitoringConfig(void 0);
-    isClosing = false;
+    setInitOptions(void 0);
+    setIsClosing(false);
   }
 }
+async function forceReconnectDatabase(reason = "manual") {
+  return await triggerForceReconnect(reason);
+}
 function getDatabaseInfo() {
   const writeInst = getWriteInstance();
   const readInst = getReadInstance();
@@ -854,6 +903,122 @@ function getDatabaseInfo() {
     isReplica: !!(readInst && readInst !== writeInst)
   };
 }
+var dbLogger5 = logger.child("@spfn/core:database");
+var POSTGRES_JS_CONNECTION_CODES = /* @__PURE__ */ new Set([
+  "CONNECTION_ENDED",
+  "CONNECTION_CLOSED",
+  "CONNECTION_DESTROYED",
+  "CONNECT_TIMEOUT",
+  "CONNECTION_CONNECT_TIMEOUT"
+]);
+var NODE_NET_ERROR_CODES = /* @__PURE__ */ new Set([
+  "ECONNRESET",
+  "ECONNREFUSED",
+  "EPIPE",
+  "ETIMEDOUT",
+  "EHOSTUNREACH",
+  "ENETUNREACH",
+  "ENOTFOUND"
+]);
+function isConnectionSqlState(code) {
+  if (code.startsWith("08")) return true;
+  if (code === "53300") return true;
+  if (code === "57P01" || code === "57P02" || code === "57P03") return true;
+  return false;
+}
+function* unwrap(error) {
+  const seen = /* @__PURE__ */ new Set();
+  const stack = [error];
+  while (stack.length > 0) {
+    const current = stack.pop();
+    if (!current || typeof current !== "object" || seen.has(current)) {
+      continue;
+    }
+    seen.add(current);
+    const obj = current;
+    yield obj;
+    for (const key of ["cause", "original", "error", "err", "inner"]) {
+      const nested = obj[key];
+      if (nested && typeof nested === "object" && !seen.has(nested)) {
+        stack.push(nested);
+      }
+    }
+  }
+}
+function isConnectionLevelError(error) {
+  if (!error) return false;
+  if (error instanceof ConnectionError) return true;
+  for (const candidate of unwrap(error)) {
+    if (candidate instanceof ConnectionError) return true;
+    const code = candidate.code;
+    if (typeof code === "string") {
+      if (POSTGRES_JS_CONNECTION_CODES.has(code)) return true;
+      if (NODE_NET_ERROR_CODES.has(code)) return true;
+      if (isConnectionSqlState(code)) return true;
+    }
+  }
+  return false;
+}
+var DEFAULT_THRESHOLD = 3;
+var DEFAULT_WINDOW_MS = 1e4;
+var MIN_WINDOW_MS = 1e3;
+function readPositiveIntEnv(key, defaultValue, min) {
+  const raw = process.env[key];
+  if (raw === void 0) return defaultValue;
+  try {
+    return parseNumber(raw, { min, integer: true });
+  } catch {
+    return defaultValue;
+  }
+}
+var ERROR_THRESHOLD = readPositiveIntEnv("DB_RECONNECT_ERROR_THRESHOLD", DEFAULT_THRESHOLD, 1);
+var ERROR_WINDOW_MS = readPositiveIntEnv("DB_RECONNECT_ERROR_WINDOW_MS", DEFAULT_WINDOW_MS, MIN_WINDOW_MS);
+var errorTimestamps = [];
+var reportedErrors = /* @__PURE__ */ new WeakSet();
+function resetConnectionErrorCounter() {
+  errorTimestamps.length = 0;
+}
+function checkAndMarkReported(error) {
+  let alreadySeen = false;
+  const toMark = [];
+  for (const candidate of unwrap(error)) {
+    if (reportedErrors.has(candidate)) {
+      alreadySeen = true;
+    }
+    toMark.push(candidate);
+  }
+  if (alreadySeen) return true;
+  for (const obj of toMark) {
+    reportedErrors.add(obj);
+  }
+  return false;
+}
+function reportDatabaseError(error) {
+  try {
+    if (!isConnectionLevelError(error)) return;
+    if (isReconnectingNow()) return;
+    if (checkAndMarkReported(error)) return;
+    const now = Date.now();
+    errorTimestamps.push(now);
+    const cutoff = now - ERROR_WINDOW_MS;
+    while (errorTimestamps.length > 0 && errorTimestamps[0] < cutoff) {
+      errorTimestamps.shift();
+    }
+    if (errorTimestamps.length < ERROR_THRESHOLD) return;
+    errorTimestamps.length = 0;
+    dbLogger5.error("Connection-error threshold crossed, forcing pool rebuild", {
+      threshold: ERROR_THRESHOLD,
+      windowMs: ERROR_WINDOW_MS
+    });
+    triggerForceReconnect("query_error_threshold").catch((err) => {
+      const message = err instanceof Error ? err.message : String(err);
+      dbLogger5.error("Forced reconnect after error threshold failed", { error: message });
+    });
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    dbLogger5.debug("reportDatabaseError itself threw, ignoring", { error: message });
+  }
+}
 var BARREL_FILE_PATTERNS = [
   "/index",
   "/index.ts",
@@ -1416,6 +1581,7 @@ function Transactional(options = {}) {
         }
       );
     } catch (error) {
+      reportDatabaseError(error);
       if (error instanceof DatabaseError) {
         throw error;
       }
@@ -1663,6 +1829,7 @@ var BaseRepository = class {
     try {
       return await queryFn();
     } catch (error) {
+      reportDatabaseError(error);
       const err = error instanceof Error ? error : new Error(String(error));
       const repositoryName = this.constructor.name;
       throw new RepositoryError(
@@ -1915,6 +2082,6 @@ var BaseRepository = class {
   }
 };
 
-export { BaseRepository, RepositoryError, Transactional, auditFields, checkConnection, closeDatabase, count, create, createDatabaseConnection, createDatabaseFromEnv, createMany, createSchema, deleteMany, deleteOne, detectDialect, enumText, findMany, findOne, foreignKey, fromPostgresError, generateDrizzleConfigFile, getDatabase, getDatabaseInfo, getDrizzleConfig, getSchemaInfo, getTransaction, id, initDatabase, onAfterCommit, optionalForeignKey, packageNameToSchema, publishingFields, runInTransaction, runWithTransaction, setDatabase, softDelete, timestamps, typedJsonb, updateMany, updateOne, upsert, utcTimestamp, uuid, verificationTimestamp };
+export { BaseRepository, RepositoryError, Transactional, auditFields, checkConnection, closeDatabase, count, create, createDatabaseConnection, createDatabaseFromEnv, createMany, createSchema, deleteMany, deleteOne, detectDialect, enumText, findMany, findOne, forceReconnectDatabase, foreignKey, fromPostgresError, generateDrizzleConfigFile, getDatabase, getDatabaseInfo, getDrizzleConfig, getSchemaInfo, getTransaction, id, initDatabase, isConnectionLevelError, onAfterCommit, optionalForeignKey, packageNameToSchema, publishingFields, reportDatabaseError, resetConnectionErrorCounter, runInTransaction, runWithTransaction, setDatabase, softDelete, timestamps, typedJsonb, updateMany, updateOne, upsert, utcTimestamp, uuid, verificationTimestamp };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map