bunsane 0.3.2 → 0.5.0

package/core/health.ts

@@ -1,4 +1,5 @@
 import db from "../database";
+import { runWithSignal } from "../database/cancellable";
 import { CacheManager } from "./cache/CacheManager";
 
 export interface CheckResult {
@@ -13,6 +14,14 @@ export interface HealthResponse {
     checks: {
         database: CheckResult;
         cache: CheckResult;
+        /**
+         * Present only when the DB write probe is enabled (default on).
+         * Exercises the real `db.transaction()` write path so a wedged write
+         * pool — a stuck pooled client or exhausted pool that leaves reads
+         * (`SELECT 1`) healthy — fails the liveness check and the orchestrator
+         * restarts the container instead of it serving 504s indefinitely.
+         */
+        database_write?: CheckResult;
     };
 }
 
@@ -24,6 +33,71 @@ export interface HealthResult {
 export interface HealthDeps {
     pingDb: () => Promise<boolean>;
     pingCache: () => Promise<boolean>;
+    /**
+     * Write-path probe. Optional: when omitted (e.g. tests passing custom
+     * deps) the write check is skipped and behavior matches the read-only
+     * health check. `defaultDeps` supplies the real probe.
+     */
+    pingDbWrite?: () => Promise<boolean>;
+}
+
+// Independent, short timeout for the write probe so a wedged write path is
+// caught fast (and the container restarted) rather than blocking on the 30s
+// request/save timeout. Configurable via DB_HEALTH_WRITE_TIMEOUT.
+const WRITE_PROBE_TIMEOUT_MS = parseInt(process.env.DB_HEALTH_WRITE_TIMEOUT ?? "5000", 10);
+
+function writeProbeDisabled(): boolean {
+    return process.env.HEALTH_DB_WRITE_PROBE === "false";
+}
+
+/**
+ * Exercises a genuine write through the same `db.transaction()` acquisition
+ * path `Entity.save` uses. A wedged write pool (stuck pooled client, pool
+ * exhausted by leaked transactions) hangs here while `SELECT 1` stays healthy
+ * on any idle read connection — exactly the false-healthy scenario that kept a
+ * timed-out container "healthy" and unrestarted.
+ *
+ * The whole transaction is raced against an independent timeout so even a hang
+ * during connection *acquisition* (which runWithSignal alone cannot interrupt,
+ * since it only wraps in-flight queries) is caught. The temp table is dropped
+ * at COMMIT, so the probe has no persistent side effect.
+ */
+async function probeDbWrite(): Promise<boolean> {
+    const timeoutMs = WRITE_PROBE_TIMEOUT_MS;
+    const controller = new AbortController();
+    let handle: ReturnType<typeof setTimeout> | undefined;
+    const timeoutPromise = new Promise<never>((_, reject) => {
+        handle = setTimeout(() => {
+            const err = new Error(`DB write health probe timeout after ${timeoutMs}ms`);
+            controller.abort(err);
+            reject(err);
+        }, timeoutMs);
+        (handle as any).unref?.();
+    });
+
+    const txn = db.transaction(async (trx) => {
+        await runWithSignal(
+            trx`CREATE TEMP TABLE IF NOT EXISTS _bunsane_health_write (probed_at timestamptz NOT NULL) ON COMMIT DROP`,
+            controller.signal,
+        );
+        await runWithSignal(
+            trx`INSERT INTO _bunsane_health_write (probed_at) VALUES (now())`,
+            controller.signal,
+        );
+    });
+
+    try {
+        await Promise.race([txn, timeoutPromise]);
+        return true;
+    } finally {
+        if (handle) clearTimeout(handle);
+        // Abort any in-flight query so the transaction rolls back and the
+        // pooled connection is released even when the timeout won the race.
+        if (!controller.signal.aborted) controller.abort();
+        // Swallow a late transaction settle after a lost race so it cannot
+        // surface as an unhandled rejection.
+        Promise.resolve(txn).catch(() => { /* ignore post-timeout settle */ });
+    }
 }
 
 const defaultDeps: HealthDeps = {
@@ -32,6 +106,7 @@ const defaultDeps: HealthDeps = {
         return true;
     },
     pingCache: () => CacheManager.getInstance().ping(),
+    pingDbWrite: probeDbWrite,
 };
 
 async function checkDatabase(pingDb: () => Promise<boolean>): Promise<CheckResult> {
@@ -55,24 +130,30 @@ async function checkCache(pingCache: () => Promise<boolean>): Promise<CheckResul
 }
 
 export async function deepHealthCheck(deps: HealthDeps = defaultDeps): Promise<HealthResult> {
-    const [database, cache] = await Promise.all([
+    const runWrite = !!deps.pingDbWrite && !writeProbeDisabled();
+
+    const [database, cache, databaseWrite] = await Promise.all([
         checkDatabase(deps.pingDb),
         checkCache(deps.pingCache),
+        runWrite ? checkDatabase(deps.pingDbWrite!) : Promise.resolve(undefined),
     ]);
 
     const dbUp = database.status === "up";
+    const writeUp = !databaseWrite || databaseWrite.status === "up";
     const cacheUp = cache.status === "up";
 
     let status: HealthResponse["status"];
     let httpStatus: number;
 
-    if (dbUp && cacheUp) {
+    if (dbUp && writeUp && cacheUp) {
         status = "ok";
         httpStatus = 200;
-    } else if (dbUp && !cacheUp) {
+    } else if (dbUp && writeUp && !cacheUp) {
         status = "degraded";
         httpStatus = 200;
     } else {
+        // DB read OR write down → unavailable. A wedged write path (reads fine,
+        // writes hang) lands here so liveness fails and the container restarts.
         status = "unavailable";
         httpStatus = 503;
     }
@@ -82,7 +163,11 @@ export async function deepHealthCheck(deps: HealthDeps = defaultDeps): Promise<H
             status,
             timestamp: new Date().toISOString(),
             uptime: process.uptime(),
-            checks: { database, cache },
+            checks: {
+                database,
+                cache,
+                ...(databaseWrite ? { database_write: databaseWrite } : {}),
+            },
         },
         httpStatus,
     };
@@ -94,6 +179,7 @@ export async function readinessCheck(
     deps: HealthDeps = defaultDeps,
 ): Promise<HealthResult> {
     if (!isReady || isShuttingDown) {
+        const includeWrite = !!deps.pingDbWrite && !writeProbeDisabled();
         return {
             result: {
                 status: "unavailable",
@@ -102,6 +188,9 @@ export async function readinessCheck(
                 checks: {
                     database: { status: "unknown", latency_ms: 0 },
                     cache: { status: "unknown", latency_ms: 0 },
+                    ...(includeWrite
+                        ? { database_write: { status: "unknown", latency_ms: 0 } }
+                        : {}),
                 },
             },
             httpStatus: 503,

package/core/hooks/dispatcher.ts

@@ -0,0 +1,439 @@
+import type { LifecycleEvent } from "../events/EntityLifecycleEvents";
+import { logger as MainLogger } from "../Logger";
+import type { RegisteredHook, HookMetrics, RegistryState } from "./registry";
+import { matchesComponentTarget } from "./guards";
+
+const logger = MainLogger.child({ scope: "EntityHookManager" });
+
+/**
+ * Dispatcher state owned by the manager instance
+ */
+export interface DispatcherState {
+    metrics: Map<string, HookMetrics>;
+    globalMetrics: HookMetrics;
+}
+
+/**
+ * Create initial dispatcher state
+ */
+export function createDispatcherState(): DispatcherState {
+    return {
+        metrics: new Map(),
+        globalMetrics: {
+            totalExecutions: 0,
+            totalExecutionTime: 0,
+            averageExecutionTime: 0,
+            errorCount: 0,
+            lastExecutionTime: 0
+        }
+    };
+}
+
+/**
+ * Record hook execution metrics
+ */
+export function recordMetrics(state: DispatcherState, eventType: string, executionTime: number, hadErrors: boolean): void {
+    // Update event-specific metrics
+    let eventMetrics = state.metrics.get(eventType);
+    if (!eventMetrics) {
+        eventMetrics = {
+            totalExecutions: 0,
+            totalExecutionTime: 0,
+            averageExecutionTime: 0,
+            errorCount: 0,
+            lastExecutionTime: 0
+        };
+        state.metrics.set(eventType, eventMetrics);
+    }
+
+    eventMetrics.totalExecutions++;
+    eventMetrics.totalExecutionTime += executionTime;
+    eventMetrics.averageExecutionTime = eventMetrics.totalExecutionTime / eventMetrics.totalExecutions;
+    eventMetrics.lastExecutionTime = executionTime;
+    if (hadErrors) {
+        eventMetrics.errorCount++;
+    }
+
+    // Update global metrics
+    state.globalMetrics.totalExecutions++;
+    state.globalMetrics.totalExecutionTime += executionTime;
+    state.globalMetrics.averageExecutionTime = state.globalMetrics.totalExecutionTime / state.globalMetrics.totalExecutions;
+    state.globalMetrics.lastExecutionTime = executionTime;
+    if (hadErrors) {
+        state.globalMetrics.errorCount++;
+    }
+}
+
+/**
+ * Get performance metrics for hook execution
+ */
+export function getMetrics(state: DispatcherState, eventType?: string): HookMetrics {
+    if (eventType) {
+        return state.metrics.get(eventType) || {
+            totalExecutions: 0,
+            totalExecutionTime: 0,
+            averageExecutionTime: 0,
+            errorCount: 0,
+            lastExecutionTime: 0
+        };
+    }
+    return { ...state.globalMetrics };
+}
+
+/**
+ * Reset performance metrics
+ */
+export function resetMetrics(state: DispatcherState, eventType?: string): void {
+    if (eventType) {
+        state.metrics.delete(eventType);
+    } else {
+        state.metrics.clear();
+        state.globalMetrics = {
+            totalExecutions: 0,
+            totalExecutionTime: 0,
+            averageExecutionTime: 0,
+            errorCount: 0,
+            lastExecutionTime: 0
+        };
+    }
+    logger.trace(`Reset metrics${eventType ? ` for ${eventType}` : ''}`);
+}
+
+/**
+ * Execute hooks for a specific event
+ */
+export async function executeHooks(registryState: RegistryState, dispatcherState: DispatcherState, event: LifecycleEvent): Promise<void> {
+    const eventType = event.getEventType();
+    const hooks = registryState.hooks.get(eventType) || [];
+    const startTime = performance.now();
+    let hadErrors = false;
+
+    if (hooks.length === 0) {
+        return;
+    }
+
+    logger.trace(`Executing ${hooks.length} hooks for event: ${eventType}`);
+
+    // Separate sync and async hooks
+    const syncHooks = hooks.filter(hook => !hook.options.async);
+    const asyncHooks = hooks.filter(hook => hook.options.async);
+
+    // Execute sync hooks immediately
+    for (const hook of syncHooks) {
+        // Check component targeting first
+        if (!matchesComponentTarget(event, hook.options.componentTarget)) {
+            continue;
+        }
+
+        // Check filter condition
+        if (hook.options.filter && !hook.options.filter(event)) {
+            continue;
+        }
+
+        try {
+            if (hook.options.timeout && hook.options.timeout > 0) {
+                // Execute with timeout. Timer handle is stored so the
+                // normal-completion path clears it (no leaked pending
+                // timers per successful hook). The underlying callback
+                // promise is attached with a detached .catch so a late
+                // rejection after timeout does not escape as unhandled
+                // (H-HOOK-2 / H-MEM-2).
+                let timerHandle: ReturnType<typeof setTimeout> | null = null;
+                const timeoutPromise = new Promise<never>((_, reject) => {
+                    timerHandle = setTimeout(
+                        () => reject(new Error(`Hook ${hook.id} timed out after ${hook.options.timeout}ms`)),
+                        hook.options.timeout
+                    );
+                    (timerHandle as unknown as { unref?: () => void }).unref?.();
+                });
+                const hookPromise = Promise.resolve().then(() => hook.callback(event));
+                hookPromise.catch((err) => {
+                    logger.warn({ hookId: hook.id, err }, `Late rejection from hook after timeout`);
+                });
+                try {
+                    await Promise.race([hookPromise, timeoutPromise]);
+                } finally {
+                    if (timerHandle) clearTimeout(timerHandle);
+                }
+            } else {
+                // Always await — callback may be an async function declared
+                // with async:false by mistake. Without await, a rejection
+                // from such a callback escapes as an unhandled rejection
+                // and crashes the process under strict mode (C13).
+                await hook.callback(event);
+            }
+        } catch (error) {
+            logger.error(`Error executing sync hook ${hook.id} for event ${eventType}: ${error}`);
+            hadErrors = true;
+            // Continue executing other hooks even if one fails
+        }
+    }
+
+    // Execute async hooks in parallel
+    if (asyncHooks.length > 0) {
+        const asyncPromises = asyncHooks.map(async (hook) => {
+            // Check component targeting first
+            if (!matchesComponentTarget(event, hook.options.componentTarget)) {
+                return;
+            }
+
+            // Check filter condition
+            if (hook.options.filter && !hook.options.filter(event)) {
+                return;
+            }
+
+            try {
+                if (hook.options.timeout && hook.options.timeout > 0) {
+                    // Execute with timeout. See sync path for rationale —
+                    // clear the timer on normal completion and detach a
+                    // .catch on the hook promise so late rejections do
+                    // not escape (H-HOOK-2 / H-MEM-2).
+                    let timerHandle: ReturnType<typeof setTimeout> | null = null;
+                    const hookPromise = Promise.resolve().then(() => hook.callback(event));
+                    hookPromise.catch((err) => {
+                        logger.warn({ hookId: hook.id, err }, `Late rejection from hook after timeout`);
+                    });
+                    const timeoutPromise = new Promise<never>((_, reject) => {
+                        timerHandle = setTimeout(
+                            () => reject(new Error(`Hook ${hook.id} timed out after ${hook.options.timeout}ms`)),
+                            hook.options.timeout
+                        );
+                        (timerHandle as unknown as { unref?: () => void }).unref?.();
+                    });
+                    try {
+                        await Promise.race([hookPromise, timeoutPromise]);
+                    } finally {
+                        if (timerHandle) clearTimeout(timerHandle);
+                    }
+                } else {
+                    // Execute normally
+                    await hook.callback(event);
+                }
+            } catch (error) {
+                logger.error(`Error executing async hook ${hook.id} for event ${eventType}: ${error}`);
+                hadErrors = true;
+                // Continue executing other hooks even if one fails
+            }
+        });
+
+        await Promise.allSettled(asyncPromises);
+    }
+
+    // Record performance metrics
+    const executionTime = performance.now() - startTime;
+    recordMetrics(dispatcherState, eventType, executionTime, hadErrors);
+}
+
+/**
+ * Execute hooks for multiple events in batch
+ */
+export async function executeHooksBatch(registryState: RegistryState, dispatcherState: DispatcherState, events: LifecycleEvent[]): Promise<void> {
+    if (events.length === 0) {
+        return;
+    }
+
+    logger.trace(`Executing hooks for ${events.length} events in batch`);
+
+    // Group events by type for efficient processing
+    const eventsByType = new Map<string, LifecycleEvent[]>();
+    for (const event of events) {
+        const eventType = event.getEventType();
+        if (!eventsByType.has(eventType)) {
+            eventsByType.set(eventType, []);
+        }
+        eventsByType.get(eventType)!.push(event);
+    }
+
+    // Process each event type
+    const promises: Promise<void>[] = [];
+    for (const [eventType, typeEvents] of eventsByType.entries()) {
+        promises.push(executeHooksForType(registryState, dispatcherState, eventType, typeEvents));
+    }
+
+    await Promise.allSettled(promises);
+}
+
+/**
+ * Execute hooks for a specific event type with multiple events
+ */
+async function executeHooksForType(registryState: RegistryState, dispatcherState: DispatcherState, eventType: string, events: LifecycleEvent[]): Promise<void> {
+    const hooks = registryState.hooks.get(eventType) || [];
+
+    if (hooks.length === 0 || events.length === 0) {
+        return;
+    }
+
+    logger.trace(`Executing ${hooks.length} hooks for ${events.length} ${eventType} events`);
+
+    // Pre-filter hooks by component targeting to avoid repeated checks
+    const preFilteredHooks = preFilterHooksByComponentTargeting(hooks, events);
+
+    if (preFilteredHooks.length === 0) {
+        return;
+    }
+
+    // Separate sync and async hooks
+    const syncHooks = preFilteredHooks.filter(hook => !hook.options.async);
+    const asyncHooks = preFilteredHooks.filter(hook => hook.options.async);
+
+    // Execute sync hooks for all events with batch optimization
+    if (syncHooks.length > 0) {
+        await executeSyncHooksBatch(dispatcherState, syncHooks, events, eventType);
+    }
+
+    // Execute async hooks in parallel for all events with batch optimization
+    if (asyncHooks.length > 0) {
+        await executeAsyncHooksBatch(dispatcherState, asyncHooks, events, eventType);
+    }
+}
+
+/**
+ * Pre-filter hooks based on component targeting to optimize batch processing
+ */
+function preFilterHooksByComponentTargeting(hooks: RegisteredHook[], events: LifecycleEvent[]): RegisteredHook[] {
+    // If no hooks have component targeting, return all hooks (preserving order)
+    const hasComponentTargeting = hooks.some(hook => hook.options.componentTarget);
+    if (!hasComponentTargeting) {
+        return [...hooks]; // Return a copy to avoid modifying the original
+    }
+
+    // For hooks with component targeting, check if they could match any event
+    // This is a broad pre-filter to avoid checking every hook against every event
+    const filteredHooks = hooks.filter(hook => {
+        if (!hook.options.componentTarget) {
+            return true; // No targeting means it matches all
+        }
+
+        // Check if this hook could potentially match any of the events
+        return events.some(event => matchesComponentTarget(event, hook.options.componentTarget));
+    });
+
+    // Return filtered hooks in their original order (priority should already be sorted)
+    return filteredHooks;
+}
+
+/**
+ * Execute sync hooks for multiple events with batch optimizations
+ */
+async function executeSyncHooksBatch(dispatcherState: DispatcherState, syncHooks: RegisteredHook[], events: LifecycleEvent[], eventType: string): Promise<void> {
+    const startTime = performance.now();
+    let hadErrors = false;
+
+    // Execute hooks in priority order across all events to maintain deterministic execution
+    for (const hook of syncHooks) {
+        // Process all events for this hook
+        for (const event of events) {
+            // Double-check component targeting (pre-filter may have false positives)
+            if (!matchesComponentTarget(event, hook.options.componentTarget)) {
+                continue;
+            }
+
+            // Check filter condition
+            if (hook.options.filter && !hook.options.filter(event)) {
+                continue;
+            }
+
+            try {
+                if (hook.options.timeout && hook.options.timeout > 0) {
+                    // Same cleanup pattern as single-event path (H-HOOK-2 / H-MEM-2).
+                    let timerHandle: ReturnType<typeof setTimeout> | null = null;
+                    const hookPromise = Promise.resolve().then(() => hook.callback(event));
+                    hookPromise.catch((err) => {
+                        logger.warn({ hookId: hook.id, err }, `Late rejection from hook after timeout`);
+                    });
+                    const timeoutPromise = new Promise<never>((_, reject) => {
+                        timerHandle = setTimeout(
+                            () => reject(new Error(`Hook ${hook.id} timed out after ${hook.options.timeout}ms`)),
+                            hook.options.timeout
+                        );
+                        (timerHandle as unknown as { unref?: () => void }).unref?.();
+                    });
+                    try {
+                        await Promise.race([hookPromise, timeoutPromise]);
+                    } finally {
+                        if (timerHandle) clearTimeout(timerHandle);
+                    }
+                } else {
+                    // Await so async callbacks do not escape as unhandled
+                    // rejections (C13 parity).
+                    await hook.callback(event);
+                }
+            } catch (error) {
+                logger.error(`Error executing sync hook ${hook.id} for event ${eventType}: ${error}`);
+                hadErrors = true;
+            }
+        }
+    }
+
+    // Record performance metrics
+    const executionTime = performance.now() - startTime;
+    recordMetrics(dispatcherState, eventType, executionTime, hadErrors);
+}
+
+/**
+ * Execute async hooks for multiple events with batch optimizations
+ */
+async function executeAsyncHooksBatch(dispatcherState: DispatcherState, asyncHooks: RegisteredHook[], events: LifecycleEvent[], eventType: string): Promise<void> {
+    const startTime = performance.now();
+    let hadErrors = false;
+
+    // Collect all async hook executions
+    const asyncPromises: Promise<void>[] = [];
+
+    // Use a more efficient batching strategy for async hooks
+    for (const event of events) {
+        for (const hook of asyncHooks) {
+            // Double-check component targeting
+            if (!matchesComponentTarget(event, hook.options.componentTarget)) {
+                continue;
+            }
+
+            // Check filter condition
+            if (hook.options.filter && !hook.options.filter(event)) {
+                continue;
+            }
+
+            asyncPromises.push(
+                (async () => {
+                    try {
+                        if (hook.options.timeout && hook.options.timeout > 0) {
+                            // Same cleanup pattern (H-HOOK-2 / H-MEM-2).
+                            let timerHandle: ReturnType<typeof setTimeout> | null = null;
+                            const hookPromise = Promise.resolve().then(() => hook.callback(event));
+                            hookPromise.catch((err) => {
+                                logger.warn({ hookId: hook.id, err }, `Late rejection from hook after timeout`);
+                            });
+                            const timeoutPromise = new Promise<never>((_, reject) => {
+                                timerHandle = setTimeout(
+                                    () => reject(new Error(`Hook ${hook.id} timed out after ${hook.options.timeout}ms`)),
+                                    hook.options.timeout
+                                );
+                                (timerHandle as unknown as { unref?: () => void }).unref?.();
+                            });
+                            try {
+                                await Promise.race([hookPromise, timeoutPromise]);
+                            } finally {
+                                if (timerHandle) clearTimeout(timerHandle);
+                            }
+                        } else {
+                            // Execute normally
+                            await hook.callback(event);
+                        }
+                    } catch (error) {
+                        logger.error(`Error executing async hook ${hook.id} for event ${eventType}: ${error}`);
+                        hadErrors = true;
+                    }
+                })()
+            );
+        }
+    }
+
+    // Execute all async hooks in parallel with controlled concurrency
+    if (asyncPromises.length > 0) {
+        await Promise.allSettled(asyncPromises);
+    }
+
+    // Record performance metrics
+    const executionTime = performance.now() - startTime;
+    recordMetrics(dispatcherState, eventType, executionTime, hadErrors);
+}