@hed-hog/core 0.0.295 → 0.0.296

package/src/integration/services/outbox-polling.coordinator.ts

@@ -0,0 +1,146 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { IntegrationSettingsService } from './integration-settings.service';
+import { OutboxNotifier } from './outbox.notifier';
+import { OutboxProcessor } from './outbox.processor';
+
+/**
+ * Hybrid polling coordinator
+ * Manages the adaptive polling strategy: immediate reactions + periodic checks
+ * Uses in-memory notifications for immediate wakeup + configurable polling intervals
+ */
+@Injectable()
+export class OutboxPollingCoordinator {
+  private readonly logger = new Logger(OutboxPollingCoordinator.name);
+  private processingLoopRunning = false;
+  private processingLoopPromise: Promise<void> | null = null;
+  private immediateWakeupTriggered = false;
+  private shouldStopLoop = false;
+
+  constructor(
+    private readonly processor: OutboxProcessor,
+    private readonly notifier: OutboxNotifier,
+    private readonly integrationSettingsService: IntegrationSettingsService,
+  ) {}
+
+  /**
+   * Start the hybrid processing loop
+   * Reacts to notifications immediately, polls periodically
+   */
+  async startProcessingLoop(): Promise<void> {
+    if (this.processingLoopRunning) {
+      this.logger.warn('Processing loop already running');
+      return;
+    }
+
+    this.processingLoopRunning = true;
+    this.shouldStopLoop = false;
+    this.logger.log('Starting outbox hybrid processing loop');
+
+    this.processingLoopPromise = this.runProcessingLoop();
+  }
+
+  /**
+   * Stop the processing loop gracefully
+   */
+  async stopProcessingLoop(): Promise<void> {
+    if (!this.processingLoopRunning) {
+      return;
+    }
+
+    this.logger.log('Stopping outbox processing loop');
+    this.shouldStopLoop = true;
+
+    if (this.processingLoopPromise) {
+      await this.processingLoopPromise;
+    }
+
+    this.processingLoopRunning = false;
+  }
+
+  /**
+   * Trigger immediate processing from notifier
+   */
+  triggerImmediateProcessing(): void {
+    this.immediateWakeupTriggered = true;
+  }
+
+  /**
+   * Check if loop is currently running
+   */
+  isRunning(): boolean {
+    return this.processingLoopRunning;
+  }
+
+  /**
+   * Main processing loop - runs continuously
+   */
+  private async runProcessingLoop(): Promise<void> {
+    // Subscribe to notifications
+    const immediateWakeupListener = () => {
+      this.triggerImmediateProcessing();
+    };
+    this.notifier.subscribe(immediateWakeupListener);
+
+    try {
+      while (!this.shouldStopLoop) {
+        const settings = await this.integrationSettingsService.getRuntimeSettings();
+        const isEnabled = settings.outboxEnabled;
+        const processorEnabled = settings.outboxProcessorEnabled;
+
+        if (!isEnabled || !processorEnabled) {
+          // If disabled, wait before checking again
+          await this.delay(settings.idlePollingIntervalMs);
+          continue;
+        }
+
+        // Check if we should process immediately (woken by notification)
+        const shouldProcessImmediately = this.immediateWakeupTriggered;
+        this.immediateWakeupTriggered = false;
+
+        // Process batch
+        const processedCount = await this.processor.processBatch();
+
+        // Determine next polling interval
+        let nextDelayMs: number;
+
+        if (shouldProcessImmediately && processedCount > 0) {
+          // We processed items from immediate wakeup - use active polling interval
+          nextDelayMs = settings.pollingIntervalMs;
+          this.logger.debug(
+            `Processed ${processedCount} events from immediate wakeup, next poll in ${nextDelayMs}ms`,
+          );
+        } else if (processedCount > 0) {
+          // Processed items from periodic polling - use active interval
+          nextDelayMs = settings.pollingIntervalMs;
+          this.logger.debug(
+            `Processed ${processedCount} events, next poll in ${nextDelayMs}ms`,
+          );
+        } else {
+          // No items processed - use idle interval
+          nextDelayMs = settings.idlePollingIntervalMs;
+          this.logger.debug(
+            `No events processed, idle polling for ${nextDelayMs}ms`,
+          );
+        }
+
+        // Wait for next poll, but be ready for immediate wakeup
+        await this.delay(nextDelayMs);
+      }
+    } finally {
+      // Cleanup: unsubscribe from notifications
+      this.notifier.unsubscribe(immediateWakeupListener);
+      this.logger.log('Processing loop stopped');
+    }
+  }
+
+  /**
+   * Utility: delay with cancellation support
+   */
+  private delay(ms: number): Promise<void> {
+    return new Promise((resolve) => {
+      const timeout = setTimeout(resolve, ms);
+      // Store timeout for potential cancellation (not used yet, but available)
+      (this as any)._currentTimeout = timeout;
+    });
+  }
+}

package/src/integration/services/outbox.notifier.ts

@@ -0,0 +1,48 @@
+import { Injectable } from '@nestjs/common';
+import { EventEmitter } from 'events';
+
+/**
+ * In-memory notifier for signaling new outbox events
+ * Provides immediate wakeup to the processor without database polling
+ * NOT the source of truth; database outbox is the authoritative queue
+ */
+@Injectable()
+export class OutboxNotifier {
+  private readonly emitter = new EventEmitter();
+  private readonly EVENT_WRITTEN = 'outbox:event:written';
+
+  /**
+   * Subscribe to outbox event notifications
+   */
+  subscribe(listener: () => void): void {
+    this.emitter.on(this.EVENT_WRITTEN, listener);
+  }
+
+  /**
+   * Unsubscribe from outbox event notifications
+   */
+  unsubscribe(listener: () => void): void {
+    this.emitter.removeListener(this.EVENT_WRITTEN, listener);
+  }
+
+  /**
+   * Emit notification that a new event was written to outbox
+   */
+  notifyEventWritten(): void {
+    this.emitter.emit(this.EVENT_WRITTEN);
+  }
+
+  /**
+   * Get current number of subscribers
+   */
+  getSubscriberCount(): number {
+    return this.emitter.listenerCount(this.EVENT_WRITTEN);
+  }
+
+  /**
+   * Remove all listeners
+   */
+  removeAllListeners(): void {
+    this.emitter.removeAllListeners(this.EVENT_WRITTEN);
+  }
+}

package/src/integration/services/outbox.processor.job.ts

@@ -0,0 +1,97 @@
+import {
+    Injectable,
+    Logger,
+    OnModuleDestroy,
+    OnModuleInit,
+} from '@nestjs/common';
+import { IntegrationSettingsService } from './integration-settings.service';
+import { OutboxPollingCoordinator } from './outbox-polling.coordinator';
+import { OutboxNotifier } from './outbox.notifier';
+import { OutboxProcessor } from './outbox.processor';
+
+/**
+ * Hybrid background job for outbox processing
+ * Combines startup recovery, immediate notification reactions, and periodic polling
+ */
+@Injectable()
+export class OutboxProcessorJob implements OnModuleInit, OnModuleDestroy {
+  private readonly logger = new Logger(OutboxProcessorJob.name);
+
+  constructor(
+    private readonly processor: OutboxProcessor,
+    private readonly notifier: OutboxNotifier,
+    private readonly coordinator: OutboxPollingCoordinator,
+    private readonly integrationSettingsService: IntegrationSettingsService,
+  ) {}
+
+  /**
+   * Called when module initializes
+   * 1. Drain pending events from startup
+   * 2. Start hybrid processing loop
+   */
+  async onModuleInit(): Promise<void> {
+    const settings = await this.integrationSettingsService.getRuntimeSettings();
+    const isEnabled = settings.outboxEnabled;
+    if (!isEnabled) {
+      this.logger.debug('Outbox integration disabled in settings');
+      return;
+    }
+
+    // Step 1: Startup drain
+    const startupDrainEnabled = settings.startupDrainEnabled;
+
+    if (startupDrainEnabled) {
+      this.logger.log('Starting outbox startup drain...');
+      try {
+        const processed = await this.processor.startupDrain();
+        this.logger.log(
+          `Outbox startup drain completed: ${processed} events`,
+        );
+      } catch (error) {
+        this.logger.error('Outbox startup drain failed:', error);
+      }
+    }
+
+    // Step 2: Start hybrid processing loop
+    const processorEnabled = settings.outboxProcessorEnabled;
+
+    if (processorEnabled) {
+      this.logger.log('Starting outbox hybrid processing loop');
+      try {
+        await this.coordinator.startProcessingLoop();
+      } catch (error) {
+        this.logger.error('Failed to start outbox processing loop:', error);
+      }
+    } else {
+      this.logger.debug('Outbox processor disabled in settings');
+    }
+  }
+
+  /**
+   * Called when module is destroyed
+   * Gracefully stop the processing loop
+   */
+  async onModuleDestroy(): Promise<void> {
+    this.logger.log('Shutting down outbox processing');
+    try {
+      await this.coordinator.stopProcessingLoop();
+    } catch (error) {
+      this.logger.error(
+        'Error stopping outbox processing loop:',
+        error,
+      );
+    }
+  }
+
+  /**
+   * Get current processing statistics
+   * Useful for health checks and monitoring
+   */
+  async getStats() {
+    const stats = await this.processor.getStats();
+    return {
+      ...stats,
+      pollingLoopRunning: this.coordinator.isRunning(),
+    };
+  }
+}

package/src/integration/services/outbox.processor.ts

@@ -0,0 +1,266 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { InboxEventStatus, OutboxEventStatus } from '../types';
+import { EventSubscriberRegistry } from './event-subscriber.registry';
+import { InboxService } from './inbox.service';
+import { IntegrationLinkService } from './integration-link.service';
+import { IntegrationSettingsService } from './integration-settings.service';
+import { OutboxService } from './outbox.service';
+
+@Injectable()
+export class OutboxProcessor {
+  private readonly logger = new Logger(OutboxProcessor.name);
+
+  constructor(
+    private readonly outboxService: OutboxService,
+    private readonly inboxService: InboxService,
+    private readonly linkService: IntegrationLinkService,
+    private readonly registry: EventSubscriberRegistry,
+    private readonly integrationSettingsService: IntegrationSettingsService,
+  ) {}
+
+  /**
+   * Process a batch of pending events
+   * Called in a loop or by scheduled job
+   */
+  async processBatch(options?: { batchSizeOverride?: number }): Promise<number> {
+    const settings = await this.integrationSettingsService.getRuntimeSettings();
+    const batchSize = options?.batchSizeOverride ?? settings.batchSize;
+    const leaseMs = settings.processingLeaseMs;
+
+    // Find pending events
+    const events = await this.outboxService.findPending(batchSize, leaseMs);
+
+    if (events.length === 0) {
+      return 0;
+    }
+
+    this.logger.debug(`Processing ${events.length} outbox events`);
+
+    let processedCount = 0;
+
+    for (const event of events) {
+      try {
+        // Mark as processing with lease
+        await this.outboxService.markProcessing(event.id, leaseMs);
+
+        // Get handlers for this event
+        const handlers = this.registry.getHandlers(event.eventName);
+
+        if (handlers.length === 0) {
+          this.logger.warn(
+            `No handlers registered for event: ${event.eventName}`,
+          );
+          // Mark as processed even though no handlers
+          await this.outboxService.updateStatus(
+            event.id,
+            OutboxEventStatus.PROCESSED,
+            {
+              processedAt: new Date(),
+            },
+          );
+          processedCount++;
+          continue;
+        }
+
+        // Execute each handler
+        let anyFailed = false;
+        for (const handlerDef of handlers) {
+          const inboxEvent = await this.inboxService.getOrCreate(
+            event.id,
+            handlerDef.consumerName,
+          );
+
+          // Skip if already processed
+          if (inboxEvent.status === InboxEventStatus.PROCESSED) {
+            this.logger.debug(
+              `Event ${event.id} already processed by ${handlerDef.consumerName}`,
+            );
+            continue;
+          }
+
+          await this.inboxService.markProcessing(inboxEvent.id);
+
+          try {
+            // Execute handler
+            await handlerDef.handler(
+              {
+                eventName: event.eventName,
+                sourceModule: event.sourceModule,
+                aggregateType: event.aggregateType,
+                aggregateId: event.aggregateId,
+                payload: event.payload,
+                timestamp: event.createdAt,
+              },
+              {
+                logger: this.logger,
+                inboxService: this.inboxService,
+                linkService: this.linkService,
+              },
+            );
+
+            // Mark as processed
+            await this.inboxService.markProcessed(inboxEvent.id);
+            this.logger.debug(
+              `Handler ${handlerDef.consumerName} processed event ${event.id}`,
+            );
+          } catch (error) {
+            anyFailed = true;
+            this.logger.error(
+              `Handler ${handlerDef.consumerName} failed for event ${event.id}:`,
+              error,
+            );
+            await this.inboxService.markFailed(
+              inboxEvent.id,
+              error instanceof Error ? error.message : String(error),
+            );
+          }
+        }
+
+        // Update outbox event status based on handler results
+        if (!anyFailed) {
+          await this.outboxService.updateStatus(
+            event.id,
+            OutboxEventStatus.PROCESSED,
+            {
+              processedAt: new Date(),
+            },
+          );
+          processedCount++;
+        } else {
+          // At least one handler failed, retry logic applies
+          await this.handleEventFailure(event.id);
+        }
+      } catch (error) {
+        this.logger.error(`Error processing event ${event.id}:`, error);
+        await this.handleEventFailure(event.id);
+      }
+    }
+
+    return processedCount;
+  }
+
+  /**
+   * Handle event that failed to process
+   */
+  private async handleEventFailure(eventId: string): Promise<void> {
+    const settings = await this.integrationSettingsService.getRuntimeSettings();
+    const maxAttempts = settings.maxAttempts;
+    const retryBaseDelayMs = settings.retryBaseDelayMs;
+    const deadLetterEnabled = settings.deadLetterEnabled;
+
+    const event = await this.outboxService.getById(eventId);
+    if (!event) {
+      return;
+    }
+
+    const nextAttemptCount = event.attemptCount + 1;
+
+    if (nextAttemptCount >= maxAttempts) {
+      if (deadLetterEnabled) {
+        this.logger.warn(
+          `Event ${eventId} moved to dead letter after ${maxAttempts} attempts`,
+        );
+        await this.outboxService.updateStatus(
+          eventId,
+          OutboxEventStatus.DEAD_LETTER,
+          {
+            attemptCount: nextAttemptCount,
+          },
+        );
+      } else {
+        this.logger.warn(
+          `Event ${eventId} failed after ${maxAttempts} attempts (dead letter disabled)`,
+        );
+        await this.outboxService.updateStatus(
+          eventId,
+          OutboxEventStatus.FAILED,
+          {
+            attemptCount: nextAttemptCount,
+          },
+        );
+      }
+    } else {
+      // Calculate exponential backoff
+      const delayMs = retryBaseDelayMs * Math.pow(2, event.attemptCount);
+      const availableAt = new Date(Date.now() + delayMs);
+
+      this.logger.debug(
+        `Retrying event ${eventId} in ${delayMs}ms (attempt ${nextAttemptCount}/${maxAttempts})`,
+      );
+
+      await this.outboxService.updateStatus(
+        eventId,
+        OutboxEventStatus.PENDING,
+        {
+          attemptCount: nextAttemptCount,
+          availableAt,
+        },
+      );
+    }
+  }
+
+  /**
+   * Process all pending events at startup (drain)
+   */
+  async startupDrain(): Promise<number> {
+    const settings = await this.integrationSettingsService.getRuntimeSettings();
+    const drainBatchSize = settings.startupDrainBatchSize;
+
+    this.logger.log(`Starting outbox drain with batch size: ${drainBatchSize}`);
+
+    let totalProcessed = 0;
+
+    // Keep processing until no more events
+    let processed = 1;
+    while (processed > 0) {
+      processed = await this.processBatch({
+        batchSizeOverride: drainBatchSize,
+      });
+      totalProcessed += processed;
+
+      // Small delay to prevent overwhelming the system
+      if (processed > 0) {
+        await new Promise((resolve) => setTimeout(resolve, 100));
+      }
+    }
+
+    this.logger.log(`Startup drain completed. Processed ${totalProcessed} events`);
+    return totalProcessed;
+  }
+
+  /**
+   * Get processor statistics
+   */
+  async getStats() {
+    const pendingCount = await this.outboxService.countByStatus(
+      OutboxEventStatus.PENDING,
+    );
+    const processingCount = await this.outboxService.countByStatus(
+      OutboxEventStatus.PROCESSING,
+    );
+    const processedCount = await this.outboxService.countByStatus(
+      OutboxEventStatus.PROCESSED,
+    );
+    const failedCount = await this.outboxService.countByStatus(
+      OutboxEventStatus.FAILED,
+    );
+    const deadLetterCount = await this.outboxService.countByStatus(
+      OutboxEventStatus.DEAD_LETTER,
+    );
+
+    return {
+      pending: pendingCount,
+      processing: processingCount,
+      processed: processedCount,
+      failed: failedCount,
+      dead_letter: deadLetterCount,
+      total:
+        pendingCount +
+        processingCount +
+        processedCount +
+        failedCount +
+        deadLetterCount,
+      handlerCount: this.registry.getHandlerCount(),
+    };
+  }
+}