@hed-hog/core 0.0.294 → 0.0.296

package/src/integration/services/integration-developer-api.service.ts

@@ -0,0 +1,96 @@
+import { Injectable } from '@nestjs/common';
+import {
+    DomainEvent,
+    EventHandler,
+    IntegrationLink,
+    LinkType,
+    OutboxEvent,
+    SubscriberDefinition,
+} from '../types';
+import {
+    DomainEventPublisher,
+    PublishDomainEventInput,
+    PublishDomainEventOptions,
+} from './domain-event.publisher';
+import { EventSubscriberRegistry } from './event-subscriber.registry';
+import {
+    CreateIntegrationLinkDto,
+    IntegrationLinkPersistenceClient,
+    IntegrationLinkService,
+} from './integration-link.service';
+
+export interface IntegrationSubscriptionInput {
+  eventName: string;
+  consumerName: string;
+  handler: EventHandler;
+  priority?: number;
+}
+
+export interface IntegrationLinkQuery {
+  module: string;
+  entityType: string;
+  entityId: string;
+}
+
+@Injectable()
+export class IntegrationDeveloperApiService {
+  constructor(
+    private readonly publisher: DomainEventPublisher,
+    private readonly subscriberRegistry: EventSubscriberRegistry,
+    private readonly linkService: IntegrationLinkService,
+  ) {}
+
+  async publishEvent(
+    input: PublishDomainEventInput,
+    options?: PublishDomainEventOptions,
+  ): Promise<OutboxEvent> {
+    return this.publisher.publishEvent(input, options);
+  }
+
+  async publishEvents(
+    events: Array<DomainEvent | PublishDomainEventInput>,
+    options?: PublishDomainEventOptions,
+  ): Promise<OutboxEvent[]> {
+    return this.publisher.publishEvents(events, options);
+  }
+
+  subscribe(input: IntegrationSubscriptionInput): void {
+    this.subscriberRegistry.subscribe(
+      input.eventName,
+      input.consumerName,
+      input.handler,
+      input.priority || 0,
+    );
+  }
+
+  subscribeMany(definitions: SubscriberDefinition[]): void {
+    this.subscriberRegistry.subscribeMany(definitions);
+  }
+
+  async createLink(
+    dto: CreateIntegrationLinkDto,
+    persistenceClient?: IntegrationLinkPersistenceClient,
+  ): Promise<IntegrationLink> {
+    return this.linkService.createLink(dto, persistenceClient);
+  }
+
+  async findLinksBySource(query: IntegrationLinkQuery): Promise<IntegrationLink[]> {
+    return this.linkService.findOutbound(
+      query.module,
+      query.entityType,
+      query.entityId,
+    );
+  }
+
+  async findLinksByTarget(query: IntegrationLinkQuery): Promise<IntegrationLink[]> {
+    return this.linkService.findInbound(
+      query.module,
+      query.entityType,
+      query.entityId,
+    );
+  }
+
+  async findLinksByType(linkType: LinkType): Promise<IntegrationLink[]> {
+    return this.linkService.findByLinkType(linkType);
+  }
+}

package/src/integration/services/integration-link.service.ts

@@ -0,0 +1,154 @@
+import { PrismaService } from '@hed-hog/api-prisma';
+import { Injectable } from '@nestjs/common';
+import { IntegrationLink, LinkType } from '../types';
+
+export interface CreateIntegrationLinkDto {
+  sourceModule: string;
+  sourceEntityType: string;
+  sourceEntityId: string;
+  targetModule: string;
+  targetEntityType: string;
+  targetEntityId: string;
+  linkType: LinkType;
+  metadata?: Record<string, any>;
+}
+
+export interface IntegrationLinkPersistenceClient {
+  integrationLink: PrismaService['integrationLink'];
+}
+
+@Injectable()
+export class IntegrationLinkService {
+  constructor(private readonly prisma: PrismaService) {}
+
+  /**
+   * Create a link between entities from different modules
+   */
+  async createLink(
+    dto: CreateIntegrationLinkDto,
+    persistenceClient?: IntegrationLinkPersistenceClient,
+  ): Promise<IntegrationLink> {
+    const client = persistenceClient ?? this.prisma;
+
+    return client.integrationLink.create({
+      data: {
+        sourceModule: dto.sourceModule,
+        sourceEntityType: dto.sourceEntityType,
+        sourceEntityId: dto.sourceEntityId,
+        targetModule: dto.targetModule,
+        targetEntityType: dto.targetEntityType,
+        targetEntityId: dto.targetEntityId,
+        linkType: dto.linkType,
+        metadata: dto.metadata || null,
+      },
+    });
+  }
+
+  /**
+   * Find all links originating from a source entity
+   */
+  async findOutbound(
+    sourceModule: string,
+    sourceEntityType: string,
+    sourceEntityId: string,
+  ): Promise<IntegrationLink[]> {
+    return this.prisma.integrationLink.findMany({
+      where: {
+        sourceModule,
+        sourceEntityType,
+        sourceEntityId,
+      },
+    });
+  }
+
+  /**
+   * Find all links terminating at a target entity
+   */
+  async findInbound(
+    targetModule: string,
+    targetEntityType: string,
+    targetEntityId: string,
+  ): Promise<IntegrationLink[]> {
+    return this.prisma.integrationLink.findMany({
+      where: {
+        targetModule,
+        targetEntityType,
+        targetEntityId,
+      },
+    });
+  }
+
+  /**
+   * Find links of a specific type
+   */
+  async findByLinkType(linkType: LinkType): Promise<IntegrationLink[]> {
+    return this.prisma.integrationLink.findMany({
+      where: { linkType },
+    });
+  }
+
+  /**
+   * Delete a link
+   */
+  async deleteLink(linkId: string): Promise<IntegrationLink> {
+    return this.prisma.integrationLink.delete({
+      where: { id: linkId },
+    });
+  }
+
+  /**
+   * Get link by ID
+   */
+  async getById(linkId: string): Promise<IntegrationLink | null> {
+    return this.prisma.integrationLink.findUnique({
+      where: { id: linkId },
+    });
+  }
+
+  /**
+   * Find bidirectional link between two entities
+   */
+  async findBidirectional(
+    module1: string,
+    entity1Type: string,
+    entity1Id: string,
+    module2: string,
+    entity2Type: string,
+    entity2Id: string,
+  ): Promise<IntegrationLink | null> {
+    // Look for forward or reverse direction
+    const link = await this.prisma.integrationLink.findFirst({
+      where: {
+        OR: [
+          {
+            sourceModule: module1,
+            sourceEntityType: entity1Type,
+            sourceEntityId: entity1Id,
+            targetModule: module2,
+            targetEntityType: entity2Type,
+            targetEntityId: entity2Id,
+          },
+          {
+            sourceModule: module2,
+            sourceEntityType: entity2Type,
+            sourceEntityId: entity2Id,
+            targetModule: module1,
+            targetEntityType: entity1Type,
+            targetEntityId: entity1Id,
+          },
+        ],
+      },
+    });
+
+    return link;
+  }
+
+  /**
+   * Count links from a module
+   */
+  async countFromModule(sourceModule: string): Promise<number> {
+    return this.prisma.integrationLink.count({
+      where: { sourceModule },
+    });
+  }
+}

package/src/integration/services/integration-settings.service.ts

@@ -0,0 +1,128 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { SettingService } from '../../setting/setting.service';
+
+export interface IntegrationRuntimeSettings {
+  outboxEnabled: boolean;
+  outboxProcessorEnabled: boolean;
+  startupDrainEnabled: boolean;
+  pollingIntervalMs: number;
+  idlePollingIntervalMs: number;
+  batchSize: number;
+  startupDrainBatchSize: number;
+  maxAttempts: number;
+  processingLeaseMs: number;
+  retryBaseDelayMs: number;
+  deadLetterEnabled: boolean;
+}
+
+@Injectable()
+export class IntegrationSettingsService {
+  private readonly logger = new Logger(IntegrationSettingsService.name);
+
+  constructor(private readonly settingService: SettingService) {}
+
+  async getRuntimeSettings(): Promise<IntegrationRuntimeSettings> {
+    const values = await this.settingService.getSettingValues([
+      'outbox-enabled',
+      'outbox-processor-enabled',
+      'outbox-startup-drain-enabled',
+      'outbox-polling-interval-ms',
+      'outbox-idle-polling-interval-ms',
+      'outbox-batch-size',
+      'outbox-startup-drain-batch-size',
+      'outbox-max-attempts',
+      'outbox-processing-lease-ms',
+      'outbox-retry-base-delay-ms',
+      'outbox-dead-letter-enabled',
+    ]);
+
+    const pollingIntervalMs = this.getPositiveInt(
+      values['outbox-polling-interval-ms'],
+      5000,
+      100,
+      'outbox-polling-interval-ms',
+    );
+
+    let idlePollingIntervalMs = this.getPositiveInt(
+      values['outbox-idle-polling-interval-ms'],
+      30000,
+      100,
+      'outbox-idle-polling-interval-ms',
+    );
+
+    if (idlePollingIntervalMs < pollingIntervalMs) {
+      this.logger.warn(
+        `Setting outbox-idle-polling-interval-ms (${idlePollingIntervalMs}) is lower than outbox-polling-interval-ms (${pollingIntervalMs}). Using ${pollingIntervalMs}.`,
+      );
+      idlePollingIntervalMs = pollingIntervalMs;
+    }
+
+    return {
+      outboxEnabled: this.getBoolean(values['outbox-enabled'], true),
+      outboxProcessorEnabled: this.getBoolean(
+        values['outbox-processor-enabled'],
+        true,
+      ),
+      startupDrainEnabled: this.getBoolean(
+        values['outbox-startup-drain-enabled'],
+        true,
+      ),
+      pollingIntervalMs,
+      idlePollingIntervalMs,
+      batchSize: this.getPositiveInt(values['outbox-batch-size'], 10, 1, 'outbox-batch-size'),
+      startupDrainBatchSize: this.getPositiveInt(
+        values['outbox-startup-drain-batch-size'],
+        50,
+        1,
+        'outbox-startup-drain-batch-size',
+      ),
+      maxAttempts: this.getPositiveInt(values['outbox-max-attempts'], 3, 1, 'outbox-max-attempts'),
+      processingLeaseMs: this.getPositiveInt(
+        values['outbox-processing-lease-ms'],
+        30000,
+        1000,
+        'outbox-processing-lease-ms',
+      ),
+      retryBaseDelayMs: this.getPositiveInt(
+        values['outbox-retry-base-delay-ms'],
+        1000,
+        100,
+        'outbox-retry-base-delay-ms',
+      ),
+      deadLetterEnabled: this.getBoolean(values['outbox-dead-letter-enabled'], true),
+    };
+  }
+
+  private getPositiveInt(
+    value: unknown,
+    fallback: number,
+    min: number,
+    slug: string,
+  ): number {
+    const parsed = Number(value);
+    if (!Number.isFinite(parsed) || parsed < min) {
+      this.logger.warn(
+        `Invalid ${slug} setting value (${String(value)}). Using fallback ${fallback}.`,
+      );
+      return fallback;
+    }
+
+    return Math.floor(parsed);
+  }
+
+  private getBoolean(value: unknown, fallback: boolean): boolean {
+    if (typeof value === 'boolean') {
+      return value;
+    }
+
+    if (value === 'true' || value === '1') {
+      return true;
+    }
+
+    if (value === 'false' || value === '0') {
+      return false;
+    }
+
+    return fallback;
+  }
+}

package/src/integration/services/outbox-polling.coordinator.ts

@@ -0,0 +1,146 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { IntegrationSettingsService } from './integration-settings.service';
+import { OutboxNotifier } from './outbox.notifier';
+import { OutboxProcessor } from './outbox.processor';
+
+/**
+ * Hybrid polling coordinator
+ * Manages the adaptive polling strategy: immediate reactions + periodic checks
+ * Uses in-memory notifications for immediate wakeup + configurable polling intervals
+ */
+@Injectable()
+export class OutboxPollingCoordinator {
+  private readonly logger = new Logger(OutboxPollingCoordinator.name);
+  private processingLoopRunning = false;
+  private processingLoopPromise: Promise<void> | null = null;
+  private immediateWakeupTriggered = false;
+  private shouldStopLoop = false;
+
+  constructor(
+    private readonly processor: OutboxProcessor,
+    private readonly notifier: OutboxNotifier,
+    private readonly integrationSettingsService: IntegrationSettingsService,
+  ) {}
+
+  /**
+   * Start the hybrid processing loop
+   * Reacts to notifications immediately, polls periodically
+   */
+  async startProcessingLoop(): Promise<void> {
+    if (this.processingLoopRunning) {
+      this.logger.warn('Processing loop already running');
+      return;
+    }
+
+    this.processingLoopRunning = true;
+    this.shouldStopLoop = false;
+    this.logger.log('Starting outbox hybrid processing loop');
+
+    this.processingLoopPromise = this.runProcessingLoop();
+  }
+
+  /**
+   * Stop the processing loop gracefully
+   */
+  async stopProcessingLoop(): Promise<void> {
+    if (!this.processingLoopRunning) {
+      return;
+    }
+
+    this.logger.log('Stopping outbox processing loop');
+    this.shouldStopLoop = true;
+
+    if (this.processingLoopPromise) {
+      await this.processingLoopPromise;
+    }
+
+    this.processingLoopRunning = false;
+  }
+
+  /**
+   * Trigger immediate processing from notifier
+   */
+  triggerImmediateProcessing(): void {
+    this.immediateWakeupTriggered = true;
+  }
+
+  /**
+   * Check if loop is currently running
+   */
+  isRunning(): boolean {
+    return this.processingLoopRunning;
+  }
+
+  /**
+   * Main processing loop - runs continuously
+   */
+  private async runProcessingLoop(): Promise<void> {
+    // Subscribe to notifications
+    const immediateWakeupListener = () => {
+      this.triggerImmediateProcessing();
+    };
+    this.notifier.subscribe(immediateWakeupListener);
+
+    try {
+      while (!this.shouldStopLoop) {
+        const settings = await this.integrationSettingsService.getRuntimeSettings();
+        const isEnabled = settings.outboxEnabled;
+        const processorEnabled = settings.outboxProcessorEnabled;
+
+        if (!isEnabled || !processorEnabled) {
+          // If disabled, wait before checking again
+          await this.delay(settings.idlePollingIntervalMs);
+          continue;
+        }
+
+        // Check if we should process immediately (woken by notification)
+        const shouldProcessImmediately = this.immediateWakeupTriggered;
+        this.immediateWakeupTriggered = false;
+
+        // Process batch
+        const processedCount = await this.processor.processBatch();
+
+        // Determine next polling interval
+        let nextDelayMs: number;
+
+        if (shouldProcessImmediately && processedCount > 0) {
+          // We processed items from immediate wakeup - use active polling interval
+          nextDelayMs = settings.pollingIntervalMs;
+          this.logger.debug(
+            `Processed ${processedCount} events from immediate wakeup, next poll in ${nextDelayMs}ms`,
+          );
+        } else if (processedCount > 0) {
+          // Processed items from periodic polling - use active interval
+          nextDelayMs = settings.pollingIntervalMs;
+          this.logger.debug(
+            `Processed ${processedCount} events, next poll in ${nextDelayMs}ms`,
+          );
+        } else {
+          // No items processed - use idle interval
+          nextDelayMs = settings.idlePollingIntervalMs;
+          this.logger.debug(
+            `No events processed, idle polling for ${nextDelayMs}ms`,
+          );
+        }
+
+        // Wait for next poll, but be ready for immediate wakeup
+        await this.delay(nextDelayMs);
+      }
+    } finally {
+      // Cleanup: unsubscribe from notifications
+      this.notifier.unsubscribe(immediateWakeupListener);
+      this.logger.log('Processing loop stopped');
+    }
+  }
+
+  /**
+   * Utility: delay with cancellation support
+   */
+  private delay(ms: number): Promise<void> {
+    return new Promise((resolve) => {
+      const timeout = setTimeout(resolve, ms);
+      // Store timeout for potential cancellation (not used yet, but available)
+      (this as any)._currentTimeout = timeout;
+    });
+  }
+}

package/src/integration/services/outbox.notifier.ts

@@ -0,0 +1,48 @@
+import { Injectable } from '@nestjs/common';
+import { EventEmitter } from 'events';
+
+/**
+ * In-memory notifier for signaling new outbox events
+ * Provides immediate wakeup to the processor without database polling
+ * NOT the source of truth; database outbox is the authoritative queue
+ */
+@Injectable()
+export class OutboxNotifier {
+  private readonly emitter = new EventEmitter();
+  private readonly EVENT_WRITTEN = 'outbox:event:written';
+
+  /**
+   * Subscribe to outbox event notifications
+   */
+  subscribe(listener: () => void): void {
+    this.emitter.on(this.EVENT_WRITTEN, listener);
+  }
+
+  /**
+   * Unsubscribe from outbox event notifications
+   */
+  unsubscribe(listener: () => void): void {
+    this.emitter.removeListener(this.EVENT_WRITTEN, listener);
+  }
+
+  /**
+   * Emit notification that a new event was written to outbox
+   */
+  notifyEventWritten(): void {
+    this.emitter.emit(this.EVENT_WRITTEN);
+  }
+
+  /**
+   * Get current number of subscribers
+   */
+  getSubscriberCount(): number {
+    return this.emitter.listenerCount(this.EVENT_WRITTEN);
+  }
+
+  /**
+   * Remove all listeners
+   */
+  removeAllListeners(): void {
+    this.emitter.removeAllListeners(this.EVENT_WRITTEN);
+  }
+}

package/src/integration/services/outbox.processor.job.ts

@@ -0,0 +1,97 @@
+import {
+    Injectable,
+    Logger,
+    OnModuleDestroy,
+    OnModuleInit,
+} from '@nestjs/common';
+import { IntegrationSettingsService } from './integration-settings.service';
+import { OutboxPollingCoordinator } from './outbox-polling.coordinator';
+import { OutboxNotifier } from './outbox.notifier';
+import { OutboxProcessor } from './outbox.processor';
+
+/**
+ * Hybrid background job for outbox processing
+ * Combines startup recovery, immediate notification reactions, and periodic polling
+ */
+@Injectable()
+export class OutboxProcessorJob implements OnModuleInit, OnModuleDestroy {
+  private readonly logger = new Logger(OutboxProcessorJob.name);
+
+  constructor(
+    private readonly processor: OutboxProcessor,
+    private readonly notifier: OutboxNotifier,
+    private readonly coordinator: OutboxPollingCoordinator,
+    private readonly integrationSettingsService: IntegrationSettingsService,
+  ) {}
+
+  /**
+   * Called when module initializes
+   * 1. Drain pending events from startup
+   * 2. Start hybrid processing loop
+   */
+  async onModuleInit(): Promise<void> {
+    const settings = await this.integrationSettingsService.getRuntimeSettings();
+    const isEnabled = settings.outboxEnabled;
+    if (!isEnabled) {
+      this.logger.debug('Outbox integration disabled in settings');
+      return;
+    }
+
+    // Step 1: Startup drain
+    const startupDrainEnabled = settings.startupDrainEnabled;
+
+    if (startupDrainEnabled) {
+      this.logger.log('Starting outbox startup drain...');
+      try {
+        const processed = await this.processor.startupDrain();
+        this.logger.log(
+          `Outbox startup drain completed: ${processed} events`,
+        );
+      } catch (error) {
+        this.logger.error('Outbox startup drain failed:', error);
+      }
+    }
+
+    // Step 2: Start hybrid processing loop
+    const processorEnabled = settings.outboxProcessorEnabled;
+
+    if (processorEnabled) {
+      this.logger.log('Starting outbox hybrid processing loop');
+      try {
+        await this.coordinator.startProcessingLoop();
+      } catch (error) {
+        this.logger.error('Failed to start outbox processing loop:', error);
+      }
+    } else {
+      this.logger.debug('Outbox processor disabled in settings');
+    }
+  }
+
+  /**
+   * Called when module is destroyed
+   * Gracefully stop the processing loop
+   */
+  async onModuleDestroy(): Promise<void> {
+    this.logger.log('Shutting down outbox processing');
+    try {
+      await this.coordinator.stopProcessingLoop();
+    } catch (error) {
+      this.logger.error(
+        'Error stopping outbox processing loop:',
+        error,
+      );
+    }
+  }
+
+  /**
+   * Get current processing statistics
+   * Useful for health checks and monitoring
+   */
+  async getStats() {
+    const stats = await this.processor.getStats();
+    return {
+      ...stats,
+      pollingLoopRunning: this.coordinator.isRunning(),
+    };
+  }
+}