@hed-hog/core 0.0.294 → 0.0.296

package/src/integration/services/outbox.processor.ts

@@ -0,0 +1,266 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { InboxEventStatus, OutboxEventStatus } from '../types';
+import { EventSubscriberRegistry } from './event-subscriber.registry';
+import { InboxService } from './inbox.service';
+import { IntegrationLinkService } from './integration-link.service';
+import { IntegrationSettingsService } from './integration-settings.service';
+import { OutboxService } from './outbox.service';
+
+@Injectable()
+export class OutboxProcessor {
+  private readonly logger = new Logger(OutboxProcessor.name);
+
+  constructor(
+    private readonly outboxService: OutboxService,
+    private readonly inboxService: InboxService,
+    private readonly linkService: IntegrationLinkService,
+    private readonly registry: EventSubscriberRegistry,
+    private readonly integrationSettingsService: IntegrationSettingsService,
+  ) {}
+
+  /**
+   * Process a batch of pending events
+   * Called in a loop or by scheduled job
+   */
+  async processBatch(options?: { batchSizeOverride?: number }): Promise<number> {
+    const settings = await this.integrationSettingsService.getRuntimeSettings();
+    const batchSize = options?.batchSizeOverride ?? settings.batchSize;
+    const leaseMs = settings.processingLeaseMs;
+
+    // Find pending events
+    const events = await this.outboxService.findPending(batchSize, leaseMs);
+
+    if (events.length === 0) {
+      return 0;
+    }
+
+    this.logger.debug(`Processing ${events.length} outbox events`);
+
+    let processedCount = 0;
+
+    for (const event of events) {
+      try {
+        // Mark as processing with lease
+        await this.outboxService.markProcessing(event.id, leaseMs);
+
+        // Get handlers for this event
+        const handlers = this.registry.getHandlers(event.eventName);
+
+        if (handlers.length === 0) {
+          this.logger.warn(
+            `No handlers registered for event: ${event.eventName}`,
+          );
+          // Mark as processed even though no handlers
+          await this.outboxService.updateStatus(
+            event.id,
+            OutboxEventStatus.PROCESSED,
+            {
+              processedAt: new Date(),
+            },
+          );
+          processedCount++;
+          continue;
+        }
+
+        // Execute each handler
+        let anyFailed = false;
+        for (const handlerDef of handlers) {
+          const inboxEvent = await this.inboxService.getOrCreate(
+            event.id,
+            handlerDef.consumerName,
+          );
+
+          // Skip if already processed
+          if (inboxEvent.status === InboxEventStatus.PROCESSED) {
+            this.logger.debug(
+              `Event ${event.id} already processed by ${handlerDef.consumerName}`,
+            );
+            continue;
+          }
+
+          await this.inboxService.markProcessing(inboxEvent.id);
+
+          try {
+            // Execute handler
+            await handlerDef.handler(
+              {
+                eventName: event.eventName,
+                sourceModule: event.sourceModule,
+                aggregateType: event.aggregateType,
+                aggregateId: event.aggregateId,
+                payload: event.payload,
+                timestamp: event.createdAt,
+              },
+              {
+                logger: this.logger,
+                inboxService: this.inboxService,
+                linkService: this.linkService,
+              },
+            );
+
+            // Mark as processed
+            await this.inboxService.markProcessed(inboxEvent.id);
+            this.logger.debug(
+              `Handler ${handlerDef.consumerName} processed event ${event.id}`,
+            );
+          } catch (error) {
+            anyFailed = true;
+            this.logger.error(
+              `Handler ${handlerDef.consumerName} failed for event ${event.id}:`,
+              error,
+            );
+            await this.inboxService.markFailed(
+              inboxEvent.id,
+              error instanceof Error ? error.message : String(error),
+            );
+          }
+        }
+
+        // Update outbox event status based on handler results
+        if (!anyFailed) {
+          await this.outboxService.updateStatus(
+            event.id,
+            OutboxEventStatus.PROCESSED,
+            {
+              processedAt: new Date(),
+            },
+          );
+          processedCount++;
+        } else {
+          // At least one handler failed, retry logic applies
+          await this.handleEventFailure(event.id);
+        }
+      } catch (error) {
+        this.logger.error(`Error processing event ${event.id}:`, error);
+        await this.handleEventFailure(event.id);
+      }
+    }
+
+    return processedCount;
+  }
+
+  /**
+   * Handle event that failed to process
+   */
+  private async handleEventFailure(eventId: string): Promise<void> {
+    const settings = await this.integrationSettingsService.getRuntimeSettings();
+    const maxAttempts = settings.maxAttempts;
+    const retryBaseDelayMs = settings.retryBaseDelayMs;
+    const deadLetterEnabled = settings.deadLetterEnabled;
+
+    const event = await this.outboxService.getById(eventId);
+    if (!event) {
+      return;
+    }
+
+    const nextAttemptCount = event.attemptCount + 1;
+
+    if (nextAttemptCount >= maxAttempts) {
+      if (deadLetterEnabled) {
+        this.logger.warn(
+          `Event ${eventId} moved to dead letter after ${maxAttempts} attempts`,
+        );
+        await this.outboxService.updateStatus(
+          eventId,
+          OutboxEventStatus.DEAD_LETTER,
+          {
+            attemptCount: nextAttemptCount,
+          },
+        );
+      } else {
+        this.logger.warn(
+          `Event ${eventId} failed after ${maxAttempts} attempts (dead letter disabled)`,
+        );
+        await this.outboxService.updateStatus(
+          eventId,
+          OutboxEventStatus.FAILED,
+          {
+            attemptCount: nextAttemptCount,
+          },
+        );
+      }
+    } else {
+      // Calculate exponential backoff
+      const delayMs = retryBaseDelayMs * Math.pow(2, event.attemptCount);
+      const availableAt = new Date(Date.now() + delayMs);
+
+      this.logger.debug(
+        `Retrying event ${eventId} in ${delayMs}ms (attempt ${nextAttemptCount}/${maxAttempts})`,
+      );
+
+      await this.outboxService.updateStatus(
+        eventId,
+        OutboxEventStatus.PENDING,
+        {
+          attemptCount: nextAttemptCount,
+          availableAt,
+        },
+      );
+    }
+  }
+
+  /**
+   * Process all pending events at startup (drain)
+   */
+  async startupDrain(): Promise<number> {
+    const settings = await this.integrationSettingsService.getRuntimeSettings();
+    const drainBatchSize = settings.startupDrainBatchSize;
+
+    this.logger.log(`Starting outbox drain with batch size: ${drainBatchSize}`);
+
+    let totalProcessed = 0;
+
+    // Keep processing until no more events
+    let processed = 1;
+    while (processed > 0) {
+      processed = await this.processBatch({
+        batchSizeOverride: drainBatchSize,
+      });
+      totalProcessed += processed;
+
+      // Small delay to prevent overwhelming the system
+      if (processed > 0) {
+        await new Promise((resolve) => setTimeout(resolve, 100));
+      }
+    }
+
+    this.logger.log(`Startup drain completed. Processed ${totalProcessed} events`);
+    return totalProcessed;
+  }
+
+  /**
+   * Get processor statistics
+   */
+  async getStats() {
+    const pendingCount = await this.outboxService.countByStatus(
+      OutboxEventStatus.PENDING,
+    );
+    const processingCount = await this.outboxService.countByStatus(
+      OutboxEventStatus.PROCESSING,
+    );
+    const processedCount = await this.outboxService.countByStatus(
+      OutboxEventStatus.PROCESSED,
+    );
+    const failedCount = await this.outboxService.countByStatus(
+      OutboxEventStatus.FAILED,
+    );
+    const deadLetterCount = await this.outboxService.countByStatus(
+      OutboxEventStatus.DEAD_LETTER,
+    );
+
+    return {
+      pending: pendingCount,
+      processing: processingCount,
+      processed: processedCount,
+      failed: failedCount,
+      dead_letter: deadLetterCount,
+      total:
+        pendingCount +
+        processingCount +
+        processedCount +
+        failedCount +
+        deadLetterCount,
+      handlerCount: this.registry.getHandlerCount(),
+    };
+  }
+}

package/src/integration/services/outbox.service.ts

@@ -0,0 +1,209 @@
+import { PrismaService } from '@hed-hog/api-prisma';
+import { Injectable } from '@nestjs/common';
+import { OutboxEvent, OutboxEventStatus } from '../types';
+
+export interface CreateOutboxEventDto {
+  eventName: string;
+  sourceModule: string;
+  aggregateType: string;
+  aggregateId: string;
+  payload: Record<string, any>;
+}
+
+export interface OutboxEventPersistenceClient {
+  outbox_event: PrismaService['outbox_event'];
+}
+
+interface OutboxEventRecord {
+  id: number;
+  event_name: string;
+  source_module: string;
+  aggregate_type: string;
+  aggregate_id: string;
+  payload: Record<string, any>;
+  status: OutboxEventStatus;
+  attempt_count: number;
+  last_error: string | null;
+  available_at: Date;
+  processed_at: Date | null;
+  created_at: Date;
+  updated_at: Date;
+}
+
+@Injectable()
+export class OutboxService {
+  constructor(private readonly prisma: PrismaService) {}
+
+  private toDomainEvent(record: OutboxEventRecord): OutboxEvent {
+    return {
+      id: String(record.id),
+      eventName: record.event_name,
+      sourceModule: record.source_module,
+      aggregateType: record.aggregate_type,
+      aggregateId: record.aggregate_id,
+      payload: record.payload,
+      status: record.status,
+      attemptCount: record.attempt_count,
+      lastError: record.last_error,
+      availableAt: record.available_at,
+      processedAt: record.processed_at,
+      createdAt: record.created_at,
+      updatedAt: record.updated_at,
+    };
+  }
+
+  private toDatabaseId(eventId: string | number): number {
+    return typeof eventId === 'number' ? eventId : Number(eventId);
+  }
+
+  private toDatabaseUpdate(
+    partialUpdate?: {
+      attemptCount?: number;
+      lastError?: string | null;
+      availableAt?: Date;
+      processedAt?: Date | null;
+    },
+  ) {
+    if (!partialUpdate) {
+      return undefined;
+    }
+
+    return {
+      attempt_count: partialUpdate.attemptCount,
+      last_error: partialUpdate.lastError,
+      available_at: partialUpdate.availableAt,
+      processed_at: partialUpdate.processedAt,
+    };
+  }
+
+  /**
+   * Write a new event to the outbox
+   */
+  async createEvent(
+    dto: CreateOutboxEventDto,
+    persistenceClient?: OutboxEventPersistenceClient,
+  ): Promise<OutboxEvent> {
+    const client = persistenceClient ?? this.prisma;
+
+    const record = await client.outbox_event.create({
+      data: {
+        event_name: dto.eventName,
+        source_module: dto.sourceModule,
+        aggregate_type: dto.aggregateType,
+        aggregate_id: dto.aggregateId,
+        payload: dto.payload,
+        status: OutboxEventStatus.PENDING,
+        available_at: new Date(),
+        attempt_count: 0,
+      },
+    });
+
+    return this.toDomainEvent(record as OutboxEventRecord);
+  }
+
+  /**
+   * Find pending events available for processing, ordered by creation time
+   */
+  async findPending(
+    limit: number,
+    leaseMs: number,
+  ): Promise<OutboxEvent[]> {
+    const now = new Date();
+    const leaseThreshold = new Date(now.getTime() - leaseMs);
+
+    const records = await this.prisma.outbox_event.findMany({
+      where: {
+        OR: [
+          { status: OutboxEventStatus.PENDING },
+          {
+            status: OutboxEventStatus.PROCESSING,
+            available_at: { lte: leaseThreshold },
+          },
+        ],
+      },
+      orderBy: { created_at: 'asc' },
+      take: limit,
+    });
+
+    return records.map((record) => this.toDomainEvent(record as OutboxEventRecord));
+  }
+
+  /**
+   * Update event status and related fields
+   */
+  async updateStatus(
+    eventId: string | number,
+    status: OutboxEventStatus,
+    partialUpdate?: {
+      attemptCount?: number;
+      lastError?: string | null;
+      availableAt?: Date;
+      processedAt?: Date | null;
+    },
+  ): Promise<OutboxEvent> {
+    const record = await this.prisma.outbox_event.update({
+      where: { id: this.toDatabaseId(eventId) },
+      data: {
+        status,
+        ...this.toDatabaseUpdate(partialUpdate),
+      },
+    });
+
+    return this.toDomainEvent(record as OutboxEventRecord);
+  }
+
+  /**
+   * Mark event as processing and set lease time
+   */
+  async markProcessing(
+    eventId: string | number,
+    leaseMs: number,
+  ): Promise<OutboxEvent> {
+    const leaseUntil = new Date(Date.now() + leaseMs);
+    return this.updateStatus(eventId, OutboxEventStatus.PROCESSING, {
+      availableAt: leaseUntil,
+    });
+  }
+
+  /**
+   * Increment attempt count and optionally update error
+   */
+  async incrementAttempt(
+    eventId: string | number,
+    error?: string,
+  ): Promise<OutboxEvent> {
+    const record = await this.prisma.outbox_event.findUnique({
+      where: { id: this.toDatabaseId(eventId) },
+    });
+    if (!record) {
+      throw new Error(`Event ${eventId} not found`);
+    }
+
+    const event = this.toDomainEvent(record as OutboxEventRecord);
+
+    return this.updateStatus(eventId, event.status, {
+      attemptCount: event.attemptCount + 1,
+      lastError: error || null,
+    });
+  }
+
+  /**
+   * Get event by ID
+   */
+  async getById(eventId: string | number): Promise<OutboxEvent | null> {
+    const record = await this.prisma.outbox_event.findUnique({
+      where: { id: this.toDatabaseId(eventId) },
+    });
+
+    return record ? this.toDomainEvent(record as OutboxEventRecord) : null;
+  }
+
+  /**
+   * Count events by status
+   */
+  async countByStatus(status: OutboxEventStatus): Promise<number> {
+    return this.prisma.outbox_event.count({
+      where: { status },
+    });
+  }
+}

package/src/integration/types/event.types.ts

@@ -0,0 +1,93 @@
+/**
+ * Event status throughout the integration lifecycle
+ */
+export enum OutboxEventStatus {
+  PENDING = 'pending',
+  PROCESSING = 'processing',
+  PROCESSED = 'processed',
+  FAILED = 'failed',
+  DEAD_LETTER = 'dead_letter',
+}
+
+/**
+ * Inbox event processing status per consumer
+ */
+export enum InboxEventStatus {
+  RECEIVED = 'received',
+  PROCESSING = 'processing',
+  PROCESSED = 'processed',
+  FAILED = 'failed',
+  SKIPPED = 'skipped',
+}
+
+/**
+ * Type of link between entities from different modules
+ */
+export enum LinkType {
+  REFERENCE = 'reference',
+  CASCADE = 'cascade',
+  AGGREGATE_ROOT = 'aggregate_root',
+}
+
+/**
+ * Domain event interface that is published and processed
+ */
+export interface DomainEvent {
+  eventName: string;
+  sourceModule: string;
+  aggregateType: string;
+  aggregateId: string;
+  payload: Record<string, any>;
+  timestamp: Date;
+}
+
+/**
+ * Outbox event record from database
+ */
+export interface OutboxEvent {
+  id: string;
+  eventName: string;
+  sourceModule: string;
+  aggregateType: string;
+  aggregateId: string;
+  payload: Record<string, any>;
+  status: OutboxEventStatus;
+  attemptCount: number;
+  lastError: string | null;
+  availableAt: Date;
+  processedAt: Date | null;
+  createdAt: Date;
+  updatedAt: Date;
+}
+
+/**
+ * Inbox event record from database for idempotency tracking
+ */
+export interface InboxEvent {
+  id: string;
+  outboxEventId: string;
+  consumerName: string;
+  status: InboxEventStatus;
+  attemptCount: number;
+  lastError: string | null;
+  processedAt: Date | null;
+  createdAt: Date;
+  updatedAt: Date;
+}
+
+/**
+ * Integration link between entities across modules
+ */
+export interface IntegrationLink {
+  id: string;
+  sourceModule: string;
+  sourceEntityType: string;
+  sourceEntityId: string;
+  targetModule: string;
+  targetEntityType: string;
+  targetEntityId: string;
+  linkType: LinkType;
+  metadata: Record<string, any> | null;
+  createdAt: Date;
+  updatedAt: Date;
+}

package/src/integration/types/index.ts

@@ -0,0 +1,3 @@
+export * from './event.types';
+export * from './subscriber.types';
+

package/src/integration/types/subscriber.types.ts

@@ -0,0 +1,37 @@
+import { Logger } from '@nestjs/common';
+import { DomainEvent } from './event.types';
+
+/**
+ * Context passed to event handlers
+ */
+export interface SubscriberContext {
+  logger: Logger;
+  inboxService: any; // To be injected at runtime
+  linkService: any; // To be injected at runtime
+}
+
+/**
+ * Event handler function signature
+ */
+export type EventHandler = (
+  event: DomainEvent,
+  context: SubscriberContext,
+) => Promise<void>;
+
+/**
+ * Event subscriber definition
+ */
+export interface SubscriberDefinition {
+  eventName: string;
+  consumerName: string;
+  priority?: number;
+  handler: EventHandler;
+}
+
+/**
+ * Subscription registry entry
+ */
+export interface SubscriptionEntry {
+  definition: SubscriberDefinition;
+  handler: EventHandler;
+}

package/src/oauth/oauth.controller.ts

@@ -1,17 +1,17 @@
 
 import { Public, User } from '@hed-hog/api';
 import { Locale } from '@hed-hog/api-locale';
-import { user_account_provider_enum } from '@hed-hog/api-prisma';
+import { user_account_provider_52222e2ecb_enum } from '@hed-hog/api-prisma';
 import {
-  Body,
-  Controller,
-  Delete,
-  Get,
-  Headers,
-  Ip,
-  Param,
-  Query,
-  Res,
+    Body,
+    Controller,
+    Delete,
+    Get,
+    Headers,
+    Ip,
+    Param,
+    Query,
+    Res,
 } from '@nestjs/common';
 import { SettingService } from '../setting/setting.service';
 import { OAuthService } from './oauth.service';
@@ -37,7 +37,7 @@ export class OAuthController {
   @Get(':provider/login')
   async login(@Param('provider') provider: string, @Res() res) {
     const url = await this.service.getAuthUrl(
-      provider as user_account_provider_enum,
+      provider as user_account_provider_52222e2ecb_enum,
       `/callback/${provider}/login`,
     );
     return res.redirect(url);
@@ -48,7 +48,7 @@ export class OAuthController {
   @Get(':provider/register')
   async register(@Param('provider') provider: string, @Res() res) {
     const url = await this.service.getAuthUrl(
-      provider as user_account_provider_enum,
+      provider as user_account_provider_52222e2ecb_enum,
       `/callback/${provider}/register`,
     );
     return res.redirect(url);
@@ -59,7 +59,7 @@ export class OAuthController {
   @Get(':provider/connect')
   async connect(@Param('provider') provider: string, @Res() res) {
     const url = this.service.getAuthUrl(
-      provider as user_account_provider_enum,
+      provider as user_account_provider_52222e2ecb_enum,
       `/callback/${provider}/connect`,
     );
     return res.redirect(url);
@@ -76,7 +76,7 @@ export class OAuthController {
     @Query('code') code: string,
     @Res({ passthrough: true }) res,
   ) {
-    return this.service.handleCallback({ res, locale, ipAddress, userAgent, provider: provider as user_account_provider_enum, code, type: 'login' });
+    return this.service.handleCallback({ res, locale, ipAddress, userAgent, provider: provider as user_account_provider_52222e2ecb_enum, code, type: 'login' });
   }
 
 
@@ -90,7 +90,7 @@ export class OAuthController {
     @Query('code') code: string,
     @Res({ passthrough: true }) res,
   ) {
-    return this.service.handleCallback({ res, locale, ipAddress, userAgent, provider: provider as user_account_provider_enum, code, type: 'register' });
+    return this.service.handleCallback({ res, locale, ipAddress, userAgent, provider: provider as user_account_provider_52222e2ecb_enum, code, type: 'register' });
   }
 
 
@@ -104,7 +104,7 @@ export class OAuthController {
     @Query('code') code: string,
     @Res({ passthrough: true }) res,
   ) {
-    return this.service.handleCallback({ res, locale, ipAddress, userAgent, provider: provider as user_account_provider_enum, code, type: 'connect', userId: id });
+    return this.service.handleCallback({ res, locale, ipAddress, userAgent, provider: provider as user_account_provider_52222e2ecb_enum, code, type: 'connect', userId: id });
   }
 
 
@@ -117,6 +117,6 @@ export class OAuthController {
     @Body('email') email: string,
     @Res({ passthrough: true }) res,
   ) {
-    return this.service.handleCallback({ res, locale, ipAddress, userAgent, provider: provider as user_account_provider_enum, email, type: 'disconnect' });
+    return this.service.handleCallback({ res, locale, ipAddress, userAgent, provider: provider as user_account_provider_52222e2ecb_enum, email, type: 'disconnect' });
   }
 }