@squiz/dx-common-lib 1.72.1 → 1.72.3

package/src/stream/EventStreamHandler.spec.ts

@@ -0,0 +1,440 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+import { createEventStreamHandler, EventMapper } from './EventStreamHandler';
+import { DynamoDBStreamEvent, DynamoDBRecord, Context } from 'aws-lambda';
+
+interface MockLogger {
+  info: jest.Mock;
+  debug: jest.Mock;
+  error: jest.Mock;
+}
+
+interface MockEventBusService {
+  setTenantId: jest.Mock;
+  publishEvents: jest.Mock;
+}
+
+describe('EventStreamHandler', () => {
+  let mockLogger: MockLogger;
+  let mockEventBusService: MockEventBusService;
+
+  beforeEach(() => {
+    mockLogger = {
+      info: jest.fn(),
+      debug: jest.fn(),
+      error: jest.fn(),
+    };
+
+    mockEventBusService = {
+      setTenantId: jest.fn(),
+      publishEvents: jest.fn().mockResolvedValue(undefined),
+    };
+  });
+
+  const createMockRecord = (overrides: Partial<DynamoDBRecord> = {}): DynamoDBRecord => {
+    return {
+      eventID: '1',
+      eventVersion: '1.0',
+      dynamodb: {
+        Keys: {
+          pk: { S: 'cv#component-123' },
+          sk: { S: 'v#1.0.0' },
+        },
+        SequenceNumber: '111',
+        SizeBytes: 26,
+        StreamViewType: 'NEW_AND_OLD_IMAGES',
+      },
+      awsRegion: 'us-east-1',
+      eventName: 'INSERT',
+      eventSource: 'aws:dynamodb',
+      eventSourceARN:
+        'arn:aws:dynamodb:us-east-1:123456789012:table/dev.cmp-service.tenant123/stream/2024-01-01T00:00:00.000',
+      ...overrides,
+    } as DynamoDBRecord;
+  };
+
+  const createMockContext = (): Context =>
+    ({
+      awsRequestId: 'test-request-id',
+      invokeid: 'test-invoke-id',
+      logGroupName: 'test-log-group',
+      logStreamName: 'test-log-stream',
+      functionName: 'test-function',
+      memoryLimitInMB: '128',
+      functionVersion: '1',
+      getRemainingTimeInMillis: jest.fn(() => 30000),
+      done: jest.fn(),
+      fail: jest.fn(),
+      succeed: jest.fn(),
+    } as unknown as Context);
+
+  describe('createEventStreamHandler', () => {
+    it('should successfully process records with mapped events', async () => {
+      const mockMapper: EventMapper = jest.fn(async () => [
+        { detailType: 'component.change.create', detail: { name: 'test' } },
+      ]);
+
+      const handle = createEventStreamHandler({
+        logger: mockLogger as any,
+        eventBusService: mockEventBusService as any,
+        tableServiceIdentifier: 'cmp',
+        entityTypePrefixMap: { 'cv#': 'componentVersion' },
+        mappers: { componentVersion: mockMapper },
+        extractTenantId: (arn: string, _identifier: string) => {
+          const tableName = arn.split('/')[1];
+          const parts = tableName.split('.');
+          return parts[parts.length - 1];
+        },
+        batchSize: 10,
+      });
+
+      const event: DynamoDBStreamEvent = {
+        Records: [createMockRecord()],
+      };
+
+      const response = await handle(event, createMockContext());
+
+      expect(mockLogger.info).toHaveBeenCalledWith('Processing DynamoDB stream batch', expect.any(Object));
+      expect(mockEventBusService.setTenantId).toHaveBeenCalledWith('tenant123');
+      expect(mockEventBusService.publishEvents).toHaveBeenCalledWith([
+        { detailType: 'component.change.create', detail: { name: 'test' } },
+      ]);
+      expect(response.batchItemFailures).toHaveLength(0);
+    });
+
+    it('should pass the stream record as the third argument to extractTenantId', async () => {
+      const extractTenantId = jest.fn((arn: string, _identifier: string, _record: DynamoDBRecord) => {
+        const tableName = arn.split('/')[1];
+        const parts = tableName.split('.');
+        return parts[parts.length - 1];
+      });
+
+      const mockMapper: EventMapper = jest.fn(async () => [{ detailType: 'component.change.create', detail: {} }]);
+
+      const handle = createEventStreamHandler({
+        logger: mockLogger as any,
+        eventBusService: mockEventBusService as any,
+        tableServiceIdentifier: 'cmp',
+        entityTypePrefixMap: { 'cv#': 'componentVersion' },
+        mappers: { componentVersion: mockMapper },
+        extractTenantId,
+      });
+
+      const streamRecord = createMockRecord();
+      await handle({ Records: [streamRecord] }, createMockContext());
+
+      expect(extractTenantId).toHaveBeenCalledWith(streamRecord.eventSourceARN, 'cmp', streamRecord);
+    });
+
+    it('should handle records with no mapper registered', async () => {
+      const handle = createEventStreamHandler({
+        logger: mockLogger as any,
+        eventBusService: mockEventBusService as any,
+        tableServiceIdentifier: 'cmp',
+        entityTypePrefixMap: { 'cv#': 'componentVersion' },
+        mappers: {},
+        extractTenantId: (arn: string, _identifier: string) => {
+          const tableName = arn.split('/')[1];
+          const parts = tableName.split('.');
+          return parts[parts.length - 1];
+        },
+      });
+
+      const event: DynamoDBStreamEvent = {
+        Records: [createMockRecord()],
+      };
+
+      const response = await handle(event, createMockContext());
+
+      expect(mockLogger.debug).toHaveBeenCalledWith('No event mapper registered for entity type: componentVersion');
+      expect(mockEventBusService.publishEvents).not.toHaveBeenCalled();
+      expect(response.batchItemFailures).toHaveLength(0);
+    });
+
+    it('should handle records with unknown entity types', async () => {
+      const handle = createEventStreamHandler({
+        logger: mockLogger as any,
+        eventBusService: mockEventBusService as any,
+        tableServiceIdentifier: 'cmp',
+        entityTypePrefixMap: { 'cs#': 'componentSet' },
+        mappers: {},
+        extractTenantId: (arn: string, _identifier: string) => {
+          const tableName = arn.split('/')[1];
+          const parts = tableName.split('.');
+          return parts[parts.length - 1];
+        },
+      });
+
+      const event: DynamoDBStreamEvent = {
+        Records: [createMockRecord()],
+      };
+
+      const response = await handle(event, createMockContext());
+
+      expect(mockEventBusService.publishEvents).not.toHaveBeenCalled();
+      expect(response.batchItemFailures).toHaveLength(0);
+    });
+
+    it('should handle records without eventSourceARN', async () => {
+      const handle = createEventStreamHandler({
+        logger: mockLogger as any,
+        eventBusService: mockEventBusService as any,
+        tableServiceIdentifier: 'cmp',
+        entityTypePrefixMap: { 'cv#': 'componentVersion' },
+        mappers: {},
+        extractTenantId: (arn: string, _identifier: string) => {
+          const tableName = arn.split('/')[1];
+          const parts = tableName.split('.');
+          return parts[parts.length - 1];
+        },
+      });
+
+      const event: DynamoDBStreamEvent = {
+        Records: [createMockRecord({ eventSourceARN: undefined })],
+      };
+
+      const response = await handle(event, createMockContext());
+
+      expect(mockLogger.error).toHaveBeenCalledWith('Failed to process record', expect.any(Object));
+      expect(response.batchItemFailures).toHaveLength(0);
+    });
+
+    it('should batch records according to batchSize', async () => {
+      const mockMapper: EventMapper = jest.fn(async () => [{ detailType: 'component.change.create', detail: {} }]);
+
+      const handle = createEventStreamHandler({
+        logger: mockLogger as any,
+        eventBusService: mockEventBusService as any,
+        tableServiceIdentifier: 'cmp',
+        entityTypePrefixMap: { 'cv#': 'componentVersion' },
+        mappers: { componentVersion: mockMapper },
+        extractTenantId: (arn: string, _identifier: string) => {
+          const tableName = arn.split('/')[1];
+          const parts = tableName.split('.');
+          return parts[parts.length - 1];
+        },
+        batchSize: 2,
+      });
+
+      const records = [
+        createMockRecord({ eventID: '1' }),
+        createMockRecord({ eventID: '2' }),
+        createMockRecord({ eventID: '3' }),
+      ];
+
+      const event: DynamoDBStreamEvent = { Records: records };
+      await handle(event, createMockContext());
+
+      expect(mockMapper).toHaveBeenCalledTimes(3);
+    });
+
+    it('should mark records as failed when event publishing fails', async () => {
+      const mockMapper: EventMapper = jest.fn(async () => [
+        { detailType: 'component.change.create', detail: { name: 'test' } },
+      ]);
+
+      mockEventBusService.publishEvents.mockRejectedValue(new Error('Publishing failed'));
+
+      const handle = createEventStreamHandler({
+        logger: mockLogger as any,
+        eventBusService: mockEventBusService as any,
+        tableServiceIdentifier: 'cmp',
+        entityTypePrefixMap: { 'cv#': 'componentVersion' },
+        mappers: { componentVersion: mockMapper },
+        extractTenantId: (arn: string, _identifier: string) => {
+          const tableName = arn.split('/')[1];
+          const parts = tableName.split('.');
+          return parts[parts.length - 1];
+        },
+      });
+
+      const event: DynamoDBStreamEvent = {
+        Records: [createMockRecord()],
+      };
+
+      const response = await handle(event, createMockContext());
+
+      expect(mockLogger.error).toHaveBeenCalledWith('Failed to publish events for tenant', expect.any(Object));
+      expect(response.batchItemFailures).toHaveLength(1);
+      expect(response.batchItemFailures[0].itemIdentifier).toBe('111');
+    });
+
+    it('should handle mapper returning empty events', async () => {
+      const mockMapper: EventMapper = jest.fn(async () => []);
+
+      const handle = createEventStreamHandler({
+        logger: mockLogger as any,
+        eventBusService: mockEventBusService as any,
+        tableServiceIdentifier: 'cmp',
+        entityTypePrefixMap: { 'cv#': 'componentVersion' },
+        mappers: { componentVersion: mockMapper },
+        extractTenantId: (arn: string, _identifier: string) => {
+          const tableName = arn.split('/')[1];
+          const parts = tableName.split('.');
+          return parts[parts.length - 1];
+        },
+      });
+
+      const event: DynamoDBStreamEvent = {
+        Records: [createMockRecord()],
+      };
+
+      const response = await handle(event, createMockContext());
+
+      expect(mockEventBusService.publishEvents).not.toHaveBeenCalled();
+      expect(response.batchItemFailures).toHaveLength(0);
+    });
+
+    it('should handle multiple mappers for different entity types', async () => {
+      const cvMapper: EventMapper = jest.fn(async () => [
+        { detailType: 'component.change.create', detail: { type: 'cv' } },
+      ]);
+      const csMapper: EventMapper = jest.fn(async () => [{ detailType: 'set.change.create', detail: { type: 'cs' } }]);
+
+      const handle = createEventStreamHandler({
+        logger: mockLogger as any,
+        eventBusService: mockEventBusService as any,
+        tableServiceIdentifier: 'cmp',
+        entityTypePrefixMap: { 'cv#': 'componentVersion', 'cs#': 'componentSet' },
+        mappers: { componentVersion: cvMapper, componentSet: csMapper },
+        extractTenantId: (arn: string, _identifier: string) => {
+          const tableName = arn.split('/')[1];
+          const parts = tableName.split('.');
+          return parts[parts.length - 1];
+        },
+      });
+
+      const event: DynamoDBStreamEvent = {
+        Records: [
+          createMockRecord({ eventID: '1' }),
+          createMockRecord({ eventID: '2', dynamodb: { Keys: { pk: { S: 'cs#set-123' } } } as any }),
+        ],
+      };
+
+      await handle(event, createMockContext());
+
+      expect(cvMapper).toHaveBeenCalledTimes(1);
+      expect(csMapper).toHaveBeenCalledTimes(1);
+    });
+
+    it('should handle mapper errors gracefully', async () => {
+      const mockMapper: EventMapper = jest.fn(async () => {
+        throw new Error('Mapper failed');
+      });
+
+      const handle = createEventStreamHandler({
+        logger: mockLogger as any,
+        eventBusService: mockEventBusService as any,
+        tableServiceIdentifier: 'cmp',
+        entityTypePrefixMap: { 'cv#': 'componentVersion' },
+        mappers: { componentVersion: mockMapper },
+        extractTenantId: (arn: string, _identifier: string) => {
+          const tableName = arn.split('/')[1];
+          const parts = tableName.split('.');
+          return parts[parts.length - 1];
+        },
+      });
+
+      const event: DynamoDBStreamEvent = {
+        Records: [createMockRecord()],
+      };
+
+      const response = await handle(event, createMockContext());
+
+      expect(mockLogger.error).toHaveBeenCalledWith('Failed to process record', expect.any(Object));
+      expect(response.batchItemFailures).toHaveLength(0);
+    });
+
+    it('should group events by tenant and publish separately', async () => {
+      const mockMapper: EventMapper = jest.fn(async () => [{ detailType: 'component.change.create', detail: {} }]);
+
+      const handle = createEventStreamHandler({
+        logger: mockLogger as any,
+        eventBusService: mockEventBusService as any,
+        tableServiceIdentifier: 'cmp',
+        entityTypePrefixMap: { 'cv#': 'componentVersion' },
+        mappers: { componentVersion: mockMapper },
+        extractTenantId: (arn: string) => {
+          if (arn.includes('tenant1')) return 'tenant1';
+          if (arn.includes('tenant2')) return 'tenant2';
+          return 'unknown';
+        },
+      });
+
+      const event: DynamoDBStreamEvent = {
+        Records: [
+          createMockRecord({
+            eventID: '1',
+            eventSourceARN: 'arn:aws:dynamodb:us-east-1:123456789012:table/dev.cmp-service.tenant1/stream/ts',
+          }),
+          createMockRecord({
+            eventID: '2',
+            eventSourceARN: 'arn:aws:dynamodb:us-east-1:123456789012:table/dev.cmp-service.tenant2/stream/ts',
+          }),
+        ],
+      };
+
+      await handle(event, createMockContext());
+
+      expect(mockEventBusService.setTenantId).toHaveBeenCalledWith('tenant1');
+      expect(mockEventBusService.setTenantId).toHaveBeenCalledWith('tenant2');
+      expect(mockEventBusService.publishEvents).toHaveBeenCalledTimes(2);
+    });
+
+    it('should handle records without primary key', async () => {
+      const handle = createEventStreamHandler({
+        logger: mockLogger as any,
+        eventBusService: mockEventBusService as any,
+        tableServiceIdentifier: 'cmp',
+        entityTypePrefixMap: { 'cv#': 'componentVersion' },
+        mappers: {},
+        extractTenantId: (arn: string, _identifier: string) => {
+          const tableName = arn.split('/')[1];
+          const parts = tableName.split('.');
+          return parts[parts.length - 1];
+        },
+      });
+
+      const event: DynamoDBStreamEvent = {
+        Records: [createMockRecord({ dynamodb: { SequenceNumber: '111', SizeBytes: 0 } as any })],
+      };
+
+      const response = await handle(event, createMockContext());
+
+      expect(response.batchItemFailures).toHaveLength(0);
+    });
+
+    it('should log batch processing summary', async () => {
+      const mockMapper: EventMapper = jest.fn(async () => [{ detailType: 'component.change.create', detail: {} }]);
+
+      const handle = createEventStreamHandler({
+        logger: mockLogger as any,
+        eventBusService: mockEventBusService as any,
+        tableServiceIdentifier: 'cmp',
+        entityTypePrefixMap: { 'cv#': 'componentVersion' },
+        mappers: { componentVersion: mockMapper },
+        extractTenantId: (arn: string, _identifier: string) => {
+          const tableName = arn.split('/')[1];
+          const parts = tableName.split('.');
+          return parts[parts.length - 1];
+        },
+      });
+
+      const event: DynamoDBStreamEvent = {
+        Records: [createMockRecord()],
+      };
+
+      await handle(event, createMockContext());
+
+      const finalLog = mockLogger.info.mock.calls.find((call) => call[0] === 'Batch processing complete');
+      expect(finalLog).toBeDefined();
+      expect(finalLog[1]).toEqual({
+        totalRecords: 1,
+        failedRecords: 0,
+        service: 'cmp',
+      });
+    });
+  });
+});

package/src/stream/EventStreamHandler.ts

@@ -0,0 +1,192 @@
+/*!
+ * @license
+ * Copyright Squiz Australia Pty Ltd. All Rights Reserved.
+ */
+import { DynamoDBStreamEvent, DynamoDBRecord, Context, DynamoDBBatchResponse } from 'aws-lambda';
+import { Logger } from '@squiz/dx-logger-lib';
+import { EventBusService } from '../events/EventBusService';
+
+export interface EventPayload {
+  detailType: string;
+  detail: object;
+}
+
+export type EventMapper = (record: DynamoDBRecord) => Promise<EventPayload[]>;
+
+/** Resolves tenant id for publishing; use `record` when tenant is carried on the item rather than derivable from the table ARN. */
+export type ExtractTenantId = (arn: string, tableServiceIdentifier: string, record: DynamoDBRecord) => string;
+
+interface MappedRecord {
+  recordIndex: number;
+  record: DynamoDBRecord;
+  tenantId: string;
+  events: EventPayload[];
+}
+
+interface EventStreamHandlerConfig<T extends string> {
+  logger: Logger;
+  eventBusService: EventBusService;
+  /** Matches the table name segment in ARNs (e.g. `cmp` for `*.cmp-service.*`); also used as log field `service`. */
+  tableServiceIdentifier: string;
+  entityTypePrefixMap: Record<string, T>;
+  mappers: Partial<Record<T, EventMapper>>;
+  extractTenantId: ExtractTenantId;
+  batchSize?: number;
+}
+
+/**
+ * DynamoDB stream Lambda handler factory for mapping stream records to domain events and publishing via the event bus.
+ *
+ * Handles:
+ * - Record batching by configurable size
+ * - Tenant extraction from stream ARN and/or stream record (via `extractTenantId`)
+ * - Entity type detection via prefix matching
+ * - Mapper invocation for known entity types
+ * - Event grouping by tenant
+ * - Batch event publishing with retry tracking
+ * - Failure reporting for Lambda retry mechanism
+ */
+export function createEventStreamHandler<T extends string>(config: EventStreamHandlerConfig<T>) {
+  const {
+    logger,
+    eventBusService,
+    tableServiceIdentifier,
+    entityTypePrefixMap,
+    mappers,
+    extractTenantId,
+    batchSize = 10,
+  } = config;
+
+  function extractEntityType(record: DynamoDBRecord): T | undefined {
+    const pk = record.dynamodb?.Keys?.pk?.S;
+    if (!pk) {
+      return undefined;
+    }
+
+    for (const [prefix, entityType] of Object.entries(entityTypePrefixMap)) {
+      if (pk.startsWith(prefix)) {
+        return entityType as T;
+      }
+    }
+
+    return undefined;
+  }
+
+  return async function handler(event: DynamoDBStreamEvent, context: Context): Promise<DynamoDBBatchResponse> {
+    logger.info('Processing DynamoDB stream batch', {
+      recordCount: event.Records.length,
+      requestId: context.awsRequestId,
+      service: tableServiceIdentifier,
+    });
+
+    const mappedRecords: MappedRecord[] = [];
+
+    // Process records in batches
+    for (let i = 0; i < event.Records.length; i += batchSize) {
+      const batch = event.Records.slice(i, i + batchSize);
+
+      const batchResults = await Promise.all(
+        batch.map(async (record, batchIndex) => {
+          try {
+            if (!record.eventSourceARN) {
+              throw new Error('Missing eventSourceARN on stream record');
+            }
+
+            const tenantId = extractTenantId(record.eventSourceARN, tableServiceIdentifier, record);
+            const tableName = record.eventSourceARN.split('/')[1];
+            const entityType = extractEntityType(record);
+
+            logger.info('Processed stream record', {
+              eventName: record.eventName,
+              tenantId,
+              tableName,
+              entityType,
+              pk: record.dynamodb?.Keys?.pk?.S,
+              sk: record.dynamodb?.Keys?.sk?.S,
+            });
+
+            if (entityType !== undefined) {
+              const mapper = mappers[entityType];
+              if (mapper) {
+                // Note: For CMP, mappers return 0 or 1 event per record
+                const events = await mapper(record);
+                return { recordIndex: i + batchIndex, record, tenantId, events };
+              } else {
+                logger.debug(`No event mapper registered for entity type: ${entityType}`);
+                return { recordIndex: i + batchIndex, record, tenantId, events: [] };
+              }
+            }
+
+            return { recordIndex: i + batchIndex, record, tenantId, events: [] };
+          } catch (error) {
+            logger.error('Failed to process record', {
+              error: error instanceof Error ? error.message : String(error),
+              eventSourceARN: record.eventSourceARN,
+            });
+            return { recordIndex: i + batchIndex, record, tenantId: 'unknown', events: [] };
+          }
+        }),
+      );
+
+      mappedRecords.push(...batchResults);
+    }
+
+    // Group events by tenant and publish separately for each tenant
+    const eventsByTenant = new Map<
+      string,
+      Array<{ recordIndex: number; record: DynamoDBRecord; events: EventPayload[] }>
+    >();
+
+    for (const { recordIndex, record, tenantId, events } of mappedRecords) {
+      if (events.length > 0) {
+        const existing = eventsByTenant.get(tenantId) || [];
+        existing.push({ recordIndex, record, events });
+        eventsByTenant.set(tenantId, existing);
+      }
+    }
+
+    // Track failed records for retry
+    const failedRecordIndices = new Set<string>();
+
+    // Publish batches for each tenant with their own context in parallel
+    await Promise.all(
+      Array.from(eventsByTenant.entries()).map(async ([tenantId, recordsWithEvents]) => {
+        const events = recordsWithEvents.flatMap((r) => r.events);
+        logger.info('Publishing batch of events for tenant', { tenantId, eventCount: events.length });
+
+        try {
+          eventBusService.setTenantId(tenantId);
+          await eventBusService.publishEvents(events);
+          logger.info('Successfully published all events for tenant', { tenantId });
+        } catch (error) {
+          logger.error('Failed to publish events for tenant', {
+            tenantId,
+            error: error instanceof Error ? error.message : String(error),
+          });
+
+          // Mark all records for this tenant as failed
+          for (const { record } of recordsWithEvents) {
+            const sequenceNumber = record.dynamodb?.SequenceNumber;
+            if (sequenceNumber) {
+              failedRecordIndices.add(sequenceNumber);
+            }
+          }
+        }
+      }),
+    );
+
+    const batchItemFailures: Array<{ itemIdentifier: string }> = Array.from(failedRecordIndices).map(
+      (sequenceNumber) => ({
+        itemIdentifier: sequenceNumber,
+      }),
+    );
+
+    logger.info('Batch processing complete', {
+      totalRecords: event.Records.length,
+      failedRecords: batchItemFailures.length,
+      service: tableServiceIdentifier,
+    });
+
+    return { batchItemFailures };
+  };
+}