@od-oneapp/analytics 2026.1.1301

package/src/shared/ingestion/service.ts

@@ -0,0 +1,628 @@
+/**
+ * @fileoverview Event Ingestion Service
+ *
+ * Provides server-side event ingestion functionality. Handles validation,
+ * normalization, and forwarding of events to the analytics system.
+ *
+ * **Key Features**:
+ * - Validates incoming events against Zod schemas
+ * - Normalizes event payloads with defaults
+ * - Forwards events to AnalyticsManager
+ * - Tracks ingestion metrics
+ * - Handles batch processing efficiently
+ *
+ * @module @od-oneapp/analytics/shared/ingestion/service
+ */
+
+import { logDebug, logError, logInfo, logWarn } from '@repo/shared/logs';
+
+import { sanitizeProperties, validateEventName } from '../utils/security';
+
+import { BatchEventRequestSchema, EventPayloadSchema, IngestionRequestSchema } from './schemas';
+
+import type {
+  BatchEventRequest,
+  EmitterContext,
+  EventPayload,
+  EventResult,
+  IngestionErrorResponse,
+  IngestionRequest,
+  IngestionSuccessResponse,
+} from './schemas';
+import type { EmitterPayload } from '../emitters/emitter-types';
+import type { AnalyticsManager } from '../types/types';
+
+// =============================================================================
+// Types
+// =============================================================================
+
+/**
+ * Ingestion context provided by the caller (usually the API route handler).
+ */
+export interface IngestionContext {
+  /** Source identifier (app/module/emitter id) */
+  source: string;
+
+  /** Tenant/project ID for multi-tenancy */
+  tenantId?: string;
+
+  /** User ID from auth context */
+  userId?: string;
+
+  /** Account/organization ID */
+  accountId?: string;
+
+  /** Client IP address */
+  ip?: string;
+
+  /** Client user agent */
+  userAgent?: string;
+
+  /** Environment (dev/stage/prod) */
+  environment?: string;
+
+  /** Trace ID for observability */
+  traceId?: string;
+
+  /** Correlation ID for request tracing */
+  correlationId?: string;
+
+  /** API version */
+  apiVersion?: string;
+
+  /** SDK version from client */
+  sdkVersion?: string;
+}
+
+/**
+ * Ingestion service configuration.
+ */
+export interface IngestionServiceConfig {
+  /** Maximum events per batch (default: 100) */
+  maxBatchSize?: number;
+
+  /** Maximum payload size in bytes (default: 1MB) */
+  maxPayloadSize?: number;
+
+  /** Whether to strip PII from properties (default: true in production) */
+  stripPII?: boolean;
+
+  /** Whether to strip HTML from properties (default: true) */
+  stripHTML?: boolean;
+
+  /** Timeout for processing each event in ms (default: 5000) */
+  eventTimeout?: number;
+
+  /** Concurrency for batch processing (default: 10) */
+  batchConcurrency?: number;
+}
+
+/**
+ * Ingestion metrics for observability.
+ */
+export interface IngestionMetrics {
+  /** Total events received */
+  totalReceived: number;
+
+  /** Events accepted */
+  accepted: number;
+
+  /** Events rejected */
+  rejected: number;
+
+  /** Events by type */
+  byType: Record<string, number>;
+
+  /** Processing time in ms */
+  processingTimeMs: number;
+}
+
+// =============================================================================
+// Default Configuration
+// =============================================================================
+
+const DEFAULT_CONFIG: Required<IngestionServiceConfig> = {
+  maxBatchSize: 100,
+  maxPayloadSize: 1024 * 1024, // 1MB
+  stripPII: true, // Default to safe; callers can override
+  stripHTML: true,
+  eventTimeout: 5000,
+  batchConcurrency: 10,
+};
+
+// =============================================================================
+// Utility Functions
+// =============================================================================
+
+/**
+ * Generate a UUID v4.
+ */
+function generateUUID(): string {
+  return crypto.randomUUID();
+}
+
+/**
+ * Get current ISO timestamp.
+ */
+function getCurrentTimestamp(): string {
+  return new Date().toISOString();
+}
+
+/**
+ * Check if request is a batch request.
+ */
+function isBatchRequest(request: IngestionRequest): request is BatchEventRequest {
+  return 'batch' in request && Array.isArray(request.batch);
+}
+
+/**
+ * Normalize an event payload with defaults and server metadata.
+ */
+function normalizeEvent(
+  event: EventPayload,
+  context: IngestionContext,
+  receivedAt: string,
+): EventPayload {
+  const normalized = { ...event };
+
+  // Generate message ID if not provided
+  if (!normalized.messageId) {
+    normalized.messageId = generateUUID();
+  }
+
+  // Set timestamp if not provided
+  if (!normalized.timestamp) {
+    normalized.timestamp = receivedAt;
+  }
+
+  // Preserve original timestamp
+  if (!normalized.originalTimestamp) {
+    normalized.originalTimestamp = normalized.timestamp;
+  }
+
+  // Merge context with server-provided context
+  const mergedContext: EmitterContext = {
+    ...normalized.context,
+    channel: normalized.context?.channel ?? 'api',
+    ip: context.ip ?? normalized.context?.ip,
+    userAgent: context.userAgent ?? normalized.context?.userAgent,
+    library: normalized.context?.library ?? {
+      name: context.source,
+      version: context.sdkVersion ?? 'unknown',
+    },
+  };
+
+  normalized.context = mergedContext;
+
+  // Set userId from auth context if not provided
+  if (!normalized.userId && context.userId) {
+    normalized.userId = context.userId;
+  }
+
+  return normalized;
+}
+
+/**
+ * Convert EventPayload to EmitterPayload format for AnalyticsManager.
+ * Explicitly maps fields to ensure type safety.
+ */
+function toEmitterPayload(event: EventPayload): EmitterPayload {
+  // Explicitly construct the EmitterPayload from EventPayload fields
+  // to maintain type safety and avoid unsafe double casts
+  const basePayload = {
+    userId: event.userId,
+    anonymousId: event.anonymousId,
+    timestamp: event.timestamp,
+    context: event.context,
+    messageId: event.messageId,
+  };
+
+  switch (event.type) {
+    case 'track':
+      return {
+        type: 'track',
+        event: event.event,
+        properties: event.properties,
+        ...basePayload,
+      } as EmitterPayload;
+    case 'identify':
+      return {
+        type: 'identify',
+        traits: event.traits,
+        ...basePayload,
+      } as EmitterPayload;
+    case 'page':
+      return {
+        type: 'page',
+        name: event.name,
+        category: event.category,
+        properties: event.properties,
+        ...basePayload,
+      } as EmitterPayload;
+    case 'screen':
+      return {
+        type: 'screen',
+        name: event.name,
+        category: event.category,
+        properties: event.properties,
+        ...basePayload,
+      } as EmitterPayload;
+    case 'group':
+      return {
+        type: 'group',
+        groupId: event.groupId,
+        traits: event.traits,
+        ...basePayload,
+      } as EmitterPayload;
+    case 'alias':
+      return {
+        type: 'alias',
+        previousId: event.previousId,
+        ...basePayload,
+      } as EmitterPayload;
+    default:
+      // This should never happen due to discriminated union validation
+      throw new Error(`Unknown event type: ${(event as { type: string }).type}`);
+  }
+}
+
+// =============================================================================
+// Ingestion Service Class
+// =============================================================================
+
+/**
+ * Event Ingestion Service.
+ *
+ * Handles validation, normalization, and forwarding of analytics events.
+ * Designed for high-volume ingestion with batching and rate limiting support.
+ *
+ * @example
+ * ```typescript
+ * const service = new IngestionService(analyticsManager);
+ *
+ * const result = await service.ingest(requestBody, {
+ *   source: 'web-app',
+ *   tenantId: 'tenant-123',
+ *   userId: 'user-456',
+ * });
+ *
+ * if (result.success) {
+ *   console.log(`Accepted ${result.accepted} events`);
+ * }
+ * ```
+ */
+export class IngestionService {
+  private readonly config: Required<IngestionServiceConfig>;
+
+  constructor(
+    private readonly analyticsManager: AnalyticsManager,
+    config: IngestionServiceConfig = {},
+  ) {
+    this.config = { ...DEFAULT_CONFIG, ...config };
+  }
+
+  /**
+   * Parse and validate the ingestion request payload.
+   *
+   * @param payload - Raw request payload (parsed JSON)
+   * @returns Parsed and validated request, or error response
+   */
+  parseRequest(
+    payload: unknown,
+  ): { success: true; data: IngestionRequest } | { success: false; error: IngestionErrorResponse } {
+    // Validate payload size (rough estimate)
+    try {
+      const payloadSize = JSON.stringify(payload).length;
+      if (payloadSize > this.config.maxPayloadSize) {
+        return {
+          success: false,
+          error: {
+            success: false,
+            code: 'PAYLOAD_TOO_LARGE',
+            error: `Payload size ${payloadSize} exceeds maximum ${this.config.maxPayloadSize} bytes`,
+          },
+        };
+      }
+    } catch {
+      return {
+        success: false,
+        error: {
+          success: false,
+          code: 'INVALID_JSON',
+          error: 'Failed to serialize payload for size check',
+        },
+      };
+    }
+
+    // Validate against schema
+    const result = IngestionRequestSchema.safeParse(payload);
+
+    if (!result.success) {
+      const fieldErrors = result.error.issues.map(err => ({
+        path: err.path.map(p => (typeof p === 'symbol' ? String(p) : p)),
+        message: err.message,
+      }));
+
+      return {
+        success: false,
+        error: {
+          success: false,
+          code: 'VALIDATION_ERROR',
+          error: 'Request validation failed',
+          fieldErrors,
+        },
+      };
+    }
+
+    // Check batch size
+    if (isBatchRequest(result.data) && result.data.batch.length > this.config.maxBatchSize) {
+      return {
+        success: false,
+        error: {
+          success: false,
+          code: 'BATCH_TOO_LARGE',
+          error: `Batch size ${result.data.batch.length} exceeds maximum ${this.config.maxBatchSize}`,
+        },
+      };
+    }
+
+    return { success: true, data: result.data };
+  }
+
+  /**
+   * Ingest events from a validated request.
+   *
+   * @param request - Validated ingestion request
+   * @param context - Ingestion context from the caller
+   * @returns Ingestion response with per-event results
+   */
+  async ingest(
+    request: IngestionRequest,
+    context: IngestionContext,
+  ): Promise<IngestionSuccessResponse> {
+    const startTime = process.hrtime.bigint();
+    const receivedAt = getCurrentTimestamp();
+
+    // Extract events from request (single or batch)
+    const events: EventPayload[] = isBatchRequest(request) ? request.batch : [request];
+
+    const results: EventResult[] = [];
+    const metrics: IngestionMetrics = {
+      totalReceived: events.length,
+      accepted: 0,
+      rejected: 0,
+      byType: {},
+      processingTimeMs: 0,
+    };
+
+    // Process events
+    const processedEvents: EmitterPayload[] = [];
+
+    for (const event of events) {
+      const eventId = generateUUID();
+
+      try {
+        // Validate event name for track events
+        if (event.type === 'track') {
+          const validation = validateEventName(event.event);
+          if (!validation.valid) {
+            results.push({
+              id: eventId,
+              messageId: event.messageId,
+              type: event.type,
+              status: 'rejected',
+              error: `Invalid event name: ${validation.reason}`,
+            });
+            metrics.rejected++;
+            continue;
+          }
+        }
+
+        // Normalize event
+        const normalized = normalizeEvent(event, context, receivedAt);
+
+        // Sanitize properties
+        if ('properties' in normalized && normalized.properties) {
+          const sanitized = sanitizeProperties(normalized.properties, {
+            stripPII: this.config.stripPII,
+            stripHTML: this.config.stripHTML,
+            allowDangerousKeys: false,
+          });
+          (normalized as { properties: Record<string, unknown> }).properties = sanitized.data;
+
+          if (sanitized.warnings.length > 0) {
+            logDebug('Event properties sanitized', {
+              eventId,
+              warnings: sanitized.warnings,
+            });
+          }
+        }
+
+        if ('traits' in normalized && normalized.traits) {
+          const sanitized = sanitizeProperties(normalized.traits, {
+            stripPII: this.config.stripPII,
+            stripHTML: this.config.stripHTML,
+            allowDangerousKeys: false,
+          });
+          (normalized as { traits: Record<string, unknown> }).traits = sanitized.data;
+        }
+
+        // Convert to emitter payload and queue for processing
+        const emitterPayload = toEmitterPayload(normalized);
+        processedEvents.push(emitterPayload);
+
+        results.push({
+          id: eventId,
+          messageId: normalized.messageId,
+          type: event.type,
+          status: 'accepted',
+        });
+
+        metrics.accepted++;
+        metrics.byType[event.type] = (metrics.byType[event.type] ?? 0) + 1;
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+
+        results.push({
+          id: eventId,
+          messageId: event.messageId,
+          type: event.type,
+          status: 'rejected',
+          error: errorMessage,
+        });
+
+        metrics.rejected++;
+
+        logWarn('Event processing failed', {
+          eventId,
+          type: event.type,
+          error: errorMessage,
+        });
+      }
+    }
+
+    // Forward accepted events to analytics manager (non-blocking)
+    // Note: This provides at-most-once delivery semantics. Events may be lost
+    // if the analytics manager fails. For guaranteed delivery, consider using
+    // a durable queue (Redis, SQS) for critical events.
+    if (processedEvents.length > 0) {
+      // Use emitBatch for efficient processing
+      void (async () => {
+        try {
+          await this.analyticsManager.emitBatch(processedEvents, {
+            timeout: this.config.eventTimeout,
+            concurrency: this.config.batchConcurrency,
+            failFast: false, // Process all events even if some fail
+          });
+        } catch (error) {
+          logError('Failed to emit events to analytics manager', {
+            error: error instanceof Error ? error.message : 'Unknown error',
+            eventCount: processedEvents.length,
+            traceId: context.traceId,
+          });
+        }
+      })();
+    }
+
+    // Calculate processing time
+    const endTime = process.hrtime.bigint();
+    metrics.processingTimeMs = Number(endTime - startTime) / 1_000_000;
+
+    // Log metrics
+    logInfo('Event ingestion completed', {
+      traceId: context.traceId,
+      source: context.source,
+      tenantId: context.tenantId,
+      totalReceived: metrics.totalReceived,
+      accepted: metrics.accepted,
+      rejected: metrics.rejected,
+      processingTimeMs: metrics.processingTimeMs.toFixed(2),
+      byType: metrics.byType,
+    });
+
+    return {
+      success: true,
+      accepted: metrics.accepted,
+      rejected: metrics.rejected,
+      results,
+      receivedAt,
+    };
+  }
+
+  /**
+   * Process a raw request body through parsing and ingestion.
+   *
+   * Convenience method that combines parseRequest and ingest.
+   *
+   * @param payload - Raw request payload
+   * @param context - Ingestion context
+   * @returns Ingestion response (success or error)
+   */
+  async processRequest(
+    payload: unknown,
+    context: IngestionContext,
+  ): Promise<IngestionSuccessResponse | IngestionErrorResponse> {
+    const parseResult = this.parseRequest(payload);
+
+    if (!parseResult.success) {
+      return parseResult.error;
+    }
+
+    return this.ingest(parseResult.data, context);
+  }
+}
+
+// =============================================================================
+// Factory Functions
+// =============================================================================
+
+/**
+ * Create an ingestion service instance.
+ *
+ * @param analyticsManager - Initialized AnalyticsManager instance
+ * @param config - Optional service configuration
+ * @returns Configured IngestionService instance
+ *
+ * @example
+ * ```typescript
+ * import { createServerAnalytics } from '@od-oneapp/analytics/server';
+ * import { createIngestionService } from '@od-oneapp/analytics/server';
+ *
+ * const analytics = await createServerAnalytics(config);
+ * const ingestionService = createIngestionService(analytics);
+ * ```
+ */
+export function createIngestionService(
+  analyticsManager: AnalyticsManager,
+  config?: IngestionServiceConfig,
+): IngestionService {
+  return new IngestionService(analyticsManager, config);
+}
+
+/**
+ * Validate a single event payload.
+ *
+ * Useful for pre-validation before queuing events.
+ *
+ * @param payload - Event payload to validate
+ * @returns Validation result with parsed data or errors
+ */
+export function validateEventPayload(
+  payload: unknown,
+): { success: true; data: EventPayload } | { success: false; errors: string[] } {
+  const result = EventPayloadSchema.safeParse(payload);
+
+  if (!result.success) {
+    return {
+      success: false,
+      errors: result.error.issues.map(
+        e => `${e.path.map(p => (typeof p === 'symbol' ? String(p) : p)).join('.')}: ${e.message}`,
+      ),
+    };
+  }
+
+  return { success: true, data: result.data };
+}
+
+/**
+ * Validate a batch of event payloads.
+ *
+ * @param payloads - Array of event payloads to validate
+ * @returns Validation result with parsed data or errors
+ */
+export function validateBatchPayload(
+  payloads: unknown[],
+): { success: true; data: EventPayload[] } | { success: false; errors: string[] } {
+  const result = BatchEventRequestSchema.safeParse({ batch: payloads });
+
+  if (!result.success) {
+    return {
+      success: false,
+      errors: result.error.issues.map(
+        e => `${e.path.map(p => (typeof p === 'symbol' ? String(p) : p)).join('.')}: ${e.message}`,
+      ),
+    };
+  }
+
+  return { success: true, data: result.data.batch };
+}