@fluentcommerce/fc-connect-sdk 0.1.54 → 0.1.56

package/docs/02-CORE-GUIDES/ingestion/examples/error-recovery.ts

@@ -1,728 +1,728 @@
-/**
- * Error Recovery and Resilience Example
- *
- * Comprehensive error handling strategies for production-ready ingestion:
- * - Exponential backoff retry logic
- * - Circuit breaker pattern
- * - Partial failure handling
- * - Dead letter queue (DLQ)
- * - Graceful degradation
- * - Error categorization and logging
- *
- * Prerequisites:
- * - npm install @fluentcommerce/fc-connect-sdk
- *
- * Usage:
- * npx tsx examples/error-recovery.ts
- */
-
-import {
-  createClient,
-  S3DataSource,
-  CSVParserService,
-  UniversalMapper,
-  StateService,
-  SimpleKVStore,
-  FluentClient,
-} from '@fluentcommerce/fc-connect-sdk';
-
-// ============================================================================
-// Configuration
-// ============================================================================
-
-const config = {
-  fluent: {
-    baseUrl: process.env.FLUENT_BASE_URL!,
-    clientId: process.env.FLUENT_CLIENT_ID!,
-    clientSecret: process.env.FLUENT_CLIENT_SECRET!,
-    retailerId: process.env.FLUENT_RETAILER_ID!,
-  },
-  s3: {
-    accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
-    region: process.env.AWS_REGION || 'us-east-1',
-  },
-  retry: {
-    maxRetries: 3,
-    initialDelayMs: 1000,
-    maxDelayMs: 30000,
-    backoffMultiplier: 2,
-  },
-  circuitBreaker: {
-    failureThreshold: 5,
-    resetTimeoutMs: 60000,
-  },
-  deadLetterQueue: {
-    bucket: 'dlq-bucket',
-    prefix: 'failed-ingestion/',
-  },
-};
-
-const mappingConfig = {
-  fields: {
-    ref: { source: 'sku', required: true },
-    productRef: { source: 'sku', required: true },
-    locationRef: { source: 'location', required: true },
-    qty: { source: 'quantity', resolver: 'sdk.parseInt', required: true },
-    status: { source: 'status', resolver: 'sdk.uppercase', defaultValue: 'AVAILABLE' },
-  },
-};
-
-// ============================================================================
-// Error Types & Classification
-// ============================================================================
-
-enum ErrorCategory {
-  TRANSIENT = 'TRANSIENT', // Retry possible (network, rate limit)
-  VALIDATION = 'VALIDATION', // Data quality issue (skip/dlq)
-  PERMANENT = 'PERMANENT', // Cannot be retried (auth, config)
-  UNKNOWN = 'UNKNOWN', // Unclear, safe to retry
-}
-
-interface ClassifiedError {
-  category: ErrorCategory;
-  message: string;
-  retryable: boolean;
-  originalError: Error;
-}
-
-/**
- * Classify error to determine retry strategy
- */
-function classifyError(error: any): ClassifiedError {
-  const message = error.message || String(error);
-
-  // Transient errors - safe to retry
-  if (
-    message.includes('ECONNRESET') ||
-    message.includes('ETIMEDOUT') ||
-    message.includes('ENOTFOUND') ||
-    message.includes('Network error') ||
-    message.includes('Rate limit') ||
-    message.includes('429')
-  ) {
-    return {
-      category: ErrorCategory.TRANSIENT,
-      message: 'Network or rate limit error',
-      retryable: true,
-      originalError: error,
-    };
-  }
-
-  // Validation errors - data quality issues
-  if (
-    message.includes('Validation failed') ||
-    message.includes('Invalid field') ||
-    message.includes('Required field missing') ||
-    message.includes('Schema mismatch')
-  ) {
-    return {
-      category: ErrorCategory.VALIDATION,
-      message: 'Data validation error',
-      retryable: false,
-      originalError: error,
-    };
-  }
-
-  // Permanent errors - configuration/auth issues
-  if (
-    message.includes('Unauthorized') ||
-    message.includes('Authentication failed') ||
-    message.includes('Access denied') ||
-    message.includes('Invalid credentials') ||
-    message.includes('401') ||
-    message.includes('403')
-  ) {
-    return {
-      category: ErrorCategory.PERMANENT,
-      message: 'Authentication or authorization error',
-      retryable: false,
-      originalError: error,
-    };
-  }
-
-  // Unknown errors - default to retryable
-  return {
-    category: ErrorCategory.UNKNOWN,
-    message: 'Unknown error',
-    retryable: true,
-    originalError: error,
-  };
-}
-
-// ============================================================================
-// Retry Logic with Exponential Backoff
-// ============================================================================
-
-interface RetryOptions {
-  maxRetries?: number;
-  initialDelayMs?: number;
-  maxDelayMs?: number;
-  backoffMultiplier?: number;
-  onRetry?: (attempt: number, delay: number, error: Error) => void;
-}
-
-/**
- * Retry operation with exponential backoff
- */
-async function retryWithBackoff<T>(
-  operation: () => Promise<T>,
-  options: RetryOptions = {}
-): Promise<T> {
-  const {
-    maxRetries = config.retry.maxRetries,
-    initialDelayMs = config.retry.initialDelayMs,
-    maxDelayMs = config.retry.maxDelayMs,
-    backoffMultiplier = config.retry.backoffMultiplier,
-    onRetry,
-  } = options;
-
-  let lastError: Error;
-  let delay = initialDelayMs;
-
-  for (let attempt = 1; attempt <= maxRetries; attempt++) {
-    try {
-      return await operation();
-    } catch (error: any) {
-      lastError = error;
-
-      // Classify error to determine if retry is appropriate
-      const classified = classifyError(error);
-
-      if (!classified.retryable) {
-        console.error(`❌ Non-retryable error (${classified.category}):`, error.message);
-        throw error;
-      }
-
-      if (attempt === maxRetries) {
-        console.error(`❌ Max retries (${maxRetries}) exceeded`);
-        throw error;
-      }
-
-      // Calculate next delay with exponential backoff
-      const nextDelay = Math.min(delay, maxDelayMs);
-
-      console.log(`⚠️  Attempt ${attempt}/${maxRetries} failed: ${error.message}`);
-      console.log(`   Retrying in ${nextDelay}ms...`);
-
-      if (onRetry) {
-        onRetry(attempt, nextDelay, error);
-      }
-
-      await sleep(nextDelay);
-      delay *= backoffMultiplier;
-    }
-  }
-
-  throw lastError!;
-}
-
-/**
- * Sleep utility
- */
-function sleep(ms: number): Promise<void> {
-  return new Promise(resolve => setTimeout(resolve, ms));
-}
-
-// ============================================================================
-// Circuit Breaker Pattern
-// ============================================================================
-
-enum CircuitState {
-  CLOSED = 'CLOSED', // Normal operation
-  OPEN = 'OPEN', // Failing, reject immediately
-  HALF_OPEN = 'HALF_OPEN', // Testing if service recovered
-}
-
-class CircuitBreaker {
-  private state: CircuitState = CircuitState.CLOSED;
-  private failureCount = 0;
-  private lastFailureTime: number = 0;
-  private successCount = 0;
-
-  constructor(
-    private failureThreshold: number = config.circuitBreaker.failureThreshold,
-    private resetTimeoutMs: number = config.circuitBreaker.resetTimeoutMs
-  ) {}
-
-  async execute<T>(operation: () => Promise<T>): Promise<T> {
-    // Check if circuit should transition from OPEN to HALF_OPEN
-    if (this.state === CircuitState.OPEN) {
-      const timeSinceLastFailure = Date.now() - this.lastFailureTime;
-
-      if (timeSinceLastFailure >= this.resetTimeoutMs) {
-        console.log('🔄 Circuit breaker transitioning to HALF_OPEN (testing)');
-        this.state = CircuitState.HALF_OPEN;
-        this.successCount = 0;
-      } else {
-        throw new Error(
-          `Circuit breaker is OPEN (wait ${Math.ceil((this.resetTimeoutMs - timeSinceLastFailure) / 1000)}s)`
-        );
-      }
-    }
-
-    try {
-      const result = await operation();
-
-      // Success - record it
-      this.onSuccess();
-
-      return result;
-    } catch (error) {
-      // Failure - record it
-      this.onFailure();
-
-      throw error;
-    }
-  }
-
-  private onSuccess(): void {
-    this.failureCount = 0;
-
-    if (this.state === CircuitState.HALF_OPEN) {
-      this.successCount++;
-
-      // After 3 successes in HALF_OPEN, transition to CLOSED
-      if (this.successCount >= 3) {
-        console.log('✅ Circuit breaker closing (service recovered)');
-        this.state = CircuitState.CLOSED;
-        this.successCount = 0;
-      }
-    }
-  }
-
-  private onFailure(): void {
-    this.failureCount++;
-    this.lastFailureTime = Date.now();
-
-    if (this.failureCount >= this.failureThreshold) {
-      console.log(`🚫 Circuit breaker opening (${this.failureCount} failures)`);
-      this.state = CircuitState.OPEN;
-    }
-  }
-
-  getState(): CircuitState {
-    return this.state;
-  }
-}
-
-// ============================================================================
-// Dead Letter Queue (DLQ)
-// ============================================================================
-
-class DeadLetterQueue {
-  constructor(
-    private s3: S3DataSource,
-    private bucket: string = config.deadLetterQueue.bucket,
-    private prefix: string = config.deadLetterQueue.prefix
-  ) {}
-
-  /**
-   * Send failed record to DLQ for manual review
-   */
-  async sendToDLQ(
-    record: any,
-    error: ClassifiedError,
-    metadata: {
-      originalFile?: string;
-      attempt?: number;
-      timestamp?: string;
-    } = {}
-  ): Promise<void> {
-    const dlqEntry = {
-      record,
-      error: {
-        category: error.category,
-        message: error.message,
-        stack: error.originalError.stack,
-      },
-      metadata: {
-        ...metadata,
-        timestamp: metadata.timestamp || new Date().toISOString(),
-        dlqTimestamp: new Date().toISOString(),
-      },
-    };
-
-    const key = `${this.prefix}${Date.now()}-${Math.random().toString(36).substr(2, 9)}.json`;
-
-    try {
-      await this.s3.putObject(this.bucket, key, Buffer.from(JSON.stringify(dlqEntry, null, 2)));
-
-      console.log(`📮 Sent to DLQ: s3://${this.bucket}/${key}`);
-    } catch (dlqError) {
-      console.error('❌ Failed to send to DLQ:', dlqError);
-      // Log to local file as fallback
-      console.error('DLQ Entry:', JSON.stringify(dlqEntry));
-    }
-  }
-
-  /**
-   * Retrieve failed records from DLQ for reprocessing
-   */
-  async retrieveFromDLQ(limit: number = 100): Promise<any[]> {
-    const files = await this.s3.listFiles({ prefix: this.prefix });
-
-    const entries = [];
-
-    for (const file of files.slice(0, limit)) {
-      try {
-        const data = await this.s3.downloadFile(file.path);
-        const entry = JSON.parse(data.toString());
-        entries.push(entry);
-      } catch (error) {
-        console.error(`Failed to read DLQ entry ${file.path}:`, error);
-      }
-    }
-
-    return entries;
-  }
-}
-
-// ============================================================================
-// Resilient Ingestion Workflow
-// ============================================================================
-
-interface IngestionResult {
-  totalFiles: number;
-  processedFiles: number;
-  failedFiles: number;
-  totalRecords: number;
-  successfulRecords: number;
-  failedRecords: number;
-  sentToDLQ: number;
-  errors: Array<{ file: string; error: string }>;
-}
-
-async function resilientIngestion(): Promise<IngestionResult> {
-  console.log('🚀 Starting resilient ingestion workflow\n');
-
-  // Initialize services
-  const client = await createClient({ config: config.fluent });
-  const s3 = new S3DataSource(
-    {
-      type: 'S3_CSV',
-      connectionId: 's3-error-recovery',
-      name: 'S3 Error Recovery',
-      s3Config: {
-        ...config.s3,
-        bucket: 'inventory-bucket',
-      },
-    },
-    console
-  );
-  const logger = console; // Use console as logger in examples
-  const parser = new CSVParserService(logger);
-  const mapper = new UniversalMapper(mappingConfig);
-  const state = new StateService(logger);
-  const circuitBreaker = new CircuitBreaker();
-  const dlq = new DeadLetterQueue(s3);
-
-  const result: IngestionResult = {
-    totalFiles: 0,
-    processedFiles: 0,
-    failedFiles: 0,
-    totalRecords: 0,
-    successfulRecords: 0,
-    failedRecords: 0,
-    sentToDLQ: 0,
-    errors: [],
-  };
-
-  // List files
-  const files = await retryWithBackoff(() => s3.listFiles({ prefix: 'data/' }), {
-    maxRetries: 5,
-  });
-
-  const csvFiles = files.filter(f => f.path.endsWith('.csv'));
-  result.totalFiles = csvFiles.length;
-
-  console.log(`📁 Processing ${csvFiles.length} files\n`);
-
-  // Process each file with error recovery
-  for (const file of csvFiles) {
-    console.log(`📄 File: ${file.path}`);
-
-    try {
-      // Check if already processed
-      if (await state.isFileProcessed(file.path)) {
-        console.log('  ✓ Already processed, skipping\n');
-        continue;
-      }
-
-      // Acquire lock with retry
-      const lock = await retryWithBackoff(() => state.acquireLock(file.path, 60000), {
-        maxRetries: 3,
-      });
-
-      if (!lock.acquired) {
-        console.log('  ⏸ Locked by another process, skipping\n');
-        continue;
-      }
-
-      try {
-        // Read file with retry and circuit breaker
-        const data = await circuitBreaker.execute(() =>
-          retryWithBackoff(() => s3.downloadFile(file.path), {
-            maxRetries: 3,
-            onRetry: (attempt, delay) => {
-              console.log(`  ⏳ Retrying S3 read (attempt ${attempt})...`);
-            },
-          })
-        );
-
-        console.log(`  📖 Read ${data.length} bytes`);
-
-        // Parse CSV
-        const records = await parser.parse(data);
-        result.totalRecords += records.length;
-
-        console.log(`  🔍 Parsed ${records.length} records`);
-
-        // Process records with partial failure handling
-        const successfulRecords = [];
-        const failedRecordsForDLQ = [];
-
-        for (const [index, record] of records.entries()) {
-          try {
-            // Map individual record
-            const mapResult = await mapper.map([record]);
-
-            if (mapResult.success && mapResult.data.length > 0) {
-              successfulRecords.push(mapResult.data[0]);
-            } else {
-              // Validation error - send to DLQ
-              const error: ClassifiedError = {
-                category: ErrorCategory.VALIDATION,
-                message: mapResult.errors.join('; '),
-                retryable: false,
-                originalError: new Error(mapResult.errors.join('; ')),
-              };
-
-              failedRecordsForDLQ.push({ record, error });
-              result.failedRecords++;
-            }
-          } catch (error: any) {
-            const classified = classifyError(error);
-
-            console.error(`  ⚠️  Record ${index + 1} failed:`, error.message);
-
-            failedRecordsForDLQ.push({ record, error: classified });
-            result.failedRecords++;
-          }
-        }
-
-        // Send successful records to Fluent
-        if (successfulRecords.length > 0) {
-          const job = await circuitBreaker.execute(() =>
-            retryWithBackoff(
-              () =>
-                client.createJob({
-                  name: `Import - ${file.path}`,
-                  retailerId: config.fluent.retailerId,
-                }),
-              { maxRetries: 3 }
-            )
-          );
-
-          await circuitBreaker.execute(() =>
-            retryWithBackoff(
-              () =>
-                client.sendBatch(job.id, {
-                  action: 'UPSERT',
-                  entityType: 'INVENTORY',
-                  source: 'ERROR_RECOVERY_RESILIENT',
-                  event: 'INVENTORY_UPDATE',
-                  entities: successfulRecords,
-                }),
-              { maxRetries: 3 }
-            )
-          );
-
-          result.successfulRecords += successfulRecords.length;
-
-          console.log(`  ✅ Ingested ${successfulRecords.length} records (job: ${job.id})`);
-        }
-
-        // Send failed records to DLQ
-        if (failedRecordsForDLQ.length > 0) {
-          console.log(`  📮 Sending ${failedRecordsForDLQ.length} failed records to DLQ`);
-
-          for (const { record, error } of failedRecordsForDLQ) {
-            await dlq.sendToDLQ(record, error, {
-              originalFile: file.path,
-              timestamp: new Date().toISOString(),
-            });
-            result.sentToDLQ++;
-          }
-        }
-
-        // Mark file as processed
-        await state.markFileProcessed(file.path, {
-          processedAt: new Date().toISOString(),
-          totalRecords: records.length,
-          successfulRecords: successfulRecords.length,
-          failedRecords: failedRecordsForDLQ.length,
-        });
-
-        result.processedFiles++;
-
-        console.log(`  ✓ File processing complete\n`);
-      } finally {
-        await state.releaseLock(file.path);
-      }
-    } catch (error: any) {
-      const classified = classifyError(error);
-
-      console.error(`  ❌ File failed (${classified.category}): ${error.message}\n`);
-
-      result.failedFiles++;
-      result.errors.push({
-        file: file.path,
-        error: `${classified.category}: ${error.message}`,
-      });
-
-      // For permanent errors, don't retry file
-      if (classified.category === ErrorCategory.PERMANENT) {
-        console.log('  🚫 Permanent error - marking file as failed');
-
-        await state.markFileProcessed(file.path, {
-          processedAt: new Date().toISOString(),
-          failed: true,
-          error: error.message,
-        });
-      }
-    }
-  }
-
-  return result;
-}
-
-// ============================================================================
-// DLQ Reprocessing
-// ============================================================================
-
-/**
- * Reprocess failed records from DLQ after fixing issues
- */
-async function reprocessDLQ(): Promise<void> {
-  console.log('🔄 Reprocessing records from DLQ\n');
-
-  const s3 = new S3DataSource(
-    {
-      type: 'S3_CSV',
-      connectionId: 's3-dlq-reprocess',
-      name: 'S3 DLQ Reprocess',
-      s3Config: {
-        ...config.s3,
-        bucket: 'dlq-bucket',
-      },
-    },
-    console
-  );
-  const client = await createClient({ config: config.fluent });
-  const mapper = new UniversalMapper(mappingConfig);
-  const dlq = new DeadLetterQueue(s3);
-
-  // Retrieve failed records
-  const dlqEntries = await dlq.retrieveFromDLQ(100);
-
-  console.log(`📮 Found ${dlqEntries.length} DLQ entries\n`);
-
-  const successfulRetries = [];
-
-  for (const entry of dlqEntries) {
-    try {
-      // Remap record (fixes may have been applied)
-      const result = await mapper.map([entry.record]);
-
-      if (result.success && result.data.length > 0) {
-        successfulRetries.push(result.data[0]);
-        console.log('  ✓ Successfully remapped record');
-      } else {
-        console.log('  ⚠️  Record still invalid:', result.errors.join('; '));
-      }
-    } catch (error: any) {
-      console.error('  ❌ Reprocessing failed:', error.message);
-    }
-  }
-
-  // Send successful retries to Fluent
-  if (successfulRetries.length > 0) {
-    const job = await client.createJob({
-      name: 'DLQ Reprocessing',
-      retailerId: config.fluent.retailerId,
-    });
-
-    await client.sendBatch(job.id, {
-      action: 'UPSERT',
-      entityType: 'INVENTORY',
-      source: 'DLQ_REPROCESSING',
-      event: 'INVENTORY_RECOVERY',
-      entities: successfulRetries,
-    });
-
-    console.log(`\n✅ Reprocessed ${successfulRetries.length} records from DLQ`);
-  }
-}
-
-// ============================================================================
-// Main Entry Point
-// ============================================================================
-
-async function main() {
-  const mode = process.argv[2] || 'ingest';
-
-  try {
-    switch (mode) {
-      case 'ingest':
-        const result = await resilientIngestion();
-
-        console.log('\n' + '='.repeat(60));
-        console.log('📊 Ingestion Summary');
-        console.log('='.repeat(60));
-        console.log(`Total files:          ${result.totalFiles}`);
-        console.log(`Processed files:      ${result.processedFiles}`);
-        console.log(`Failed files:         ${result.failedFiles}`);
-        console.log(`Total records:        ${result.totalRecords}`);
-        console.log(`Successful records:   ${result.successfulRecords}`);
-        console.log(`Failed records:       ${result.failedRecords}`);
-        console.log(`Sent to DLQ:          ${result.sentToDLQ}`);
-
-        if (result.errors.length > 0) {
-          console.log('\n❌ Errors:');
-          result.errors.forEach(e => console.log(`  - ${e.file}: ${e.error}`));
-        }
-
-        console.log('='.repeat(60));
-        break;
-
-      case 'reprocess-dlq':
-        await reprocessDLQ();
-        break;
-
-      default:
-        console.log('Usage:');
-        console.log('  npx tsx examples/error-recovery.ts ingest          # Run ingestion');
-        console.log('  npx tsx examples/error-recovery.ts reprocess-dlq   # Reprocess DLQ');
-    }
-  } catch (error: any) {
-    console.error('\n💥 Fatal error:', error.message);
-    process.exit(1);
-  }
-}
-
-// Run if executed directly
-if (require.main === module) {
-  main();
-}
-
-// Export for testing
-export {
-  retryWithBackoff,
-  classifyError,
-  CircuitBreaker,
-  DeadLetterQueue,
-  resilientIngestion,
-  ErrorCategory,
-  ClassifiedError,
-};
+/**
+ * Error Recovery and Resilience Example
+ *
+ * Comprehensive error handling strategies for production-ready ingestion:
+ * - Exponential backoff retry logic
+ * - Circuit breaker pattern
+ * - Partial failure handling
+ * - Dead letter queue (DLQ)
+ * - Graceful degradation
+ * - Error categorization and logging
+ *
+ * Prerequisites:
+ * - npm install @fluentcommerce/fc-connect-sdk
+ *
+ * Usage:
+ * npx tsx examples/error-recovery.ts
+ */
+
+import {
+  createClient,
+  S3DataSource,
+  CSVParserService,
+  UniversalMapper,
+  StateService,
+  SimpleKVStore,
+  FluentClient,
+} from '@fluentcommerce/fc-connect-sdk';
+
+// ============================================================================
+// Configuration
+// ============================================================================
+
+const config = {
+  fluent: {
+    baseUrl: process.env.FLUENT_BASE_URL!,
+    clientId: process.env.FLUENT_CLIENT_ID!,
+    clientSecret: process.env.FLUENT_CLIENT_SECRET!,
+    retailerId: process.env.FLUENT_RETAILER_ID!,
+  },
+  s3: {
+    accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
+    region: process.env.AWS_REGION || 'us-east-1',
+  },
+  retry: {
+    maxRetries: 3,
+    initialDelayMs: 1000,
+    maxDelayMs: 30000,
+    backoffMultiplier: 2,
+  },
+  circuitBreaker: {
+    failureThreshold: 5,
+    resetTimeoutMs: 60000,
+  },
+  deadLetterQueue: {
+    bucket: 'dlq-bucket',
+    prefix: 'failed-ingestion/',
+  },
+};
+
+const mappingConfig = {
+  fields: {
+    ref: { source: 'sku', required: true },
+    productRef: { source: 'sku', required: true },
+    locationRef: { source: 'location', required: true },
+    qty: { source: 'quantity', resolver: 'sdk.parseInt', required: true },
+    status: { source: 'status', resolver: 'sdk.uppercase', defaultValue: 'AVAILABLE' },
+  },
+};
+
+// ============================================================================
+// Error Types & Classification
+// ============================================================================
+
+enum ErrorCategory {
+  TRANSIENT = 'TRANSIENT', // Retry possible (network, rate limit)
+  VALIDATION = 'VALIDATION', // Data quality issue (skip/dlq)
+  PERMANENT = 'PERMANENT', // Cannot be retried (auth, config)
+  UNKNOWN = 'UNKNOWN', // Unclear, safe to retry
+}
+
+interface ClassifiedError {
+  category: ErrorCategory;
+  message: string;
+  retryable: boolean;
+  originalError: Error;
+}
+
+/**
+ * Classify error to determine retry strategy
+ */
+function classifyError(error: any): ClassifiedError {
+  const message = error.message || String(error);
+
+  // Transient errors - safe to retry
+  if (
+    message.includes('ECONNRESET') ||
+    message.includes('ETIMEDOUT') ||
+    message.includes('ENOTFOUND') ||
+    message.includes('Network error') ||
+    message.includes('Rate limit') ||
+    message.includes('429')
+  ) {
+    return {
+      category: ErrorCategory.TRANSIENT,
+      message: 'Network or rate limit error',
+      retryable: true,
+      originalError: error,
+    };
+  }
+
+  // Validation errors - data quality issues
+  if (
+    message.includes('Validation failed') ||
+    message.includes('Invalid field') ||
+    message.includes('Required field missing') ||
+    message.includes('Schema mismatch')
+  ) {
+    return {
+      category: ErrorCategory.VALIDATION,
+      message: 'Data validation error',
+      retryable: false,
+      originalError: error,
+    };
+  }
+
+  // Permanent errors - configuration/auth issues
+  if (
+    message.includes('Unauthorized') ||
+    message.includes('Authentication failed') ||
+    message.includes('Access denied') ||
+    message.includes('Invalid credentials') ||
+    message.includes('401') ||
+    message.includes('403')
+  ) {
+    return {
+      category: ErrorCategory.PERMANENT,
+      message: 'Authentication or authorization error',
+      retryable: false,
+      originalError: error,
+    };
+  }
+
+  // Unknown errors - default to retryable
+  return {
+    category: ErrorCategory.UNKNOWN,
+    message: 'Unknown error',
+    retryable: true,
+    originalError: error,
+  };
+}
+
+// ============================================================================
+// Retry Logic with Exponential Backoff
+// ============================================================================
+
+interface RetryOptions {
+  maxRetries?: number;
+  initialDelayMs?: number;
+  maxDelayMs?: number;
+  backoffMultiplier?: number;
+  onRetry?: (attempt: number, delay: number, error: Error) => void;
+}
+
+/**
+ * Retry operation with exponential backoff
+ */
+async function retryWithBackoff<T>(
+  operation: () => Promise<T>,
+  options: RetryOptions = {}
+): Promise<T> {
+  const {
+    maxRetries = config.retry.maxRetries,
+    initialDelayMs = config.retry.initialDelayMs,
+    maxDelayMs = config.retry.maxDelayMs,
+    backoffMultiplier = config.retry.backoffMultiplier,
+    onRetry,
+  } = options;
+
+  let lastError: Error;
+  let delay = initialDelayMs;
+
+  for (let attempt = 1; attempt <= maxRetries; attempt++) {
+    try {
+      return await operation();
+    } catch (error: any) {
+      lastError = error;
+
+      // Classify error to determine if retry is appropriate
+      const classified = classifyError(error);
+
+      if (!classified.retryable) {
+        console.error(`❌ Non-retryable error (${classified.category}):`, error.message);
+        throw error;
+      }
+
+      if (attempt === maxRetries) {
+        console.error(`❌ Max retries (${maxRetries}) exceeded`);
+        throw error;
+      }
+
+      // Calculate next delay with exponential backoff
+      const nextDelay = Math.min(delay, maxDelayMs);
+
+      console.log(`⚠️  Attempt ${attempt}/${maxRetries} failed: ${error.message}`);
+      console.log(`   Retrying in ${nextDelay}ms...`);
+
+      if (onRetry) {
+        onRetry(attempt, nextDelay, error);
+      }
+
+      await sleep(nextDelay);
+      delay *= backoffMultiplier;
+    }
+  }
+
+  throw lastError!;
+}
+
+/**
+ * Sleep utility
+ */
+function sleep(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+// ============================================================================
+// Circuit Breaker Pattern
+// ============================================================================
+
+enum CircuitState {
+  CLOSED = 'CLOSED', // Normal operation
+  OPEN = 'OPEN', // Failing, reject immediately
+  HALF_OPEN = 'HALF_OPEN', // Testing if service recovered
+}
+
+class CircuitBreaker {
+  private state: CircuitState = CircuitState.CLOSED;
+  private failureCount = 0;
+  private lastFailureTime: number = 0;
+  private successCount = 0;
+
+  constructor(
+    private failureThreshold: number = config.circuitBreaker.failureThreshold,
+    private resetTimeoutMs: number = config.circuitBreaker.resetTimeoutMs
+  ) {}
+
+  async execute<T>(operation: () => Promise<T>): Promise<T> {
+    // Check if circuit should transition from OPEN to HALF_OPEN
+    if (this.state === CircuitState.OPEN) {
+      const timeSinceLastFailure = Date.now() - this.lastFailureTime;
+
+      if (timeSinceLastFailure >= this.resetTimeoutMs) {
+        console.log('🔄 Circuit breaker transitioning to HALF_OPEN (testing)');
+        this.state = CircuitState.HALF_OPEN;
+        this.successCount = 0;
+      } else {
+        throw new Error(
+          `Circuit breaker is OPEN (wait ${Math.ceil((this.resetTimeoutMs - timeSinceLastFailure) / 1000)}s)`
+        );
+      }
+    }
+
+    try {
+      const result = await operation();
+
+      // Success - record it
+      this.onSuccess();
+
+      return result;
+    } catch (error) {
+      // Failure - record it
+      this.onFailure();
+
+      throw error;
+    }
+  }
+
+  private onSuccess(): void {
+    this.failureCount = 0;
+
+    if (this.state === CircuitState.HALF_OPEN) {
+      this.successCount++;
+
+      // After 3 successes in HALF_OPEN, transition to CLOSED
+      if (this.successCount >= 3) {
+        console.log('✅ Circuit breaker closing (service recovered)');
+        this.state = CircuitState.CLOSED;
+        this.successCount = 0;
+      }
+    }
+  }
+
+  private onFailure(): void {
+    this.failureCount++;
+    this.lastFailureTime = Date.now();
+
+    if (this.failureCount >= this.failureThreshold) {
+      console.log(`🚫 Circuit breaker opening (${this.failureCount} failures)`);
+      this.state = CircuitState.OPEN;
+    }
+  }
+
+  getState(): CircuitState {
+    return this.state;
+  }
+}
+
+// ============================================================================
+// Dead Letter Queue (DLQ)
+// ============================================================================
+
+class DeadLetterQueue {
+  constructor(
+    private s3: S3DataSource,
+    private bucket: string = config.deadLetterQueue.bucket,
+    private prefix: string = config.deadLetterQueue.prefix
+  ) {}
+
+  /**
+   * Send failed record to DLQ for manual review
+   */
+  async sendToDLQ(
+    record: any,
+    error: ClassifiedError,
+    metadata: {
+      originalFile?: string;
+      attempt?: number;
+      timestamp?: string;
+    } = {}
+  ): Promise<void> {
+    const dlqEntry = {
+      record,
+      error: {
+        category: error.category,
+        message: error.message,
+        stack: error.originalError.stack,
+      },
+      metadata: {
+        ...metadata,
+        timestamp: metadata.timestamp || new Date().toISOString(),
+        dlqTimestamp: new Date().toISOString(),
+      },
+    };
+
+    const key = `${this.prefix}${Date.now()}-${Math.random().toString(36).substr(2, 9)}.json`;
+
+    try {
+      await this.s3.putObject(this.bucket, key, Buffer.from(JSON.stringify(dlqEntry, null, 2)));
+
+      console.log(`📮 Sent to DLQ: s3://${this.bucket}/${key}`);
+    } catch (dlqError) {
+      console.error('❌ Failed to send to DLQ:', dlqError);
+      // Log to local file as fallback
+      console.error('DLQ Entry:', JSON.stringify(dlqEntry));
+    }
+  }
+
+  /**
+   * Retrieve failed records from DLQ for reprocessing
+   */
+  async retrieveFromDLQ(limit: number = 100): Promise<any[]> {
+    const files = await this.s3.listFiles({ prefix: this.prefix });
+
+    const entries = [];
+
+    for (const file of files.slice(0, limit)) {
+      try {
+        const data = await this.s3.downloadFile(file.path);
+        const entry = JSON.parse(data.toString());
+        entries.push(entry);
+      } catch (error) {
+        console.error(`Failed to read DLQ entry ${file.path}:`, error);
+      }
+    }
+
+    return entries;
+  }
+}
+
+// ============================================================================
+// Resilient Ingestion Workflow
+// ============================================================================
+
+interface IngestionResult {
+  totalFiles: number;
+  processedFiles: number;
+  failedFiles: number;
+  totalRecords: number;
+  successfulRecords: number;
+  failedRecords: number;
+  sentToDLQ: number;
+  errors: Array<{ file: string; error: string }>;
+}
+
+async function resilientIngestion(): Promise<IngestionResult> {
+  console.log('🚀 Starting resilient ingestion workflow\n');
+
+  // Initialize services
+  const client = await createClient({ config: config.fluent });
+  const s3 = new S3DataSource(
+    {
+      type: 'S3_CSV',
+      connectionId: 's3-error-recovery',
+      name: 'S3 Error Recovery',
+      s3Config: {
+        ...config.s3,
+        bucket: 'inventory-bucket',
+      },
+    },
+    console
+  );
+  const logger = console; // Use console as logger in examples
+  const parser = new CSVParserService(logger);
+  const mapper = new UniversalMapper(mappingConfig);
+  const state = new StateService(logger);
+  const circuitBreaker = new CircuitBreaker();
+  const dlq = new DeadLetterQueue(s3);
+
+  const result: IngestionResult = {
+    totalFiles: 0,
+    processedFiles: 0,
+    failedFiles: 0,
+    totalRecords: 0,
+    successfulRecords: 0,
+    failedRecords: 0,
+    sentToDLQ: 0,
+    errors: [],
+  };
+
+  // List files
+  const files = await retryWithBackoff(() => s3.listFiles({ prefix: 'data/' }), {
+    maxRetries: 5,
+  });
+
+  const csvFiles = files.filter(f => f.path.endsWith('.csv'));
+  result.totalFiles = csvFiles.length;
+
+  console.log(`📁 Processing ${csvFiles.length} files\n`);
+
+  // Process each file with error recovery
+  for (const file of csvFiles) {
+    console.log(`📄 File: ${file.path}`);
+
+    try {
+      // Check if already processed
+      if (await state.isFileProcessed(file.path)) {
+        console.log('  ✓ Already processed, skipping\n');
+        continue;
+      }
+
+      // Acquire lock with retry
+      const lock = await retryWithBackoff(() => state.acquireLock(file.path, 60000), {
+        maxRetries: 3,
+      });
+
+      if (!lock.acquired) {
+        console.log('  ⏸ Locked by another process, skipping\n');
+        continue;
+      }
+
+      try {
+        // Read file with retry and circuit breaker
+        const data = await circuitBreaker.execute(() =>
+          retryWithBackoff(() => s3.downloadFile(file.path), {
+            maxRetries: 3,
+            onRetry: (attempt, delay) => {
+              console.log(`  ⏳ Retrying S3 read (attempt ${attempt})...`);
+            },
+          })
+        );
+
+        console.log(`  📖 Read ${data.length} bytes`);
+
+        // Parse CSV
+        const records = await parser.parse(data);
+        result.totalRecords += records.length;
+
+        console.log(`  🔍 Parsed ${records.length} records`);
+
+        // Process records with partial failure handling
+        const successfulRecords = [];
+        const failedRecordsForDLQ = [];
+
+        for (const [index, record] of records.entries()) {
+          try {
+            // Map individual record
+            const mapResult = await mapper.map([record]);
+
+            if (mapResult.success && mapResult.data.length > 0) {
+              successfulRecords.push(mapResult.data[0]);
+            } else {
+              // Validation error - send to DLQ
+              const error: ClassifiedError = {
+                category: ErrorCategory.VALIDATION,
+                message: mapResult.errors.join('; '),
+                retryable: false,
+                originalError: new Error(mapResult.errors.join('; ')),
+              };
+
+              failedRecordsForDLQ.push({ record, error });
+              result.failedRecords++;
+            }
+          } catch (error: any) {
+            const classified = classifyError(error);
+
+            console.error(`  ⚠️  Record ${index + 1} failed:`, error.message);
+
+            failedRecordsForDLQ.push({ record, error: classified });
+            result.failedRecords++;
+          }
+        }
+
+        // Send successful records to Fluent
+        if (successfulRecords.length > 0) {
+          const job = await circuitBreaker.execute(() =>
+            retryWithBackoff(
+              () =>
+                client.createJob({
+                  name: `Import - ${file.path}`,
+                  retailerId: config.fluent.retailerId,
+                }),
+              { maxRetries: 3 }
+            )
+          );
+
+          await circuitBreaker.execute(() =>
+            retryWithBackoff(
+              () =>
+                client.sendBatch(job.id, {
+                  action: 'UPSERT',
+                  entityType: 'INVENTORY',
+                  source: 'ERROR_RECOVERY_RESILIENT',
+                  event: 'INVENTORY_UPDATE',
+                  entities: successfulRecords,
+                }),
+              { maxRetries: 3 }
+            )
+          );
+
+          result.successfulRecords += successfulRecords.length;
+
+          console.log(`  ✅ Ingested ${successfulRecords.length} records (job: ${job.id})`);
+        }
+
+        // Send failed records to DLQ
+        if (failedRecordsForDLQ.length > 0) {
+          console.log(`  📮 Sending ${failedRecordsForDLQ.length} failed records to DLQ`);
+
+          for (const { record, error } of failedRecordsForDLQ) {
+            await dlq.sendToDLQ(record, error, {
+              originalFile: file.path,
+              timestamp: new Date().toISOString(),
+            });
+            result.sentToDLQ++;
+          }
+        }
+
+        // Mark file as processed
+        await state.markFileProcessed(file.path, {
+          processedAt: new Date().toISOString(),
+          totalRecords: records.length,
+          successfulRecords: successfulRecords.length,
+          failedRecords: failedRecordsForDLQ.length,
+        });
+
+        result.processedFiles++;
+
+        console.log(`  ✓ File processing complete\n`);
+      } finally {
+        await state.releaseLock(file.path);
+      }
+    } catch (error: any) {
+      const classified = classifyError(error);
+
+      console.error(`  ❌ File failed (${classified.category}): ${error.message}\n`);
+
+      result.failedFiles++;
+      result.errors.push({
+        file: file.path,
+        error: `${classified.category}: ${error.message}`,
+      });
+
+      // For permanent errors, don't retry file
+      if (classified.category === ErrorCategory.PERMANENT) {
+        console.log('  🚫 Permanent error - marking file as failed');
+
+        await state.markFileProcessed(file.path, {
+          processedAt: new Date().toISOString(),
+          failed: true,
+          error: error.message,
+        });
+      }
+    }
+  }
+
+  return result;
+}
+
+// ============================================================================
+// DLQ Reprocessing
+// ============================================================================
+
+/**
+ * Reprocess failed records from DLQ after fixing issues
+ */
+async function reprocessDLQ(): Promise<void> {
+  console.log('🔄 Reprocessing records from DLQ\n');
+
+  const s3 = new S3DataSource(
+    {
+      type: 'S3_CSV',
+      connectionId: 's3-dlq-reprocess',
+      name: 'S3 DLQ Reprocess',
+      s3Config: {
+        ...config.s3,
+        bucket: 'dlq-bucket',
+      },
+    },
+    console
+  );
+  const client = await createClient({ config: config.fluent });
+  const mapper = new UniversalMapper(mappingConfig);
+  const dlq = new DeadLetterQueue(s3);
+
+  // Retrieve failed records
+  const dlqEntries = await dlq.retrieveFromDLQ(100);
+
+  console.log(`📮 Found ${dlqEntries.length} DLQ entries\n`);
+
+  const successfulRetries = [];
+
+  for (const entry of dlqEntries) {
+    try {
+      // Remap record (fixes may have been applied)
+      const result = await mapper.map([entry.record]);
+
+      if (result.success && result.data.length > 0) {
+        successfulRetries.push(result.data[0]);
+        console.log('  ✓ Successfully remapped record');
+      } else {
+        console.log('  ⚠️  Record still invalid:', result.errors.join('; '));
+      }
+    } catch (error: any) {
+      console.error('  ❌ Reprocessing failed:', error.message);
+    }
+  }
+
+  // Send successful retries to Fluent
+  if (successfulRetries.length > 0) {
+    const job = await client.createJob({
+      name: 'DLQ Reprocessing',
+      retailerId: config.fluent.retailerId,
+    });
+
+    await client.sendBatch(job.id, {
+      action: 'UPSERT',
+      entityType: 'INVENTORY',
+      source: 'DLQ_REPROCESSING',
+      event: 'INVENTORY_RECOVERY',
+      entities: successfulRetries,
+    });
+
+    console.log(`\n✅ Reprocessed ${successfulRetries.length} records from DLQ`);
+  }
+}
+
+// ============================================================================
+// Main Entry Point
+// ============================================================================
+
+async function main() {
+  const mode = process.argv[2] || 'ingest';
+
+  try {
+    switch (mode) {
+      case 'ingest':
+        const result = await resilientIngestion();
+
+        console.log('\n' + '='.repeat(60));
+        console.log('📊 Ingestion Summary');
+        console.log('='.repeat(60));
+        console.log(`Total files:          ${result.totalFiles}`);
+        console.log(`Processed files:      ${result.processedFiles}`);
+        console.log(`Failed files:         ${result.failedFiles}`);
+        console.log(`Total records:        ${result.totalRecords}`);
+        console.log(`Successful records:   ${result.successfulRecords}`);
+        console.log(`Failed records:       ${result.failedRecords}`);
+        console.log(`Sent to DLQ:          ${result.sentToDLQ}`);
+
+        if (result.errors.length > 0) {
+          console.log('\n❌ Errors:');
+          result.errors.forEach(e => console.log(`  - ${e.file}: ${e.error}`));
+        }
+
+        console.log('='.repeat(60));
+        break;
+
+      case 'reprocess-dlq':
+        await reprocessDLQ();
+        break;
+
+      default:
+        console.log('Usage:');
+        console.log('  npx tsx examples/error-recovery.ts ingest          # Run ingestion');
+        console.log('  npx tsx examples/error-recovery.ts reprocess-dlq   # Reprocess DLQ');
+    }
+  } catch (error: any) {
+    console.error('\n💥 Fatal error:', error.message);
+    process.exit(1);
+  }
+}
+
+// Run if executed directly
+if (require.main === module) {
+  main();
+}
+
+// Export for testing
+export {
+  retryWithBackoff,
+  classifyError,
+  CircuitBreaker,
+  DeadLetterQueue,
+  resilientIngestion,
+  ErrorCategory,
+  ClassifiedError,
+};