@fluentcommerce/fc-connect-sdk 0.1.54 → 0.1.56

package/docs/02-CORE-GUIDES/ingestion/examples/performance-optimized.ts

@@ -1,647 +1,647 @@
-/**
- * Performance-Optimized Ingestion Example
- *
- * Demonstrates high-performance ingestion patterns for large-scale data processing:
- * - Dynamic batch sizing based on record complexity
- * - Parallel processing with worker pools
- * - Streaming for memory efficiency
- * - Connection pooling and reuse
- * - Performance monitoring and metrics
- * - Auto-tuning based on throughput
- *
- * Prerequisites:
- * - npm install @fluentcommerce/fc-connect-sdk
- *
- * Usage:
- * npx tsx examples/performance-optimized.ts
- */
-
-import {
-  createClient,
-  S3DataSource,
-  CSVParserService,
-  UniversalMapper,
-  StateService,
-  SimpleKVStore,
-  FluentClient,
-} from '@fluentcommerce/fc-connect-sdk';
-import { cpus } from 'os';
-
-// ============================================================================
-// Configuration
-// ============================================================================
-
-const config = {
-  fluent: {
-    baseUrl: process.env.FLUENT_BASE_URL!,
-    clientId: process.env.FLUENT_CLIENT_ID!,
-    clientSecret: process.env.FLUENT_CLIENT_SECRET!,
-    retailerId: process.env.FLUENT_RETAILER_ID!,
-  },
-  s3: {
-    accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
-    region: process.env.AWS_REGION || 'us-east-1',
-  },
-  performance: {
-    // Worker pool configuration
-    minWorkers: 2,
-    maxWorkers: Math.max(2, cpus().length - 1), // Leave 1 CPU free
-
-    // Batch size configuration
-    minBatchSize: 500,
-    maxBatchSize: 10000,
-    targetBatchTimeMs: 5000, // Target 5s per batch
-
-    // Streaming configuration
-    streamThresholdBytes: 10 * 1024 * 1024, // 10 MB
-    streamChunkSize: 1000,
-
-    // Connection pooling
-    maxConcurrentRequests: 10,
-    requestTimeoutMs: 30000,
-
-    // Auto-tuning
-    enableAutoTuning: true,
-    samplingWindow: 10, // Sample last N batches for tuning
-  },
-};
-
-const mappingConfig = {
-  fields: {
-    ref: { source: 'sku', required: true },
-    productRef: { source: 'sku', required: true },
-    locationRef: { source: 'location', required: true },
-    qty: { source: 'quantity', resolver: 'sdk.parseInt', required: true },
-    status: { source: 'status', resolver: 'sdk.uppercase', defaultValue: 'AVAILABLE' },
-  },
-};
-
-// ============================================================================
-// Performance Metrics
-// ============================================================================
-
-interface PerformanceMetrics {
-  startTime: number;
-  endTime?: number;
-  totalFiles: number;
-  totalRecords: number;
-  processedRecords: number;
-  failedRecords: number;
-  totalBatches: number;
-  batchMetrics: Array<{
-    batchSize: number;
-    processingTimeMs: number;
-    recordsPerSecond: number;
-  }>;
-  workerMetrics: Array<{
-    workerId: number;
-    filesProcessed: number;
-    recordsProcessed: number;
-    totalTimeMs: number;
-  }>;
-}
-
-class PerformanceMonitor {
-  private metrics: PerformanceMetrics = {
-    startTime: Date.now(),
-    totalFiles: 0,
-    totalRecords: 0,
-    processedRecords: 0,
-    failedRecords: 0,
-    totalBatches: 0,
-    batchMetrics: [],
-    workerMetrics: [],
-  };
-
-  recordBatch(batchSize: number, processingTimeMs: number): void {
-    const recordsPerSecond = (batchSize / processingTimeMs) * 1000;
-
-    this.metrics.batchMetrics.push({
-      batchSize,
-      processingTimeMs,
-      recordsPerSecond,
-    });
-
-    this.metrics.totalBatches++;
-    this.metrics.processedRecords += batchSize;
-  }
-
-  recordWorker(
-    workerId: number,
-    filesProcessed: number,
-    recordsProcessed: number,
-    totalTimeMs: number
-  ): void {
-    this.metrics.workerMetrics.push({
-      workerId,
-      filesProcessed,
-      recordsProcessed,
-      totalTimeMs,
-    });
-  }
-
-  complete(): PerformanceMetrics {
-    this.metrics.endTime = Date.now();
-    return this.metrics;
-  }
-
-  getAverageBatchTime(): number {
-    if (this.metrics.batchMetrics.length === 0) return 0;
-
-    const sum = this.metrics.batchMetrics.reduce((acc, m) => acc + m.processingTimeMs, 0);
-    return sum / this.metrics.batchMetrics.length;
-  }
-
-  getAverageRecordsPerSecond(): number {
-    if (this.metrics.batchMetrics.length === 0) return 0;
-
-    const sum = this.metrics.batchMetrics.reduce((acc, m) => acc + m.recordsPerSecond, 0);
-    return sum / this.metrics.batchMetrics.length;
-  }
-
-  getRecentBatchMetrics(
-    count: number = 10
-  ): Array<{ batchSize: number; processingTimeMs: number; recordsPerSecond: number }> {
-    return this.metrics.batchMetrics.slice(-count);
-  }
-}
-
-// ============================================================================
-// Dynamic Batch Sizing
-// ============================================================================
-
-class DynamicBatchSizer {
-  private currentBatchSize: number;
-
-  constructor(
-    private minSize: number = config.performance.minBatchSize,
-    private maxSize: number = config.performance.maxBatchSize,
-    private targetTimeMs: number = config.performance.targetBatchTimeMs,
-    private monitor: PerformanceMonitor
-  ) {
-    this.currentBatchSize = Math.floor((minSize + maxSize) / 2);
-  }
-
-  /**
-   * Auto-tune batch size based on recent performance
-   */
-  tune(): void {
-    if (!config.performance.enableAutoTuning) return;
-
-    const recentMetrics = this.monitor.getRecentBatchMetrics(config.performance.samplingWindow);
-
-    if (recentMetrics.length < 3) return; // Need minimum samples
-
-    const avgTime =
-      recentMetrics.reduce((sum, m) => sum + m.processingTimeMs, 0) / recentMetrics.length;
-    const avgBatchSize =
-      recentMetrics.reduce((sum, m) => sum + m.batchSize, 0) / recentMetrics.length;
-
-    // If batches are too fast, increase batch size
-    if (avgTime < this.targetTimeMs * 0.5 && this.currentBatchSize < this.maxSize) {
-      this.currentBatchSize = Math.min(this.maxSize, Math.floor(this.currentBatchSize * 1.5));
-      console.log(
-        `📈 Batch size increased to ${this.currentBatchSize} (batches completing too quickly)`
-      );
-    }
-
-    // If batches are too slow, decrease batch size
-    if (avgTime > this.targetTimeMs * 1.5 && this.currentBatchSize > this.minSize) {
-      this.currentBatchSize = Math.max(this.minSize, Math.floor(this.currentBatchSize * 0.75));
-      console.log(`📉 Batch size decreased to ${this.currentBatchSize} (batches taking too long)`);
-    }
-  }
-
-  getBatchSize(): number {
-    return this.currentBatchSize;
-  }
-
-  setBatchSize(size: number): void {
-    this.currentBatchSize = Math.max(this.minSize, Math.min(this.maxSize, size));
-  }
-}
-
-// ============================================================================
-// Worker Pool for Parallel Processing
-// ============================================================================
-
-interface WorkerTask {
-  fileKey: string;
-  priority: number;
-}
-
-class WorkerPool {
-  private queue: WorkerTask[] = [];
-  private activeWorkers = 0;
-  private results: Array<{ fileKey: string; success: boolean; error?: string }> = [];
-
-  constructor(
-    private maxWorkers: number = config.performance.maxWorkers,
-    private processFile: (fileKey: string, workerId: number) => Promise<void>
-  ) {}
-
-  /**
-   * Add file to processing queue
-   */
-  enqueue(fileKey: string, priority: number = 0): void {
-    this.queue.push({ fileKey, priority });
-
-    // Sort by priority (higher priority first)
-    this.queue.sort((a, b) => b.priority - a.priority);
-  }
-
-  /**
-   * Process all queued files with worker pool
-   */
-  async processAll(): Promise<void> {
-    const workers: Promise<void>[] = [];
-
-    for (let workerId = 0; workerId < this.maxWorkers; workerId++) {
-      workers.push(this.worker(workerId));
-    }
-
-    await Promise.all(workers);
-  }
-
-  /**
-   * Worker loop - processes files from queue
-   */
-  private async worker(workerId: number): Promise<void> {
-    const workerStartTime = Date.now();
-    let filesProcessed = 0;
-    let recordsProcessed = 0;
-
-    while (this.queue.length > 0) {
-      const task = this.queue.shift();
-      if (!task) break;
-
-      this.activeWorkers++;
-
-      try {
-        await this.processFile(task.fileKey, workerId);
-
-        this.results.push({
-          fileKey: task.fileKey,
-          success: true,
-        });
-
-        filesProcessed++;
-      } catch (error: any) {
-        console.error(`[Worker ${workerId}] ❌ Failed to process ${task.fileKey}:`, error.message);
-
-        this.results.push({
-          fileKey: task.fileKey,
-          success: false,
-          error: error.message,
-        });
-      } finally {
-        this.activeWorkers--;
-      }
-    }
-
-    const workerTotalTime = Date.now() - workerStartTime;
-
-    if (filesProcessed > 0) {
-      console.log(
-        `[Worker ${workerId}] ✅ Completed ${filesProcessed} files in ${(workerTotalTime / 1000).toFixed(2)}s`
-      );
-    }
-  }
-
-  getResults(): Array<{ fileKey: string; success: boolean; error?: string }> {
-    return this.results;
-  }
-
-  getActiveWorkerCount(): number {
-    return this.activeWorkers;
-  }
-}
-
-// ============================================================================
-// Streaming Processing for Large Files
-// ============================================================================
-
-async function streamLargeFile(
-  fileKey: string,
-  s3: S3DataSource,
-  parser: CSVParserService,
-  mapper: UniversalMapper,
-  client: FluentClient,
-  monitor: PerformanceMonitor,
-  batchSizer: DynamicBatchSizer
-): Promise<void> {
-  console.log(`  🌊 Streaming large file: ${fileKey}`);
-
-  // Stream file from S3
-  const stream = await s3.streamObject('inventory-bucket', fileKey);
-
-  // Parse CSV stream
-  const recordIterator = parser.parseStream(stream);
-
-  let batch: any[] = [];
-  let totalProcessed = 0;
-
-  // Create job once for all batches
-  const job = await client.createJob({
-    name: `Stream Import - ${fileKey}`,
-    retailerId: config.fluent.retailerId,
-  });
-
-  for await (const record of recordIterator) {
-    batch.push(record);
-
-    // When batch reaches target size, process it
-    if (batch.length >= batchSizer.getBatchSize()) {
-      await processBatch(batch, job.id, mapper, client, monitor);
-      totalProcessed += batch.length;
-
-      console.log(`    📦 Streamed batch: ${totalProcessed} records processed`);
-
-      // Auto-tune batch size
-      batchSizer.tune();
-
-      batch = [];
-    }
-  }
-
-  // Process remaining records
-  if (batch.length > 0) {
-    await processBatch(batch, job.id, mapper, client, monitor);
-    totalProcessed += batch.length;
-  }
-
-  console.log(`  ✅ Stream complete: ${totalProcessed} total records`);
-}
-
-/**
- * Process a single batch with performance tracking
- */
-async function processBatch(
-  records: any[],
-  jobId: string,
-  mapper: UniversalMapper,
-  client: FluentClient,
-  monitor: PerformanceMonitor
-): Promise<void> {
-  const batchStartTime = Date.now();
-
-  // Transform records
-  const result = await mapper.map(records);
-
-  if (!result.success) {
-    throw new Error(`Mapping failed: ${result.errors.join('; ')}`);
-  }
-
-  // Send to Fluent
-  await client.sendBatch(jobId, {
-    action: 'UPSERT',
-    entityType: 'INVENTORY',
-    source: 'PERFORMANCE_OPTIMIZED',
-    event: 'INVENTORY_UPDATE',
-    entities: result.data,
-  });
-
-  const batchTime = Date.now() - batchStartTime;
-
-  // Record metrics
-  monitor.recordBatch(result.data.length, batchTime);
-}
-
-// ============================================================================
-// Optimized File Processing
-// ============================================================================
-
-async function processFileOptimized(
-  fileKey: string,
-  workerId: number,
-  services: {
-    s3: S3DataSource;
-    parser: CSVParserService;
-    mapper: UniversalMapper;
-    client: FluentClient;
-    state: StateService;
-    monitor: PerformanceMonitor;
-    batchSizer: DynamicBatchSizer;
-  }
-): Promise<void> {
-  const { s3, parser, mapper, client, state, monitor, batchSizer } = services;
-
-  console.log(`[Worker ${workerId}] 📄 Processing: ${fileKey}`);
-
-  // Check if already processed
-  if (await state.isFileProcessed(fileKey)) {
-    console.log(`[Worker ${workerId}]   ✓ Already processed`);
-    return;
-  }
-
-  // Acquire lock
-  const lock = await state.acquireLock(fileKey, 60000);
-  if (!lock.acquired) {
-    console.log(`[Worker ${workerId}]   ⏸ Locked by another worker`);
-    return;
-  }
-
-  try {
-    // Get file metadata to determine if streaming is needed
-    const files = await s3.listFiles({ prefix: 'data/' });
-    const fileInfo = files.find(f => f.path === fileKey);
-
-    if (!fileInfo) {
-      throw new Error('File not found');
-    }
-
-    // Use streaming for large files
-    if (fileInfo.size && fileInfo.size > config.performance.streamThresholdBytes) {
-      await streamLargeFile(fileKey, s3, parser, mapper, client, monitor, batchSizer);
-    } else {
-      // Load entire file for small files (faster)
-      console.log(
-        `[Worker ${workerId}]   📖 Loading file (${(fileInfo.size! / 1024).toFixed(1)} KB)`
-      );
-
-      const data = await s3.downloadFile(fileKey);
-      const records = await parser.parse(data);
-
-      console.log(`[Worker ${workerId}]   🔍 Parsed ${records.length} records`);
-
-      // Create job
-      const job = await client.createJob({
-        name: `Import - ${fileKey}`,
-        retailerId: config.fluent.retailerId,
-      });
-
-      // Split into optimized batches
-      const batchSize = batchSizer.getBatchSize();
-      const batches: any[][] = [];
-
-      for (let i = 0; i < records.length; i += batchSize) {
-        batches.push(records.slice(i, i + batchSize));
-      }
-
-      console.log(
-        `[Worker ${workerId}]   📦 Processing ${batches.length} batches (size: ${batchSize})`
-      );
-
-      // Process batches
-      for (const batch of batches) {
-        await processBatch(batch, job.id, mapper, client, monitor);
-
-        // Auto-tune after each batch
-        batchSizer.tune();
-      }
-
-      console.log(`[Worker ${workerId}]   ✅ Ingested ${records.length} records`);
-    }
-
-    // Mark as processed
-    await state.markFileProcessed(fileKey, {
-      processedAt: new Date().toISOString(),
-      workerId,
-    });
-  } finally {
-    await state.releaseLock(fileKey);
-  }
-}
-
-// ============================================================================
-// Main Optimized Ingestion Workflow
-// ============================================================================
-
-async function optimizedIngestion(): Promise<PerformanceMetrics> {
-  console.log('🚀 Starting performance-optimized ingestion\n');
-  console.log(`⚙️  Configuration:`);
-  console.log(`   Workers: ${config.performance.minWorkers}-${config.performance.maxWorkers}`);
-  console.log(
-    `   Batch size: ${config.performance.minBatchSize}-${config.performance.maxBatchSize}`
-  );
-  console.log(`   Auto-tuning: ${config.performance.enableAutoTuning ? 'enabled' : 'disabled'}`);
-  console.log(
-    `   Stream threshold: ${(config.performance.streamThresholdBytes / 1024 / 1024).toFixed(1)} MB\n`
-  );
-
-  // Initialize services
-  const monitor = new PerformanceMonitor();
-  const batchSizer = new DynamicBatchSizer(
-    config.performance.minBatchSize,
-    config.performance.maxBatchSize,
-    config.performance.targetBatchTimeMs,
-    monitor
-  );
-
-  const logger = console; // Use console as logger in examples
-  const client = await createClient({ config: config.fluent });
-  const s3 = new S3DataSource(config.s3);
-  const parser = new CSVParserService(logger);
-  const mapper = new UniversalMapper(mappingConfig);
-  const state = new StateService(logger);
-
-  const services = { s3, parser, mapper, client, state, monitor, batchSizer };
-
-  // List files
-  const files = await s3.listFiles({ prefix: 'data/' });
-  const csvFiles = files.filter(f => f.path.endsWith('.csv'));
-
-  console.log(`📁 Found ${csvFiles.length} CSV files\n`);
-
-  monitor.metrics.totalFiles = csvFiles.length;
-
-  // Create worker pool
-  const workerPool = new WorkerPool(config.performance.maxWorkers, (fileKey, workerId) =>
-    processFileOptimized(fileKey, workerId, services)
-  );
-
-  // Prioritize smaller files first (faster initial throughput)
-  for (const file of csvFiles) {
-    const priority = file.size ? -file.size : 0; // Negative = smaller files first
-    workerPool.enqueue(file.path, priority);
-  }
-
-  // Process all files in parallel
-  await workerPool.processAll();
-
-  // Get results
-  const results = workerPool.getResults();
-  const successful = results.filter(r => r.success).length;
-  const failed = results.filter(r => !r.success).length;
-
-  console.log(`\n✅ Processing complete: ${successful} successful, ${failed} failed`);
-
-  return monitor.complete();
-}
-
-// ============================================================================
-// Performance Report
-// ============================================================================
-
-function printPerformanceReport(metrics: PerformanceMetrics): void {
-  const totalTimeMs = metrics.endTime! - metrics.startTime;
-  const totalTimeSec = totalTimeMs / 1000;
-
-  console.log('\n' + '='.repeat(70));
-  console.log('📊 PERFORMANCE REPORT');
-  console.log('='.repeat(70));
-
-  console.log('\n📈 Throughput:');
-  console.log(`   Total time:              ${totalTimeSec.toFixed(2)}s`);
-  console.log(`   Total records:           ${metrics.totalRecords.toLocaleString()}`);
-  console.log(`   Processed records:       ${metrics.processedRecords.toLocaleString()}`);
-  console.log(`   Failed records:          ${metrics.failedRecords.toLocaleString()}`);
-  console.log(
-    `   Records/second:          ${(metrics.processedRecords / totalTimeSec).toFixed(2)}`
-  );
-
-  console.log('\n📦 Batching:');
-  console.log(`   Total batches:           ${metrics.totalBatches}`);
-  console.log(
-    `   Avg batch time:          ${(metrics.batchMetrics.reduce((s, m) => s + m.processingTimeMs, 0) / metrics.batchMetrics.length).toFixed(0)}ms`
-  );
-  console.log(
-    `   Avg batch size:          ${(metrics.batchMetrics.reduce((s, m) => s + m.batchSize, 0) / metrics.batchMetrics.length).toFixed(0)}`
-  );
-  console.log(
-    `   Avg records/sec/batch:   ${(metrics.batchMetrics.reduce((s, m) => s + m.recordsPerSecond, 0) / metrics.batchMetrics.length).toFixed(2)}`
-  );
-
-  if (metrics.workerMetrics.length > 0) {
-    console.log('\n👷 Workers:');
-    console.log(`   Total workers:           ${metrics.workerMetrics.length}`);
-    metrics.workerMetrics.forEach(w => {
-      console.log(
-        `   Worker ${w.workerId}:              ${w.filesProcessed} files, ${w.recordsProcessed.toLocaleString()} records, ${(w.totalTimeMs / 1000).toFixed(2)}s`
-      );
-    });
-  }
-
-  console.log('\n' + '='.repeat(70));
-}
-
-// ============================================================================
-// Main Entry Point
-// ============================================================================
-
-async function main() {
-  try {
-    const metrics = await optimizedIngestion();
-    printPerformanceReport(metrics);
-  } catch (error: any) {
-    console.error('\n💥 Fatal error:', error.message);
-    process.exit(1);
-  }
-}
-
-// Run if executed directly
-if (require.main === module) {
-  main();
-}
-
-// Export for testing
-export {
-  optimizedIngestion,
-  PerformanceMonitor,
-  DynamicBatchSizer,
-  WorkerPool,
-  streamLargeFile,
-  processBatch,
-  processFileOptimized,
-};
+/**
+ * Performance-Optimized Ingestion Example
+ *
+ * Demonstrates high-performance ingestion patterns for large-scale data processing:
+ * - Dynamic batch sizing based on record complexity
+ * - Parallel processing with worker pools
+ * - Streaming for memory efficiency
+ * - Connection pooling and reuse
+ * - Performance monitoring and metrics
+ * - Auto-tuning based on throughput
+ *
+ * Prerequisites:
+ * - npm install @fluentcommerce/fc-connect-sdk
+ *
+ * Usage:
+ * npx tsx examples/performance-optimized.ts
+ */
+
+import {
+  createClient,
+  S3DataSource,
+  CSVParserService,
+  UniversalMapper,
+  StateService,
+  SimpleKVStore,
+  FluentClient,
+} from '@fluentcommerce/fc-connect-sdk';
+import { cpus } from 'os';
+
+// ============================================================================
+// Configuration
+// ============================================================================
+
+const config = {
+  fluent: {
+    baseUrl: process.env.FLUENT_BASE_URL!,
+    clientId: process.env.FLUENT_CLIENT_ID!,
+    clientSecret: process.env.FLUENT_CLIENT_SECRET!,
+    retailerId: process.env.FLUENT_RETAILER_ID!,
+  },
+  s3: {
+    accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
+    region: process.env.AWS_REGION || 'us-east-1',
+  },
+  performance: {
+    // Worker pool configuration
+    minWorkers: 2,
+    maxWorkers: Math.max(2, cpus().length - 1), // Leave 1 CPU free
+
+    // Batch size configuration
+    minBatchSize: 500,
+    maxBatchSize: 10000,
+    targetBatchTimeMs: 5000, // Target 5s per batch
+
+    // Streaming configuration
+    streamThresholdBytes: 10 * 1024 * 1024, // 10 MB
+    streamChunkSize: 1000,
+
+    // Connection pooling
+    maxConcurrentRequests: 10,
+    requestTimeoutMs: 30000,
+
+    // Auto-tuning
+    enableAutoTuning: true,
+    samplingWindow: 10, // Sample last N batches for tuning
+  },
+};
+
+const mappingConfig = {
+  fields: {
+    ref: { source: 'sku', required: true },
+    productRef: { source: 'sku', required: true },
+    locationRef: { source: 'location', required: true },
+    qty: { source: 'quantity', resolver: 'sdk.parseInt', required: true },
+    status: { source: 'status', resolver: 'sdk.uppercase', defaultValue: 'AVAILABLE' },
+  },
+};
+
+// ============================================================================
+// Performance Metrics
+// ============================================================================
+
+interface PerformanceMetrics {
+  startTime: number;
+  endTime?: number;
+  totalFiles: number;
+  totalRecords: number;
+  processedRecords: number;
+  failedRecords: number;
+  totalBatches: number;
+  batchMetrics: Array<{
+    batchSize: number;
+    processingTimeMs: number;
+    recordsPerSecond: number;
+  }>;
+  workerMetrics: Array<{
+    workerId: number;
+    filesProcessed: number;
+    recordsProcessed: number;
+    totalTimeMs: number;
+  }>;
+}
+
+class PerformanceMonitor {
+  private metrics: PerformanceMetrics = {
+    startTime: Date.now(),
+    totalFiles: 0,
+    totalRecords: 0,
+    processedRecords: 0,
+    failedRecords: 0,
+    totalBatches: 0,
+    batchMetrics: [],
+    workerMetrics: [],
+  };
+
+  recordBatch(batchSize: number, processingTimeMs: number): void {
+    const recordsPerSecond = (batchSize / processingTimeMs) * 1000;
+
+    this.metrics.batchMetrics.push({
+      batchSize,
+      processingTimeMs,
+      recordsPerSecond,
+    });
+
+    this.metrics.totalBatches++;
+    this.metrics.processedRecords += batchSize;
+  }
+
+  recordWorker(
+    workerId: number,
+    filesProcessed: number,
+    recordsProcessed: number,
+    totalTimeMs: number
+  ): void {
+    this.metrics.workerMetrics.push({
+      workerId,
+      filesProcessed,
+      recordsProcessed,
+      totalTimeMs,
+    });
+  }
+
+  complete(): PerformanceMetrics {
+    this.metrics.endTime = Date.now();
+    return this.metrics;
+  }
+
+  getAverageBatchTime(): number {
+    if (this.metrics.batchMetrics.length === 0) return 0;
+
+    const sum = this.metrics.batchMetrics.reduce((acc, m) => acc + m.processingTimeMs, 0);
+    return sum / this.metrics.batchMetrics.length;
+  }
+
+  getAverageRecordsPerSecond(): number {
+    if (this.metrics.batchMetrics.length === 0) return 0;
+
+    const sum = this.metrics.batchMetrics.reduce((acc, m) => acc + m.recordsPerSecond, 0);
+    return sum / this.metrics.batchMetrics.length;
+  }
+
+  getRecentBatchMetrics(
+    count: number = 10
+  ): Array<{ batchSize: number; processingTimeMs: number; recordsPerSecond: number }> {
+    return this.metrics.batchMetrics.slice(-count);
+  }
+}
+
+// ============================================================================
+// Dynamic Batch Sizing
+// ============================================================================
+
+class DynamicBatchSizer {
+  private currentBatchSize: number;
+
+  constructor(
+    private minSize: number = config.performance.minBatchSize,
+    private maxSize: number = config.performance.maxBatchSize,
+    private targetTimeMs: number = config.performance.targetBatchTimeMs,
+    private monitor: PerformanceMonitor
+  ) {
+    this.currentBatchSize = Math.floor((minSize + maxSize) / 2);
+  }
+
+  /**
+   * Auto-tune batch size based on recent performance
+   */
+  tune(): void {
+    if (!config.performance.enableAutoTuning) return;
+
+    const recentMetrics = this.monitor.getRecentBatchMetrics(config.performance.samplingWindow);
+
+    if (recentMetrics.length < 3) return; // Need minimum samples
+
+    const avgTime =
+      recentMetrics.reduce((sum, m) => sum + m.processingTimeMs, 0) / recentMetrics.length;
+    const avgBatchSize =
+      recentMetrics.reduce((sum, m) => sum + m.batchSize, 0) / recentMetrics.length;
+
+    // If batches are too fast, increase batch size
+    if (avgTime < this.targetTimeMs * 0.5 && this.currentBatchSize < this.maxSize) {
+      this.currentBatchSize = Math.min(this.maxSize, Math.floor(this.currentBatchSize * 1.5));
+      console.log(
+        `📈 Batch size increased to ${this.currentBatchSize} (batches completing too quickly)`
+      );
+    }
+
+    // If batches are too slow, decrease batch size
+    if (avgTime > this.targetTimeMs * 1.5 && this.currentBatchSize > this.minSize) {
+      this.currentBatchSize = Math.max(this.minSize, Math.floor(this.currentBatchSize * 0.75));
+      console.log(`📉 Batch size decreased to ${this.currentBatchSize} (batches taking too long)`);
+    }
+  }
+
+  getBatchSize(): number {
+    return this.currentBatchSize;
+  }
+
+  setBatchSize(size: number): void {
+    this.currentBatchSize = Math.max(this.minSize, Math.min(this.maxSize, size));
+  }
+}
+
+// ============================================================================
+// Worker Pool for Parallel Processing
+// ============================================================================
+
+interface WorkerTask {
+  fileKey: string;
+  priority: number;
+}
+
+class WorkerPool {
+  private queue: WorkerTask[] = [];
+  private activeWorkers = 0;
+  private results: Array<{ fileKey: string; success: boolean; error?: string }> = [];
+
+  constructor(
+    private maxWorkers: number = config.performance.maxWorkers,
+    private processFile: (fileKey: string, workerId: number) => Promise<void>
+  ) {}
+
+  /**
+   * Add file to processing queue
+   */
+  enqueue(fileKey: string, priority: number = 0): void {
+    this.queue.push({ fileKey, priority });
+
+    // Sort by priority (higher priority first)
+    this.queue.sort((a, b) => b.priority - a.priority);
+  }
+
+  /**
+   * Process all queued files with worker pool
+   */
+  async processAll(): Promise<void> {
+    const workers: Promise<void>[] = [];
+
+    for (let workerId = 0; workerId < this.maxWorkers; workerId++) {
+      workers.push(this.worker(workerId));
+    }
+
+    await Promise.all(workers);
+  }
+
+  /**
+   * Worker loop - processes files from queue
+   */
+  private async worker(workerId: number): Promise<void> {
+    const workerStartTime = Date.now();
+    let filesProcessed = 0;
+    let recordsProcessed = 0;
+
+    while (this.queue.length > 0) {
+      const task = this.queue.shift();
+      if (!task) break;
+
+      this.activeWorkers++;
+
+      try {
+        await this.processFile(task.fileKey, workerId);
+
+        this.results.push({
+          fileKey: task.fileKey,
+          success: true,
+        });
+
+        filesProcessed++;
+      } catch (error: any) {
+        console.error(`[Worker ${workerId}] ❌ Failed to process ${task.fileKey}:`, error.message);
+
+        this.results.push({
+          fileKey: task.fileKey,
+          success: false,
+          error: error.message,
+        });
+      } finally {
+        this.activeWorkers--;
+      }
+    }
+
+    const workerTotalTime = Date.now() - workerStartTime;
+
+    if (filesProcessed > 0) {
+      console.log(
+        `[Worker ${workerId}] ✅ Completed ${filesProcessed} files in ${(workerTotalTime / 1000).toFixed(2)}s`
+      );
+    }
+  }
+
+  getResults(): Array<{ fileKey: string; success: boolean; error?: string }> {
+    return this.results;
+  }
+
+  getActiveWorkerCount(): number {
+    return this.activeWorkers;
+  }
+}
+
+// ============================================================================
+// Streaming Processing for Large Files
+// ============================================================================
+
+async function streamLargeFile(
+  fileKey: string,
+  s3: S3DataSource,
+  parser: CSVParserService,
+  mapper: UniversalMapper,
+  client: FluentClient,
+  monitor: PerformanceMonitor,
+  batchSizer: DynamicBatchSizer
+): Promise<void> {
+  console.log(`  🌊 Streaming large file: ${fileKey}`);
+
+  // Stream file from S3
+  const stream = await s3.streamObject('inventory-bucket', fileKey);
+
+  // Parse CSV stream
+  const recordIterator = parser.parseStream(stream);
+
+  let batch: any[] = [];
+  let totalProcessed = 0;
+
+  // Create job once for all batches
+  const job = await client.createJob({
+    name: `Stream Import - ${fileKey}`,
+    retailerId: config.fluent.retailerId,
+  });
+
+  for await (const record of recordIterator) {
+    batch.push(record);
+
+    // When batch reaches target size, process it
+    if (batch.length >= batchSizer.getBatchSize()) {
+      await processBatch(batch, job.id, mapper, client, monitor);
+      totalProcessed += batch.length;
+
+      console.log(`    📦 Streamed batch: ${totalProcessed} records processed`);
+
+      // Auto-tune batch size
+      batchSizer.tune();
+
+      batch = [];
+    }
+  }
+
+  // Process remaining records
+  if (batch.length > 0) {
+    await processBatch(batch, job.id, mapper, client, monitor);
+    totalProcessed += batch.length;
+  }
+
+  console.log(`  ✅ Stream complete: ${totalProcessed} total records`);
+}
+
+/**
+ * Process a single batch with performance tracking
+ */
+async function processBatch(
+  records: any[],
+  jobId: string,
+  mapper: UniversalMapper,
+  client: FluentClient,
+  monitor: PerformanceMonitor
+): Promise<void> {
+  const batchStartTime = Date.now();
+
+  // Transform records
+  const result = await mapper.map(records);
+
+  if (!result.success) {
+    throw new Error(`Mapping failed: ${result.errors.join('; ')}`);
+  }
+
+  // Send to Fluent
+  await client.sendBatch(jobId, {
+    action: 'UPSERT',
+    entityType: 'INVENTORY',
+    source: 'PERFORMANCE_OPTIMIZED',
+    event: 'INVENTORY_UPDATE',
+    entities: result.data,
+  });
+
+  const batchTime = Date.now() - batchStartTime;
+
+  // Record metrics
+  monitor.recordBatch(result.data.length, batchTime);
+}
+
+// ============================================================================
+// Optimized File Processing
+// ============================================================================
+
+async function processFileOptimized(
+  fileKey: string,
+  workerId: number,
+  services: {
+    s3: S3DataSource;
+    parser: CSVParserService;
+    mapper: UniversalMapper;
+    client: FluentClient;
+    state: StateService;
+    monitor: PerformanceMonitor;
+    batchSizer: DynamicBatchSizer;
+  }
+): Promise<void> {
+  const { s3, parser, mapper, client, state, monitor, batchSizer } = services;
+
+  console.log(`[Worker ${workerId}] 📄 Processing: ${fileKey}`);
+
+  // Check if already processed
+  if (await state.isFileProcessed(fileKey)) {
+    console.log(`[Worker ${workerId}]   ✓ Already processed`);
+    return;
+  }
+
+  // Acquire lock
+  const lock = await state.acquireLock(fileKey, 60000);
+  if (!lock.acquired) {
+    console.log(`[Worker ${workerId}]   ⏸ Locked by another worker`);
+    return;
+  }
+
+  try {
+    // Get file metadata to determine if streaming is needed
+    const files = await s3.listFiles({ prefix: 'data/' });
+    const fileInfo = files.find(f => f.path === fileKey);
+
+    if (!fileInfo) {
+      throw new Error('File not found');
+    }
+
+    // Use streaming for large files
+    if (fileInfo.size && fileInfo.size > config.performance.streamThresholdBytes) {
+      await streamLargeFile(fileKey, s3, parser, mapper, client, monitor, batchSizer);
+    } else {
+      // Load entire file for small files (faster)
+      console.log(
+        `[Worker ${workerId}]   📖 Loading file (${(fileInfo.size! / 1024).toFixed(1)} KB)`
+      );
+
+      const data = await s3.downloadFile(fileKey);
+      const records = await parser.parse(data);
+
+      console.log(`[Worker ${workerId}]   🔍 Parsed ${records.length} records`);
+
+      // Create job
+      const job = await client.createJob({
+        name: `Import - ${fileKey}`,
+        retailerId: config.fluent.retailerId,
+      });
+
+      // Split into optimized batches
+      const batchSize = batchSizer.getBatchSize();
+      const batches: any[][] = [];
+
+      for (let i = 0; i < records.length; i += batchSize) {
+        batches.push(records.slice(i, i + batchSize));
+      }
+
+      console.log(
+        `[Worker ${workerId}]   📦 Processing ${batches.length} batches (size: ${batchSize})`
+      );
+
+      // Process batches
+      for (const batch of batches) {
+        await processBatch(batch, job.id, mapper, client, monitor);
+
+        // Auto-tune after each batch
+        batchSizer.tune();
+      }
+
+      console.log(`[Worker ${workerId}]   ✅ Ingested ${records.length} records`);
+    }
+
+    // Mark as processed
+    await state.markFileProcessed(fileKey, {
+      processedAt: new Date().toISOString(),
+      workerId,
+    });
+  } finally {
+    await state.releaseLock(fileKey);
+  }
+}
+
+// ============================================================================
+// Main Optimized Ingestion Workflow
+// ============================================================================
+
+async function optimizedIngestion(): Promise<PerformanceMetrics> {
+  console.log('🚀 Starting performance-optimized ingestion\n');
+  console.log(`⚙️  Configuration:`);
+  console.log(`   Workers: ${config.performance.minWorkers}-${config.performance.maxWorkers}`);
+  console.log(
+    `   Batch size: ${config.performance.minBatchSize}-${config.performance.maxBatchSize}`
+  );
+  console.log(`   Auto-tuning: ${config.performance.enableAutoTuning ? 'enabled' : 'disabled'}`);
+  console.log(
+    `   Stream threshold: ${(config.performance.streamThresholdBytes / 1024 / 1024).toFixed(1)} MB\n`
+  );
+
+  // Initialize services
+  const monitor = new PerformanceMonitor();
+  const batchSizer = new DynamicBatchSizer(
+    config.performance.minBatchSize,
+    config.performance.maxBatchSize,
+    config.performance.targetBatchTimeMs,
+    monitor
+  );
+
+  const logger = console; // Use console as logger in examples
+  const client = await createClient({ config: config.fluent });
+  const s3 = new S3DataSource(config.s3);
+  const parser = new CSVParserService(logger);
+  const mapper = new UniversalMapper(mappingConfig);
+  const state = new StateService(logger);
+
+  const services = { s3, parser, mapper, client, state, monitor, batchSizer };
+
+  // List files
+  const files = await s3.listFiles({ prefix: 'data/' });
+  const csvFiles = files.filter(f => f.path.endsWith('.csv'));
+
+  console.log(`📁 Found ${csvFiles.length} CSV files\n`);
+
+  monitor.metrics.totalFiles = csvFiles.length;
+
+  // Create worker pool
+  const workerPool = new WorkerPool(config.performance.maxWorkers, (fileKey, workerId) =>
+    processFileOptimized(fileKey, workerId, services)
+  );
+
+  // Prioritize smaller files first (faster initial throughput)
+  for (const file of csvFiles) {
+    const priority = file.size ? -file.size : 0; // Negative = smaller files first
+    workerPool.enqueue(file.path, priority);
+  }
+
+  // Process all files in parallel
+  await workerPool.processAll();
+
+  // Get results
+  const results = workerPool.getResults();
+  const successful = results.filter(r => r.success).length;
+  const failed = results.filter(r => !r.success).length;
+
+  console.log(`\n✅ Processing complete: ${successful} successful, ${failed} failed`);
+
+  return monitor.complete();
+}
+
+// ============================================================================
+// Performance Report
+// ============================================================================
+
+function printPerformanceReport(metrics: PerformanceMetrics): void {
+  const totalTimeMs = metrics.endTime! - metrics.startTime;
+  const totalTimeSec = totalTimeMs / 1000;
+
+  console.log('\n' + '='.repeat(70));
+  console.log('📊 PERFORMANCE REPORT');
+  console.log('='.repeat(70));
+
+  console.log('\n📈 Throughput:');
+  console.log(`   Total time:              ${totalTimeSec.toFixed(2)}s`);
+  console.log(`   Total records:           ${metrics.totalRecords.toLocaleString()}`);
+  console.log(`   Processed records:       ${metrics.processedRecords.toLocaleString()}`);
+  console.log(`   Failed records:          ${metrics.failedRecords.toLocaleString()}`);
+  console.log(
+    `   Records/second:          ${(metrics.processedRecords / totalTimeSec).toFixed(2)}`
+  );
+
+  console.log('\n📦 Batching:');
+  console.log(`   Total batches:           ${metrics.totalBatches}`);
+  console.log(
+    `   Avg batch time:          ${(metrics.batchMetrics.reduce((s, m) => s + m.processingTimeMs, 0) / metrics.batchMetrics.length).toFixed(0)}ms`
+  );
+  console.log(
+    `   Avg batch size:          ${(metrics.batchMetrics.reduce((s, m) => s + m.batchSize, 0) / metrics.batchMetrics.length).toFixed(0)}`
+  );
+  console.log(
+    `   Avg records/sec/batch:   ${(metrics.batchMetrics.reduce((s, m) => s + m.recordsPerSecond, 0) / metrics.batchMetrics.length).toFixed(2)}`
+  );
+
+  if (metrics.workerMetrics.length > 0) {
+    console.log('\n👷 Workers:');
+    console.log(`   Total workers:           ${metrics.workerMetrics.length}`);
+    metrics.workerMetrics.forEach(w => {
+      console.log(
+        `   Worker ${w.workerId}:              ${w.filesProcessed} files, ${w.recordsProcessed.toLocaleString()} records, ${(w.totalTimeMs / 1000).toFixed(2)}s`
+      );
+    });
+  }
+
+  console.log('\n' + '='.repeat(70));
+}
+
+// ============================================================================
+// Main Entry Point
+// ============================================================================
+
+async function main() {
+  try {
+    const metrics = await optimizedIngestion();
+    printPerformanceReport(metrics);
+  } catch (error: any) {
+    console.error('\n💥 Fatal error:', error.message);
+    process.exit(1);
+  }
+}
+
+// Run if executed directly
+if (require.main === module) {
+  main();
+}
+
+// Export for testing
+export {
+  optimizedIngestion,
+  PerformanceMonitor,
+  DynamicBatchSizer,
+  WorkerPool,
+  streamLargeFile,
+  processBatch,
+  processFileOptimized,
+};