@fluentcommerce/fc-connect-sdk 0.1.53 → 0.1.55

package/docs/03-PATTERN-GUIDES/connector-scenarios/modules/connector-scenarios-04-advanced-scenarios.md

@@ -1,650 +1,650 @@
-# Module 4: Advanced Scenarios
-
-**Level:** Advanced
-**Estimated Time:** 60 minutes
-
-## Overview
-
-This module covers advanced connector patterns including state management with checkpoints, multi-source aggregation, and large file streaming. These patterns are essential for production-grade connectors.
-
-## Learning Objectives
-
-By the end of this module, you will:
-
-- ✅ Implement record-level state management with checkpoints
-- ✅ Handle large files with streaming and chunking
-- ✅ Aggregate data from multiple sources
-- ✅ Reconcile differences between sources and Fluent
-- ✅ Implement recovery from partial failures
-- ✅ Optimize memory usage for large datasets
-
-## Scenario 4: S3 CSV Batch Processing with State Management
-
-**Goal:** Process large CSV files from S3 with state tracking to handle interruptions and prevent duplicates.
-
-**Complexity:** High
-**Components:** S3DataSource, CSVParserService, StateService, FluentClient
-**Estimated Lines:** ~400 lines
-
-### Problem Statement
-
-You need to process CSV files that may contain 100K+ records. Requirements:
-
-1. Stream large files (don't load all into memory)
-2. Process in batches of 500 records
-3. Save checkpoint after each batch
-4. Resume from checkpoint on failure
-5. Prevent duplicate processing
-
-### Key Implementation Pattern
-
-```typescript
-import {
-  createClient,
-  S3DataSource,
-  CSVParserService,
-  StateService,
-  BatchAPIClient,
-  createConsoleLogger,
-  toStructuredLogger
-} from '@fluentcommerce/fc-connect-sdk';
-
-const logger = toStructuredLogger(createConsoleLogger(), {
-  service: 's3-batch-processing',
-  module: 'advanced-scenarios'
-});
-
-// Initialize State Service
-const stateService = new StateService(logger);
-
-// For checkpoints, use KV directly:
-// const kv = openKv();
-// await kv.set(['checkpoint', jobId], { recordsProcessed, lastRecordId, timestamp });
-
-// S3 Source for CSV files
-const s3Source = new S3DataSource(
-  {
-    type: 'S3_CSV',
-    connectionId: 's3-inventory',
-    name: 'Inventory S3',
-    settings: {
-      bucketName: 'inventory-updates',
-      region: 'us-east-1',
-      accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
-      prefix: 'daily/',
-      filePattern: '*.csv',
-    },
-  },
-  logger
-);
-
-// CSV Parser configuration
-// Constructor CAN take logger as first parameter (optional)
-const csvParser = new CSVParserService(logger);
-// Parse options passed to parse() method
-
-const fluentClient = await createClient({
-  config: {
-    baseUrl: process.env.FLUENT_BASE_URL!,
-    clientId: process.env.FLUENT_CLIENT_ID!,
-    clientSecret: process.env.FLUENT_CLIENT_SECRET!,
-    username: process.env.FLUENT_USERNAME!,
-    password: process.env.FLUENT_PASSWORD!,
-    retailerId: process.env.FLUENT_RETAILER_ID!,
-  },
-});
-
-// Manual workflow composition for full control
-async function processLargeCSVBatch() {
-  logger.info('Starting large CSV batch processing');
-
-  try {
-    // Get checkpoint from KV (for resuming)
-    // Use KV adapter directly for checkpoints
-    const kv = openKv();
-    const checkpointData = await kv.get(['checkpoint', 'inventory-batch']);
-    const checkpoint = checkpointData?.value;
-
-    // List CSV files
-    const files = await s3Source.listFiles({
-      prefix: 'daily/',
-      lastProcessedTimestamp: checkpoint?.lastProcessedTimestamp,
-    });
-
-    logger.info(`Found ${files.length} files to process`);
-
-    for (const file of files) {
-      // Check if file was already processed (using KV directly)
-      const fileStateData = await kv.get(['file-state', file.name]);
-      const fileState = fileStateData?.value;
-      if (fileState?.status === 'completed') {
-        logger.info(`Skipping already processed file: ${file.name}`);
-        continue;
-      }
-
-      logger.info(`Processing large file: ${file.name} (${file.size} bytes)`);
-
-      // Mark file as in-progress (using KV directly)
-      await kv.set(['file-state', file.name], {
-        status: 'in-progress',
-        startedAt: new Date().toISOString(),
-      });
-
-      try {
-        // Stream large CSV file
-        const stream = await s3Source.getFileStream(file.name);
-
-        let batchNumber = 0;
-        let totalProcessed = 0;
-        let buffer: any[] = [];
-
-        // Process streaming data with async iterator
-        // Using CSVParserService.parseStreamStreaming for memory-efficient processing
-        for await (const record of csvParser.parseStreamStreaming(stream, {}, 1)) {
-          buffer.push(record);
-
-          // Process buffer when it reaches batch size
-          if (buffer.length >= 500) {
-            await processBatch(buffer, file.name, ++batchNumber);
-            totalProcessed += buffer.length;
-            buffer = [];
-
-            // Update checkpoint (using KV directly)
-            await kv.set(['checkpoint', 'inventory-batch'], {
-              file: file.name,
-              recordsProcessed: totalProcessed,
-              lastBatch: batchNumber,
-              timestamp: new Date().toISOString(),
-            });
-          }
-        }
-
-        // Process remaining records
-        if (buffer.length > 0) {
-          await processBatch(buffer, file.name, ++batchNumber);
-          totalProcessed += buffer.length;
-        }
-
-        // Mark file as completed (using KV directly)
-        await kv.set(['file-state', file.name], {
-          status: 'completed',
-          completedAt: new Date().toISOString(),
-          recordsProcessed: totalProcessed,
-        });
-
-        logger.info(`File completed: ${file.name}, processed ${totalProcessed} records`);
-
-        // Archive processed file
-        await s3Source.moveFile(
-          file.name,
-          `archive/${new Date().toISOString().split('T')[0]}/${file.name}`
-        );
-      } catch (error) {
-        logger.error(`Failed to process file ${file.name}:`, error);
-
-        // Mark file as failed (using KV directly)
-        await kv.set(['file-state', file.name], {
-          status: 'failed',
-          error: (error as Error).message,
-          failedAt: new Date().toISOString(),
-        });
-
-        // Move to error bucket
-        await s3Source.copyFile(file.name, `errors/${file.name}`);
-      }
-    }
-
-    logger.info('Batch processing completed');
-  } catch (error) {
-    logger.error('Batch processing failed:', error);
-    throw error;
-  }
-}
-
-async function processBatch(records: any[], fileName: string, batchNumber: number) {
-  logger.info(`Processing batch ${batchNumber} with ${records.length} records`);
-
-  // Transform records
-  const inventoryUpdates = records.map(record => ({
-    ref: record.sku,
-    productRef: record.sku,
-    locationRef: record.location,
-    qty: parseInt(record.quantity),
-    status: record.status || 'ACTIVE',
-    type: 'ADJUSTMENT',
-    attributes: {
-      source: 'S3_CSV',
-      file: fileName,
-      batch: batchNumber,
-    },
-  }));
-
-  // Create job for this batch
-  const job = await fluentClient.createJob({
-    name: `Inventory Batch ${fileName} - ${batchNumber}`,
-    retailerId: process.env.FLUENT_RETAILER_ID!,
-  });
-
-  // Send batch to Fluent
-  await fluentClient.sendBatch(job.id, {
-    action: 'UPSERT',
-    entityType: 'INVENTORY',
-    entities: inventoryUpdates,
-  });
-
-  // Poll status with getBatchStatus if needed; no explicit update call is required
-
-  logger.info(`Batch ${batchNumber} completed`);
-
-  return inventoryUpdates.length;
-}
-
-// Execute with error recovery
-async function runWithRecovery() {
-  const maxRetries = 3;
-  let attempt = 0;
-
-  while (attempt < maxRetries) {
-    try {
-      await processLargeCSVBatch();
-      break;
-    } catch (error) {
-      attempt++;
-      logger.error(`Attempt ${attempt} failed:`, error);
-
-      if (attempt >= maxRetries) {
-        logger.error('Max retries reached, giving up');
-        throw error;
-      }
-
-      // Wait before retry (exponential backoff)
-      await new Promise(resolve => setTimeout(resolve, 5000 * attempt));
-    }
-  }
-}
-
-runWithRecovery();
-```
-
-### Key Concepts: State Management
-
-**Checkpoint Strategy:**
-
-```typescript
-// Use KV adapter directly for checkpoints
-const kv = openKv();
-
-// Save checkpoint after each batch (using KV directly)
-await kv.set(['checkpoint', 'job-id'], {
-  file: currentFile,
-  lastRecordId: record.id,
-  recordsProcessed: count,
-  timestamp: new Date().toISOString(),
-});
-
-// Resume from checkpoint on failure
-const checkpointData = await kv.get(['checkpoint', 'job-id']);
-const checkpoint = checkpointData?.value;
-if (checkpoint) {
-  logger.info(`Resuming from batch ${checkpoint.lastBatch}`);
-  startFrom = checkpoint.recordsProcessed;
-}
-```
-
-**Streaming Pattern:**
-
-```typescript
-// Don't load entire file into memory
-const stream = await s3Source.getFileStream(file.name);
-let buffer = [];
-
-// Use CSVParserService.parseStreamStreaming for memory-efficient processing
-const csvParser = new CSVParserService();
-
-for await (const record of csvParser.parseStreamStreaming(stream, {}, 1)) {
-  buffer.push(record);
-
-  if (buffer.length >= BATCH_SIZE) {
-    await processBatch(buffer); // Process batch
-    buffer = []; // Clear buffer
-  }
-}
-
-// Process remaining records
-if (buffer.length > 0) {
-  await processBatch(buffer);
-}
-```
-
-## Scenario 5: Multi-Source Inventory Aggregation
-
-**Goal:** Aggregate inventory from both SFTP and S3 sources, reconcile differences, and update Fluent.
-
-**Complexity:** High
-**Components:** SftpDataSource, S3DataSource, Multiple parsers, Reconciliation logic
-**Estimated Lines:** ~350 lines
-
-### Problem Statement
-
-Inventory data comes from multiple sources:
-
-- **SFTP:** Warehouse inventory (CSV)
-- **S3:** Store inventory (JSON)
-
-You need to:
-
-1. Collect from both sources
-2. Merge by SKU + Location
-3. Compare with current Fluent inventory
-4. Send only differences (deltas)
-
-### Key Implementation Pattern
-
-```typescript
-import {
-  createClient,
-  SftpDataSource,
-  S3DataSource,
-  createConsoleLogger,
-  toStructuredLogger
-} from '@fluentcommerce/fc-connect-sdk';
-
-const logger = toStructuredLogger(createConsoleLogger(), {
-  service: 'multi-source-aggregation',
-  module: 'advanced-scenarios'
-});
-
-// Multiple data sources
-const sftpSource = new SftpDataSource(
-  {
-    type: 'SFTP_CSV',
-    connectionId: 'warehouse-sftp',
-    name: 'Warehouse SFTP',
-    settings: {
-      host: process.env.WAREHOUSE_SFTP_HOST!,
-      username: process.env.WAREHOUSE_SFTP_USER!,
-      password: process.env.WAREHOUSE_SFTP_PASS!,
-      remotePath: '/inventory',
-      filePattern: '*.csv',
-    },
-  },
-  logger
-);
-
-const s3Source = new S3DataSource(
-  {
-    type: 'S3_JSON',
-    connectionId: 'store-s3',
-    name: 'Store Inventory S3',
-    settings: {
-      bucketName: 'store-inventory',
-      region: 'us-east-1',
-      accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
-      prefix: 'stores/',
-      filePattern: '*.json',
-    },
-  },
-  logger
-);
-
-const fluentClient = await createClient({
-  config: {
-    baseUrl: process.env.FLUENT_BASE_URL!,
-    clientId: process.env.FLUENT_CLIENT_ID!,
-    clientSecret: process.env.FLUENT_CLIENT_SECRET!,
-    retailerId: process.env.FLUENT_RETAILER_ID!,
-  },
-});
-
-async function aggregateInventory() {
-  logger.info('Starting multi-source inventory aggregation');
-
-  try {
-    // Collect inventory from all sources
-    const inventoryMap = new Map<
-      string,
-      {
-        sku: string;
-        locations: Map<string, number>;
-        sources: Set<string>;
-      }
-    >();
-
-    // 1. Process SFTP warehouse data
-    logger.info('Processing warehouse inventory from SFTP');
-    const sftpFiles = await sftpSource.listFiles();
-
-    for (const file of sftpFiles) {
-      const content = await sftpSource.downloadFile(file.name);
-      const parser = new CSVParserService();
-      const records = await parser.parse(content as string);
-
-      records.forEach(record => {
-        const key = record.sku;
-        if (!inventoryMap.has(key)) {
-          inventoryMap.set(key, {
-            sku: record.sku,
-            locations: new Map(),
-            sources: new Set(),
-          });
-        }
-
-        const item = inventoryMap.get(key)!;
-        const currentQty = item.locations.get(record.location) || 0;
-        item.locations.set(record.location, currentQty + parseInt(record.quantity));
-        item.sources.add('WAREHOUSE_SFTP');
-      });
-    }
-
-    // 2. Process S3 store data
-    logger.info('Processing store inventory from S3');
-    const s3Files = await s3Source.listFiles();
-
-    for (const file of s3Files) {
-      const content = await s3Source.downloadFile(file.name);
-      const storeData = JSON.parse(content.toString());
-
-      storeData.inventory.forEach((record: any) => {
-        const key = record.productId;
-        if (!inventoryMap.has(key)) {
-          inventoryMap.set(key, {
-            sku: record.productId,
-            locations: new Map(),
-            sources: new Set(),
-          });
-        }
-
-        const item = inventoryMap.get(key)!;
-        const currentQty = item.locations.get(record.storeId) || 0;
-        item.locations.set(record.storeId, currentQty + record.availableQty);
-        item.sources.add('STORE_S3');
-      });
-    }
-
-    // 3. Query current Fluent inventory for reconciliation
-    logger.info('Fetching current Fluent inventory');
-    const query = `
-      query GetCurrentInventory($retailerId: ID!, $first: Int!) {
-        inventoryPositions(retailerId: $retailerId, first: $first) {
-          edges {
-            node {
-              ref
-              productRef
-              locationRef
-              qty
-              availableQty
-            }
-            cursor
-          }
-          pageInfo { hasNextPage }
-        }
-      }
-    `;
-
-    const currentInventory = await fluentClient.graphql({
-      query,
-      variables: {
-        retailerId: process.env.FLUENT_RETAILER_ID!,
-        first: 1000,
-      },
-      pagination: { maxPages: 10 },
-    });
-
-    // 4. Reconcile and prepare updates
-    const updates: any[] = [];
-    const reconciliationReport: any[] = [];
-
-    inventoryMap.forEach((aggregated, sku) => {
-      aggregated.locations.forEach((newQty, location) => {
-        // Find current Fluent inventory
-        const current = currentInventory.data.inventoryPositions.edges.find(
-          (edge: any) => edge.node.productRef === sku && edge.node.locationRef === location
-        );
-
-        const currentQty = current?.node.qty || 0;
-        const difference = newQty - currentQty;
-
-        if (difference !== 0) {
-          updates.push({
-            ref: `${sku}-${location}`,
-            productRef: sku,
-            locationRef: location,
-            qty: newQty,
-            type: difference > 0 ? 'RECEIPT' : 'DISPATCH',
-            adjustmentQty: Math.abs(difference),
-            reason: 'MULTI_SOURCE_SYNC',
-            attributes: {
-              sources: Array.from(aggregated.sources).join(','),
-              previousQty: currentQty,
-              newQty: newQty,
-              syncedAt: new Date().toISOString(),
-            },
-          });
-
-          reconciliationReport.push({
-            sku,
-            location,
-            previousQty: currentQty,
-            newQty: newQty,
-            difference,
-            sources: Array.from(aggregated.sources),
-          });
-        }
-      });
-    });
-
-    logger.info(`Found ${updates.length} inventory differences to sync`);
-
-    // 5. Apply updates to Fluent
-    if (updates.length > 0) {
-      const job = await fluentClient.createJob({
-        name: `Multi-Source Inventory Sync - ${new Date().toISOString()}`,
-        retailerId: process.env.FLUENT_RETAILER_ID!,
-      });
-
-      // Process in batches
-      const batchSize = 100;
-      for (let i = 0; i < updates.length; i += batchSize) {
-        const batch = updates.slice(i, i + batchSize);
-
-        await fluentClient.sendBatch(job.id, {
-          action: 'UPSERT',
-          entityType: 'INVENTORY',
-          entities: batch,
-        });
-
-        logger.info(`Batch ${Math.floor(i / batchSize) + 1} processed`);
-      }
-
-      // Optional: monitor batch status via getBatchStatus
-    }
-
-    // 6. Generate and save reconciliation report
-    const report = {
-      timestamp: new Date().toISOString(),
-      sourcesProcessed: {
-        sftp: sftpFiles.length,
-        s3: s3Files.length,
-      },
-      totalSKUs: inventoryMap.size,
-      totalUpdates: updates.length,
-      reconciliation: reconciliationReport,
-    };
-
-    // Save report to S3
-    await s3Source.uploadFile(
-      `reports/reconciliation-${new Date().toISOString()}.json`,
-      JSON.stringify(report, null, 2)
-    );
-
-    logger.info('Multi-source inventory aggregation completed', {
-      totalSKUs: inventoryMap.size,
-      totalUpdates: updates.length,
-    });
-
-    return report;
-  } catch (error) {
-    logger.error('Aggregation failed:', error);
-    throw error;
-  }
-}
-
-// Schedule hourly
-setInterval(aggregateInventory, 60 * 60 * 1000);
-aggregateInventory();
-```
-
-### Key Concepts: Multi-Source Aggregation
-
-**Data Collection Pattern:**
-
-```typescript
-// Use Map for efficient lookups and aggregation
-const inventoryMap = new Map<
-  string,
-  {
-    sku: string;
-    locations: Map<string, number>;
-    sources: Set<string>;
-  }
->();
-
-// Aggregate from multiple sources
-sources.forEach(source => {
-  source.records.forEach(record => {
-    const item = inventoryMap.get(record.sku) || createNewItem();
-    item.locations.set(record.location, record.qty);
-    item.sources.add(source.name);
-  });
-});
-```
-
-**Reconciliation Pattern:**
-
-```typescript
-// Compare aggregated vs current
-const difference = newQty - currentQty;
-
-if (difference !== 0) {
-  updates.push({
-    qty: newQty,
-    type: difference > 0 ? 'RECEIPT' : 'DISPATCH',
-    adjustmentQty: Math.abs(difference),
-  });
-}
-```
-
-## Key Takeaways
-
-- 🎯 Stream large files to avoid memory issues (don't load all at once)
-- 🎯 Save checkpoints frequently to enable recovery from failures
-- 🎯 Multi-source aggregation requires careful reconciliation logic
-- 🎯 Always track differences, not just totals (delta updates)
-- 🎯 State management is critical for production reliability
-- 🎯 Generate reports for visibility into sync operations
-
-## Next Steps
-
-Continue to [Module 5: Bidirectional Sync](./connector-scenarios-05-bidirectional-sync.md) to learn extraction patterns and Fluent-to-external system exports.
+# Module 4: Advanced Scenarios
+
+**Level:** Advanced
+**Estimated Time:** 60 minutes
+
+## Overview
+
+This module covers advanced connector patterns including state management with checkpoints, multi-source aggregation, and large file streaming. These patterns are essential for production-grade connectors.
+
+## Learning Objectives
+
+By the end of this module, you will:
+
+- ✅ Implement record-level state management with checkpoints
+- ✅ Handle large files with streaming and chunking
+- ✅ Aggregate data from multiple sources
+- ✅ Reconcile differences between sources and Fluent
+- ✅ Implement recovery from partial failures
+- ✅ Optimize memory usage for large datasets
+
+## Scenario 4: S3 CSV Batch Processing with State Management
+
+**Goal:** Process large CSV files from S3 with state tracking to handle interruptions and prevent duplicates.
+
+**Complexity:** High
+**Components:** S3DataSource, CSVParserService, StateService, FluentClient
+**Estimated Lines:** ~400 lines
+
+### Problem Statement
+
+You need to process CSV files that may contain 100K+ records. Requirements:
+
+1. Stream large files (don't load all into memory)
+2. Process in batches of 500 records
+3. Save checkpoint after each batch
+4. Resume from checkpoint on failure
+5. Prevent duplicate processing
+
+### Key Implementation Pattern
+
+```typescript
+import {
+  createClient,
+  S3DataSource,
+  CSVParserService,
+  StateService,
+  BatchAPIClient,
+  createConsoleLogger,
+  toStructuredLogger
+} from '@fluentcommerce/fc-connect-sdk';
+
+const logger = toStructuredLogger(createConsoleLogger(), {
+  service: 's3-batch-processing',
+  module: 'advanced-scenarios'
+});
+
+// Initialize State Service
+const stateService = new StateService(logger);
+
+// For checkpoints, use KV directly:
+// const kv = openKv();
+// await kv.set(['checkpoint', jobId], { recordsProcessed, lastRecordId, timestamp });
+
+// S3 Source for CSV files
+const s3Source = new S3DataSource(
+  {
+    type: 'S3_CSV',
+    connectionId: 's3-inventory',
+    name: 'Inventory S3',
+    settings: {
+      bucketName: 'inventory-updates',
+      region: 'us-east-1',
+      accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
+      prefix: 'daily/',
+      filePattern: '*.csv',
+    },
+  },
+  logger
+);
+
+// CSV Parser configuration
+// Constructor CAN take logger as first parameter (optional)
+const csvParser = new CSVParserService(logger);
+// Parse options passed to parse() method
+
+const fluentClient = await createClient({
+  config: {
+    baseUrl: process.env.FLUENT_BASE_URL!,
+    clientId: process.env.FLUENT_CLIENT_ID!,
+    clientSecret: process.env.FLUENT_CLIENT_SECRET!,
+    username: process.env.FLUENT_USERNAME!,
+    password: process.env.FLUENT_PASSWORD!,
+    retailerId: process.env.FLUENT_RETAILER_ID!,
+  },
+});
+
+// Manual workflow composition for full control
+async function processLargeCSVBatch() {
+  logger.info('Starting large CSV batch processing');
+
+  try {
+    // Get checkpoint from KV (for resuming)
+    // Use KV adapter directly for checkpoints
+    const kv = openKv();
+    const checkpointData = await kv.get(['checkpoint', 'inventory-batch']);
+    const checkpoint = checkpointData?.value;
+
+    // List CSV files
+    const files = await s3Source.listFiles({
+      prefix: 'daily/',
+      lastProcessedTimestamp: checkpoint?.lastProcessedTimestamp,
+    });
+
+    logger.info(`Found ${files.length} files to process`);
+
+    for (const file of files) {
+      // Check if file was already processed (using KV directly)
+      const fileStateData = await kv.get(['file-state', file.name]);
+      const fileState = fileStateData?.value;
+      if (fileState?.status === 'completed') {
+        logger.info(`Skipping already processed file: ${file.name}`);
+        continue;
+      }
+
+      logger.info(`Processing large file: ${file.name} (${file.size} bytes)`);
+
+      // Mark file as in-progress (using KV directly)
+      await kv.set(['file-state', file.name], {
+        status: 'in-progress',
+        startedAt: new Date().toISOString(),
+      });
+
+      try {
+        // Stream large CSV file
+        const stream = await s3Source.getFileStream(file.name);
+
+        let batchNumber = 0;
+        let totalProcessed = 0;
+        let buffer: any[] = [];
+
+        // Process streaming data with async iterator
+        // Using CSVParserService.parseStreamStreaming for memory-efficient processing
+        for await (const record of csvParser.parseStreamStreaming(stream, {}, 1)) {
+          buffer.push(record);
+
+          // Process buffer when it reaches batch size
+          if (buffer.length >= 500) {
+            await processBatch(buffer, file.name, ++batchNumber);
+            totalProcessed += buffer.length;
+            buffer = [];
+
+            // Update checkpoint (using KV directly)
+            await kv.set(['checkpoint', 'inventory-batch'], {
+              file: file.name,
+              recordsProcessed: totalProcessed,
+              lastBatch: batchNumber,
+              timestamp: new Date().toISOString(),
+            });
+          }
+        }
+
+        // Process remaining records
+        if (buffer.length > 0) {
+          await processBatch(buffer, file.name, ++batchNumber);
+          totalProcessed += buffer.length;
+        }
+
+        // Mark file as completed (using KV directly)
+        await kv.set(['file-state', file.name], {
+          status: 'completed',
+          completedAt: new Date().toISOString(),
+          recordsProcessed: totalProcessed,
+        });
+
+        logger.info(`File completed: ${file.name}, processed ${totalProcessed} records`);
+
+        // Archive processed file
+        await s3Source.moveFile(
+          file.name,
+          `archive/${new Date().toISOString().split('T')[0]}/${file.name}`
+        );
+      } catch (error) {
+        logger.error(`Failed to process file ${file.name}:`, error);
+
+        // Mark file as failed (using KV directly)
+        await kv.set(['file-state', file.name], {
+          status: 'failed',
+          error: (error as Error).message,
+          failedAt: new Date().toISOString(),
+        });
+
+        // Move to error bucket
+        await s3Source.copyFile(file.name, `errors/${file.name}`);
+      }
+    }
+
+    logger.info('Batch processing completed');
+  } catch (error) {
+    logger.error('Batch processing failed:', error);
+    throw error;
+  }
+}
+
+async function processBatch(records: any[], fileName: string, batchNumber: number) {
+  logger.info(`Processing batch ${batchNumber} with ${records.length} records`);
+
+  // Transform records
+  const inventoryUpdates = records.map(record => ({
+    ref: record.sku,
+    productRef: record.sku,
+    locationRef: record.location,
+    qty: parseInt(record.quantity),
+    status: record.status || 'ACTIVE',
+    type: 'ADJUSTMENT',
+    attributes: {
+      source: 'S3_CSV',
+      file: fileName,
+      batch: batchNumber,
+    },
+  }));
+
+  // Create job for this batch
+  const job = await fluentClient.createJob({
+    name: `Inventory Batch ${fileName} - ${batchNumber}`,
+    retailerId: process.env.FLUENT_RETAILER_ID!,
+  });
+
+  // Send batch to Fluent
+  await fluentClient.sendBatch(job.id, {
+    action: 'UPSERT',
+    entityType: 'INVENTORY',
+    entities: inventoryUpdates,
+  });
+
+  // Poll status with getBatchStatus if needed; no explicit update call is required
+
+  logger.info(`Batch ${batchNumber} completed`);
+
+  return inventoryUpdates.length;
+}
+
+// Execute with error recovery
+async function runWithRecovery() {
+  const maxRetries = 3;
+  let attempt = 0;
+
+  while (attempt < maxRetries) {
+    try {
+      await processLargeCSVBatch();
+      break;
+    } catch (error) {
+      attempt++;
+      logger.error(`Attempt ${attempt} failed:`, error);
+
+      if (attempt >= maxRetries) {
+        logger.error('Max retries reached, giving up');
+        throw error;
+      }
+
+      // Wait before retry (exponential backoff)
+      await new Promise(resolve => setTimeout(resolve, 5000 * attempt));
+    }
+  }
+}
+
+runWithRecovery();
+```
+
+### Key Concepts: State Management
+
+**Checkpoint Strategy:**
+
+```typescript
+// Use KV adapter directly for checkpoints
+const kv = openKv();
+
+// Save checkpoint after each batch (using KV directly)
+await kv.set(['checkpoint', 'job-id'], {
+  file: currentFile,
+  lastRecordId: record.id,
+  recordsProcessed: count,
+  timestamp: new Date().toISOString(),
+});
+
+// Resume from checkpoint on failure
+const checkpointData = await kv.get(['checkpoint', 'job-id']);
+const checkpoint = checkpointData?.value;
+if (checkpoint) {
+  logger.info(`Resuming from batch ${checkpoint.lastBatch}`);
+  startFrom = checkpoint.recordsProcessed;
+}
+```
+
+**Streaming Pattern:**
+
+```typescript
+// Don't load entire file into memory
+const stream = await s3Source.getFileStream(file.name);
+let buffer = [];
+
+// Use CSVParserService.parseStreamStreaming for memory-efficient processing
+const csvParser = new CSVParserService();
+
+for await (const record of csvParser.parseStreamStreaming(stream, {}, 1)) {
+  buffer.push(record);
+
+  if (buffer.length >= BATCH_SIZE) {
+    await processBatch(buffer); // Process batch
+    buffer = []; // Clear buffer
+  }
+}
+
+// Process remaining records
+if (buffer.length > 0) {
+  await processBatch(buffer);
+}
+```
+
+## Scenario 5: Multi-Source Inventory Aggregation
+
+**Goal:** Aggregate inventory from both SFTP and S3 sources, reconcile differences, and update Fluent.
+
+**Complexity:** High
+**Components:** SftpDataSource, S3DataSource, Multiple parsers, Reconciliation logic
+**Estimated Lines:** ~350 lines
+
+### Problem Statement
+
+Inventory data comes from multiple sources:
+
+- **SFTP:** Warehouse inventory (CSV)
+- **S3:** Store inventory (JSON)
+
+You need to:
+
+1. Collect from both sources
+2. Merge by SKU + Location
+3. Compare with current Fluent inventory
+4. Send only differences (deltas)
+
+### Key Implementation Pattern
+
+```typescript
+import {
+  createClient,
+  SftpDataSource,
+  S3DataSource,
+  createConsoleLogger,
+  toStructuredLogger
+} from '@fluentcommerce/fc-connect-sdk';
+
+const logger = toStructuredLogger(createConsoleLogger(), {
+  service: 'multi-source-aggregation',
+  module: 'advanced-scenarios'
+});
+
+// Multiple data sources
+const sftpSource = new SftpDataSource(
+  {
+    type: 'SFTP_CSV',
+    connectionId: 'warehouse-sftp',
+    name: 'Warehouse SFTP',
+    settings: {
+      host: process.env.WAREHOUSE_SFTP_HOST!,
+      username: process.env.WAREHOUSE_SFTP_USER!,
+      password: process.env.WAREHOUSE_SFTP_PASS!,
+      remotePath: '/inventory',
+      filePattern: '*.csv',
+    },
+  },
+  logger
+);
+
+const s3Source = new S3DataSource(
+  {
+    type: 'S3_JSON',
+    connectionId: 'store-s3',
+    name: 'Store Inventory S3',
+    settings: {
+      bucketName: 'store-inventory',
+      region: 'us-east-1',
+      accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
+      prefix: 'stores/',
+      filePattern: '*.json',
+    },
+  },
+  logger
+);
+
+const fluentClient = await createClient({
+  config: {
+    baseUrl: process.env.FLUENT_BASE_URL!,
+    clientId: process.env.FLUENT_CLIENT_ID!,
+    clientSecret: process.env.FLUENT_CLIENT_SECRET!,
+    retailerId: process.env.FLUENT_RETAILER_ID!,
+  },
+});
+
+async function aggregateInventory() {
+  logger.info('Starting multi-source inventory aggregation');
+
+  try {
+    // Collect inventory from all sources
+    const inventoryMap = new Map<
+      string,
+      {
+        sku: string;
+        locations: Map<string, number>;
+        sources: Set<string>;
+      }
+    >();
+
+    // 1. Process SFTP warehouse data
+    logger.info('Processing warehouse inventory from SFTP');
+    const sftpFiles = await sftpSource.listFiles();
+
+    for (const file of sftpFiles) {
+      const content = await sftpSource.downloadFile(file.name);
+      const parser = new CSVParserService();
+      const records = await parser.parse(content as string);
+
+      records.forEach(record => {
+        const key = record.sku;
+        if (!inventoryMap.has(key)) {
+          inventoryMap.set(key, {
+            sku: record.sku,
+            locations: new Map(),
+            sources: new Set(),
+          });
+        }
+
+        const item = inventoryMap.get(key)!;
+        const currentQty = item.locations.get(record.location) || 0;
+        item.locations.set(record.location, currentQty + parseInt(record.quantity));
+        item.sources.add('WAREHOUSE_SFTP');
+      });
+    }
+
+    // 2. Process S3 store data
+    logger.info('Processing store inventory from S3');
+    const s3Files = await s3Source.listFiles();
+
+    for (const file of s3Files) {
+      const content = await s3Source.downloadFile(file.name);
+      const storeData = JSON.parse(content.toString());
+
+      storeData.inventory.forEach((record: any) => {
+        const key = record.productId;
+        if (!inventoryMap.has(key)) {
+          inventoryMap.set(key, {
+            sku: record.productId,
+            locations: new Map(),
+            sources: new Set(),
+          });
+        }
+
+        const item = inventoryMap.get(key)!;
+        const currentQty = item.locations.get(record.storeId) || 0;
+        item.locations.set(record.storeId, currentQty + record.availableQty);
+        item.sources.add('STORE_S3');
+      });
+    }
+
+    // 3. Query current Fluent inventory for reconciliation
+    logger.info('Fetching current Fluent inventory');
+    const query = `
+      query GetCurrentInventory($retailerId: ID!, $first: Int!) {
+        inventoryPositions(retailerId: $retailerId, first: $first) {
+          edges {
+            node {
+              ref
+              productRef
+              locationRef
+              qty
+              availableQty
+            }
+            cursor
+          }
+          pageInfo { hasNextPage }
+        }
+      }
+    `;
+
+    const currentInventory = await fluentClient.graphql({
+      query,
+      variables: {
+        retailerId: process.env.FLUENT_RETAILER_ID!,
+        first: 1000,
+      },
+      pagination: { maxPages: 10 },
+    });
+
+    // 4. Reconcile and prepare updates
+    const updates: any[] = [];
+    const reconciliationReport: any[] = [];
+
+    inventoryMap.forEach((aggregated, sku) => {
+      aggregated.locations.forEach((newQty, location) => {
+        // Find current Fluent inventory
+        const current = currentInventory.data.inventoryPositions.edges.find(
+          (edge: any) => edge.node.productRef === sku && edge.node.locationRef === location
+        );
+
+        const currentQty = current?.node.qty || 0;
+        const difference = newQty - currentQty;
+
+        if (difference !== 0) {
+          updates.push({
+            ref: `${sku}-${location}`,
+            productRef: sku,
+            locationRef: location,
+            qty: newQty,
+            type: difference > 0 ? 'RECEIPT' : 'DISPATCH',
+            adjustmentQty: Math.abs(difference),
+            reason: 'MULTI_SOURCE_SYNC',
+            attributes: {
+              sources: Array.from(aggregated.sources).join(','),
+              previousQty: currentQty,
+              newQty: newQty,
+              syncedAt: new Date().toISOString(),
+            },
+          });
+
+          reconciliationReport.push({
+            sku,
+            location,
+            previousQty: currentQty,
+            newQty: newQty,
+            difference,
+            sources: Array.from(aggregated.sources),
+          });
+        }
+      });
+    });
+
+    logger.info(`Found ${updates.length} inventory differences to sync`);
+
+    // 5. Apply updates to Fluent
+    if (updates.length > 0) {
+      const job = await fluentClient.createJob({
+        name: `Multi-Source Inventory Sync - ${new Date().toISOString()}`,
+        retailerId: process.env.FLUENT_RETAILER_ID!,
+      });
+
+      // Process in batches
+      const batchSize = 100;
+      for (let i = 0; i < updates.length; i += batchSize) {
+        const batch = updates.slice(i, i + batchSize);
+
+        await fluentClient.sendBatch(job.id, {
+          action: 'UPSERT',
+          entityType: 'INVENTORY',
+          entities: batch,
+        });
+
+        logger.info(`Batch ${Math.floor(i / batchSize) + 1} processed`);
+      }
+
+      // Optional: monitor batch status via getBatchStatus
+    }
+
+    // 6. Generate and save reconciliation report
+    const report = {
+      timestamp: new Date().toISOString(),
+      sourcesProcessed: {
+        sftp: sftpFiles.length,
+        s3: s3Files.length,
+      },
+      totalSKUs: inventoryMap.size,
+      totalUpdates: updates.length,
+      reconciliation: reconciliationReport,
+    };
+
+    // Save report to S3
+    await s3Source.uploadFile(
+      `reports/reconciliation-${new Date().toISOString()}.json`,
+      JSON.stringify(report, null, 2)
+    );
+
+    logger.info('Multi-source inventory aggregation completed', {
+      totalSKUs: inventoryMap.size,
+      totalUpdates: updates.length,
+    });
+
+    return report;
+  } catch (error) {
+    logger.error('Aggregation failed:', error);
+    throw error;
+  }
+}
+
+// Schedule hourly
+setInterval(aggregateInventory, 60 * 60 * 1000);
+aggregateInventory();
+```
+
+### Key Concepts: Multi-Source Aggregation
+
+**Data Collection Pattern:**
+
+```typescript
+// Use Map for efficient lookups and aggregation
+const inventoryMap = new Map<
+  string,
+  {
+    sku: string;
+    locations: Map<string, number>;
+    sources: Set<string>;
+  }
+>();
+
+// Aggregate from multiple sources
+sources.forEach(source => {
+  source.records.forEach(record => {
+    const item = inventoryMap.get(record.sku) || createNewItem();
+    item.locations.set(record.location, record.qty);
+    item.sources.add(source.name);
+  });
+});
+```
+
+**Reconciliation Pattern:**
+
+```typescript
+// Compare aggregated vs current
+const difference = newQty - currentQty;
+
+if (difference !== 0) {
+  updates.push({
+    qty: newQty,
+    type: difference > 0 ? 'RECEIPT' : 'DISPATCH',
+    adjustmentQty: Math.abs(difference),
+  });
+}
+```
+
+## Key Takeaways
+
+- 🎯 Stream large files to avoid memory issues (don't load all at once)
+- 🎯 Save checkpoints frequently to enable recovery from failures
+- 🎯 Multi-source aggregation requires careful reconciliation logic
+- 🎯 Always track differences, not just totals (delta updates)
+- 🎯 State management is critical for production reliability
+- 🎯 Generate reports for visibility into sync operations
+
+## Next Steps
+
+Continue to [Module 5: Bidirectional Sync](./connector-scenarios-05-bidirectional-sync.md) to learn extraction patterns and Fluent-to-external system exports.