npm - @fluentcommerce/fc-connect-sdk - Versions diffs - 0.1.53 → 0.1.55 - Mend

@fluentcommerce/fc-connect-sdk 0.1.53 → 0.1.55

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (495) hide show

package/docs/02-CORE-GUIDES/data-sources/modules/data-sources-04-file-patterns.md CHANGED Viewed

@@ -1,941 +1,941 @@
-# Module 4: File Processing Patterns
-Master file processing strategies - streaming vs buffering, batch processing, state management, validation, and error recovery.
-## Table of Contents
-- [Streaming vs Buffering](#streaming-vs-buffering)
-- [Batch Processing](#batch-processing)
-- [State Management](#state-management)
-- [Format-Specific Processing](#format-specific-processing)
-- [Validation Strategies](#validation-strategies)
-- [Error Recovery](#error-recovery)
-- [Performance Optimization](#performance-optimization)
-- [Production Patterns](#production-patterns)
----
-## Streaming vs Buffering
-### When to Use Each Strategy
-| Strategy              | Best For                        | File Size | Memory Usage                 | Processing Speed |
-| --------------------- | ------------------------------- | --------- | ---------------------------- | ---------------- |
-| **Buffering**         | Small files, simple processing  | < 10MB    | High (entire file in memory) | Fastest          |
-| **Streaming**         | Large files, memory-constrained | > 10MB    | Low (chunks processed)       | Moderate         |
-| **Chunked Streaming** | Very large files                | > 100MB   | Very low (fixed chunk size)  | Slowest          |
-### Buffering Pattern (Default)
-```typescript
-// Load entire file into memory (simple, fast for small files)
-const csvContent = await s3Source.downloadFile('inventory.csv');
-// content is entire file in memory (string or Buffer)
-const records = await csvParser.parse(csvContent);
-// Process all records at once
-for (const record of records) {
-  await processRecord(record);
-}
-```
-**Pros**:
-- ✅ Simple API
-- ✅ Fastest for small files (<10MB)
-- ✅ Easy to debug (full content available)
-**Cons**:
-- ❌ High memory usage for large files
-- ❌ Can cause OOM (Out of Memory) errors
-- ❌ Slow initial load for large files
-### Chunked Processing Pattern
-```typescript
-// Process large file in chunks to minimize memory usage
-async function processLargeFile(dataSource: S3DataSource, key: string) {
-  // Download file
-  const content = await dataSource.downloadFile(key, { encoding: 'binary' });
-  // Process in 1MB chunks
-  const chunkSize = 1024 * 1024; // 1MB
-  let offset = 0;
-  while (offset < (content as Buffer).length) {
-    const chunk = (content as Buffer).slice(offset, offset + chunkSize);
-    // Process chunk
-    await processChunk(chunk);
-    offset += chunkSize;
-    // Optional: Log progress
-    const progress = ((offset / (content as Buffer).length) * 100).toFixed(2);
-    logger.info(`Processing... ${progress}%`);
-  }
-}
-async function processChunk(chunk: Buffer) {
-  // Your chunk processing logic
-  const chunkRecords = parseChunk(chunk);
-  await sendToFluent(chunkRecords);
-}
-```
-**Pros**:
-- ✅ Lower memory usage (fixed chunk size)
-- ✅ Progress tracking
-- ✅ Can handle larger files (up to 100MB)
-**Cons**:
-- ❌ More complex code
-- ❌ Still loads full file initially
-- ❌ Not suitable for very large files (GB+)
-### CSV Streaming Pattern
-```typescript
-import { CSVParserService } from '@fluentcommerce/fc-connect-sdk';
-import { Readable } from 'stream';
-// Stream large CSV file line-by-line
-async function streamLargeCSV(dataSource: S3DataSource, key: string) {
-  const csvContent = await dataSource.downloadFile(key);
-  const csvParser = new CSVParserService();
-  // Parse CSV (returns array of records)
-  const records = await csvParser.parse(csvContent as string);
-  // Process in batches
-  const batchSize = 100;
-  for (let i = 0; i < records.length; i += batchSize) {
-    const batch = records.slice(i, i + batchSize);
-    await processBatch(batch);
-    // Optional: Add delay between batches to avoid rate limiting
-    await sleep(100);
-  }
-}
-function sleep(ms: number): Promise<void> {
-  return new Promise(resolve => setTimeout(resolve, ms));
-}
-```
-**Pros**:
-- ✅ Batch processing for better throughput
-- ✅ Rate limiting control
-- ✅ Suitable for medium-large files
-**Decision Matrix**:
-```
-File Size Decision:
-< 10MB    → Buffer entire file (simplest)
-10-100MB  → Chunked processing (balanced)
-> 100MB   → Stream with batching (lowest memory)
-```
----
-## Batch Processing
-### Batch Size Optimization
-```typescript
-// Find optimal batch size based on file size and record count
-function calculateOptimalBatchSize(totalRecords: number): number {
-  if (totalRecords < 100) return totalRecords; // Small: 1 batch
-  if (totalRecords < 1000) return 100; // Medium: 100 per batch
-  if (totalRecords < 10000) return 250; // Large: 250 per batch
-  return 500; // Very large: 500 per batch
-}
-// Process records in optimal batches
-async function processInBatches(records: any[], client: FluentClient) {
-  const batchSize = calculateOptimalBatchSize(records.length);
-  logger.info('Starting batch processing', {
-    totalRecords: records.length,
-    batchSize,
-    estimatedBatches: Math.ceil(records.length / batchSize),
-  });
-  for (let i = 0; i < records.length; i += batchSize) {
-    const batch = records.slice(i, i + batchSize);
-    const batchNumber = Math.floor(i / batchSize) + 1;
-    try {
-      await client.sendBatch(jobId, { entities: batch });
-      logger.info(`Batch ${batchNumber} sent successfully`, {
-        recordCount: batch.length,
-      });
-    } catch (error) {
-      logger.error(`Batch ${batchNumber} failed`, error as Error, {
-        recordCount: batch.length,
-        startIndex: i,
-      });
-      // Continue with next batch (or throw based on strategy)
-    }
-  }
-}
-```
-### Parallel Batch Processing
-```typescript
-// Process multiple files in parallel with concurrency limit
-async function processMultipleFiles(
-  dataSource: DataSource,
-  files: string[],
-  maxConcurrency: number = 5
-) {
-  const results = [];
-  // Process files with concurrency control
-  for (let i = 0; i < files.length; i += maxConcurrency) {
-    const batch = files.slice(i, i + maxConcurrency);
-    // Process batch in parallel
-    const batchResults = await Promise.allSettled(batch.map(file => processFile(dataSource, file)));
-    results.push(...batchResults);
-    // Log progress
-    const processed = i + batch.length;
-    const total = files.length;
-    logger.info(`Progress: ${processed}/${total} files processed`);
-  }
-  // Analyze results
-  const successful = results.filter(r => r.status === 'fulfilled').length;
-  const failed = results.filter(r => r.status === 'rejected').length;
-  logger.info('Batch processing complete', { successful, failed, total: files.length });
-  return results;
-}
-async function processFile(dataSource: any, file: string) {
-  const content = await dataSource.downloadFile(file);
-  const records = parseContent(content);
-  await sendToFluent(records);
-  return { file, recordCount: records.length };
-}
-```
-**Concurrency Guidelines**:
-- Local processing: 10-20 concurrent files
-- Cloud serverless (Lambda, Versori): 3-5 concurrent files
-- SFTP servers: 1-3 concurrent connections (avoid overwhelming vendor)
-- S3: 10+ (S3 scales automatically)
----
-## State Management
-### Preventing Duplicate Processing
-```typescript
-import { StateService, VersoriKVAdapter } from '@fluentcommerce/fc-connect-sdk';
-async function processWithStateManagement(dataSource: DataSource, stateService: StateService) {
-  const files = await dataSource.listFiles({ prefix: 'inventory/' });
-  for (const file of files) {
-    // Create unique state key (file path + last modified timestamp)
-    const stateKey = `processed:${file.path}:${file.lastModified.toISOString()}`;
-    // Check if already processed
-    const processed = await stateService.isFileProcessed(kvAdapter as any, stateKey);
-    if (processed) {
-      logger.info('File already processed, skipping', {
-        file: file.name,
-        lastModified: file.lastModified,
-      });
-      continue;
-    }
-    try {
-      // Process file
-      const content = await dataSource.downloadFile(file.path);
-      const records = parseContent(content);
-      await sendToFluent(records);
-      // Mark as processed
-      await stateService.updateSyncState(
-        kvAdapter as any,
-        [
-          {
-            fileName: file.name,
-            filePath: file.path,
-            fileSize: file.size,
-            recordCount: records.length,
-            processedAt: new Date().toISOString(),
-            success: true,
-          },
-        ],
-        'file-processing'
-      );
-      logger.info('File processed successfully', { file: file.name });
-    } catch (error) {
-      logger.error('Failed to process file', error as Error, { file: file.path });
-      // Optional: Mark as failed (for retry logic)
-      await stateService.updateSyncState(
-        kvAdapter as any,
-        [
-          {
-            fileName: file.name,
-            processedAt: new Date().toISOString(),
-            success: false,
-            error: (error as Error).message,
-          },
-        ],
-        'file-processing'
-      );
-    }
-  }
-}
-```
-### State Key Strategies
-```typescript
-// Strategy 1: File path + modification time (recommended)
-const stateKey = `processed:${file.path}:${file.lastModified.toISOString()}`;
-// Pros: Handles file updates (new lastModified = new processing)
-// Cons: Same file content re-processed if touched
-// Strategy 2: File ETag (S3 only)
-const stateKey = `processed:${file.etag}`;
-// Pros: Content-based (same content = same ETag)
-// Cons: Only works with S3
-// Strategy 3: File name only (simple)
-const stateKey = `processed:${file.name}`;
-// Pros: Simplest
-// Cons: Doesn't handle file updates
-// Strategy 4: Content hash (most accurate)
-const contentHash = crypto.createHash('md5').update(content).digest('hex');
-const stateKey = `processed:${contentHash}`;
-// Pros: Truly content-based
-// Cons: Requires downloading file first (slow)
-```
-### Versori KV State Management
-```typescript
-import { VersoriKVAdapter, StateService } from '@fluentcommerce/fc-connect-sdk';
-// ✅ CORRECT: Access openKv from Versori context
-// import { openKv } from '@versori/run'; // ❌ WRONG - Not a direct export
-// In Versori workflow handler:
-const { openKv } = ctx;
-const kvStore = openKv(':project:');
-const kvAdapter = new VersoriKVAdapter(kvStore);
-const stateService = new StateService(kvAdapter);
-// Use in Versori webhook
-export const processInventory = webhook('inventory-sync', async (ctx) => {
-  const client = await createClient(ctx); // Auto-detects Versori context
-  const s3Source = new S3DataSource(
-    {
-      /* config */
-    },
-    log
-  );
-  const files = await s3Source.listFiles({ prefix: 'inventory/' });
-  for (const file of files) {
-    const stateKey = `processed:${file.etag}`;
-    if (await stateService.isFileProcessed(kvAdapter as any, stateKey)) {
-      log.info('File already processed', { file: file.name });
-      continue;
-    }
-    // Process file
-    const content = await s3Source.downloadFile(file.path);
-    await processContent(content, client);
-    // Mark as processed (persisted in Versori KV)
-    await stateService.updateSyncState(
-      kvAdapter as any,
-      [
-        {
-          fileName: file.name,
-          processedAt: new Date().toISOString(),
-        },
-      ],
-      'file-processing'
-    );
-  }
-});
-```
----
-## Format-Specific Processing
-### CSV Processing
-```typescript
-import { CSVParserService } from '@fluentcommerce/fc-connect-sdk';
-async function processCSV(dataSource: DataSource, file: string) {
-  const csvParser = new CSVParserService();
-  // Download CSV
-  const content = await dataSource.downloadFile(file);
-  // Parse with validation
-  const records = await csvParser.parse(content as string, {
-    columns: true, // Use first row as headers
-    skip_empty_lines: true, // Skip empty rows
-    trim: true, // Trim whitespace
-    cast: true, // Auto-cast types (numbers, booleans)
-    cast_date: false, // Don't auto-cast dates (handle manually)
-    relax_column_count: false, // Enforce consistent column count
-    on_record: (record, context) => {
-      // Validate each record during parsing
-      if (!record.sku || !record.quantity) {
-        throw new Error(`Missing required fields at line ${context.lines}`);
-      }
-      return record;
-    },
-  });
-  logger.info('CSV parsed successfully', { recordCount: records.length });
-  return records;
-}
-```
-### JSON Processing
-```typescript
-async function processJSON(dataSource: DataSource, file: string) {
-  const content = await dataSource.downloadFile(file);
-  try {
-    const data = JSON.parse(content as string);
-    // Handle different JSON structures
-    let records: any[];
-    if (Array.isArray(data)) {
-      // Array of records
-      records = data;
-    } else if (data.records && Array.isArray(data.records)) {
-      // Wrapped array: { "records": [...] }
-      records = data.records;
-    } else if (data.data && Array.isArray(data.data)) {
-      // Wrapped array: { "data": [...] }
-      records = data.data;
-    } else {
-      // Single record
-      records = [data];
-    }
-    logger.info('JSON parsed successfully', { recordCount: records.length });
-    return records;
-  } catch (error) {
-    throw new Error(`Invalid JSON in file ${file}: ${(error as Error).message}`);
-  }
-}
-```
-### JSON Lines (JSONL) Processing
-```typescript
-async function processJSONL(dataSource: DataSource, file: string) {
-  const content = await dataSource.downloadFile(file);
-  const lines = (content as string).split('\n').filter(line => line.trim());
-  const records = [];
-  const errors = [];
-  for (let i = 0; i < lines.length; i++) {
-    try {
-      const record = JSON.parse(lines[i]);
-      records.push(record);
-    } catch (error) {
-      errors.push({
-        line: i + 1,
-        content: lines[i],
-        error: (error as Error).message,
-      });
-    }
-  }
-  if (errors.length > 0) {
-    logger.warn('JSONL parsing errors', { errorCount: errors.length, errors });
-  }
-  logger.info('JSONL parsed successfully', {
-    totalLines: lines.length,
-    recordCount: records.length,
-    errorCount: errors.length,
-  });
-  return records;
-}
-```
-### Parquet Processing
-```typescript
-import { ParquetParserService } from '@fluentcommerce/fc-connect-sdk';
-async function processParquet(dataSource: DataSource, file: string) {
-  const parquetParser = new ParquetParserService(logger);
-  // Download Parquet file
-  const buffer = await dataSource.downloadFile(file, { encoding: 'binary' });
-  // Parse Parquet
-  const records = await parquetParser.parse(buffer as Buffer);
-  logger.info('Parquet parsed successfully', {
-    recordCount: records.length,
-    fileSize: (buffer as Buffer).length,
-  });
-  return records;
-}
-```
----
-## Validation Strategies
-### Pre-Processing Validation
-```typescript
-async function validateBeforeProcessing(file: FileMetadata): Promise<boolean> {
-  const validationRules = [
-    // Rule 1: File size check
-    () => {
-      const maxSize = 100 * 1024 * 1024; // 100MB
-      if (file.size > maxSize) {
-        logger.error('File too large', { size: file.size, maxSize });
-        return false;
-      }
-      return true;
-    },
-    // Rule 2: File name pattern check
-    () => {
-      const pattern = /^inventory_\d{8}\.csv$/;
-      if (!pattern.test(file.name)) {
-        logger.error('Invalid file name pattern', { name: file.name, expected: pattern });
-        return false;
-      }
-      return true;
-    },
-    // Rule 3: File age check (not too old)
-    () => {
-      const maxAge = 7 * 24 * 60 * 60 * 1000; // 7 days
-      const age = Date.now() - file.lastModified.getTime();
-      if (age > maxAge) {
-        logger.warn('File too old', {
-          name: file.name,
-          age: Math.floor(age / (24 * 60 * 60 * 1000)) + ' days',
-        });
-        return false;
-      }
-      return true;
-    },
-  ];
-  // Run all validation rules
-  for (const rule of validationRules) {
-    if (!rule()) {
-      return false;
-    }
-  }
-  return true;
-}
-// Use in processing pipeline
-async function processWithValidation(dataSource: DataSource) {
-  const files = await dataSource.listFiles();
-  for (const file of files) {
-    // Validate before downloading
-    if (!(await validateBeforeProcessing(file))) {
-      logger.warn('File failed validation, skipping', { file: file.name });
-      continue;
-    }
-    // Process validated file
-    await processFile(dataSource, file.path);
-  }
-}
-```
-### Record-Level Validation
-```typescript
-interface ValidationRule {
-  field: string;
-  rule: (value: any) => boolean;
-  message: string;
-}
-const validationRules: ValidationRule[] = [
-  {
-    field: 'sku',
-    rule: value => typeof value === 'string' && value.length > 0,
-    message: 'SKU must be non-empty string',
-  },
-  {
-    field: 'quantity',
-    rule: value => typeof value === 'number' && value >= 0,
-    message: 'Quantity must be non-negative number',
-  },
-  {
-    field: 'price',
-    rule: value => typeof value === 'number' && value > 0,
-    message: 'Price must be positive number',
-  },
-];
-function validateRecord(record: any, index: number): { valid: boolean; errors: string[] } {
-  const errors: string[] = [];
-  for (const rule of validationRules) {
-    if (!rule.rule(record[rule.field])) {
-      errors.push(`Record ${index}: ${rule.message} (value: ${record[rule.field]})`);
-    }
-  }
-  return { valid: errors.length === 0, errors };
-}
-// Use in processing
-async function processWithRecordValidation(records: any[]) {
-  const validRecords = [];
-  const invalidRecords = [];
-  for (let i = 0; i < records.length; i++) {
-    const validation = validateRecord(records[i], i + 1);
-    if (validation.valid) {
-      validRecords.push(records[i]);
-    } else {
-      invalidRecords.push({ record: records[i], errors: validation.errors });
-      logger.warn('Invalid record', { index: i + 1, errors: validation.errors });
-    }
-  }
-  logger.info('Validation complete', {
-    total: records.length,
-    valid: validRecords.length,
-    invalid: invalidRecords.length,
-  });
-  // Process only valid records
-  if (validRecords.length > 0) {
-    await sendToFluent(validRecords);
-  }
-  // Report invalid records
-  if (invalidRecords.length > 0) {
-    await reportInvalidRecords(invalidRecords);
-  }
-}
-```
----
-## Error Recovery
-> **For comprehensive error handling patterns:** See [File Operations Error Handling Guide](../data-sources-file-operations-error-handling.md) for detailed coverage of connection errors, discovery errors, reading errors, parsing errors, troubleshooting flowcharts, and production checklists for S3 and SFTP operations.
-### Retry Strategy
-```typescript
-async function processWithRetry<T>(
-  fn: () => Promise<T>,
-  options = {
-    maxRetries: 3,
-    initialDelay: 1000,
-    maxDelay: 30000,
-    backoffMultiplier: 2,
-  }
-): Promise<T> {
-  let lastError: Error;
-  for (let attempt = 0; attempt <= options.maxRetries; attempt++) {
-    try {
-      return await fn();
-    } catch (error: any) {
-      lastError = error;
-      // Don't retry client errors (4xx)
-      if (error.statusCode && error.statusCode >= 400 && error.statusCode < 500) {
-        throw error;
-      }
-      if (attempt < options.maxRetries) {
-        const delay = Math.min(
-          options.initialDelay * Math.pow(options.backoffMultiplier, attempt),
-          options.maxDelay
-        );
-        logger.warn(`Attempt ${attempt + 1} failed, retrying in ${delay}ms`, {
-          error: error.message,
-          retriesLeft: options.maxRetries - attempt,
-        });
-        await sleep(delay);
-      }
-    }
-  }
-  throw new Error(
-    `Operation failed after ${options.maxRetries + 1} attempts: ${lastError!.message}`
-  );
-}
-// Use in file processing
-async function processFileWithRetry(dataSource: DataSource, file: string) {
-  return processWithRetry(async () => {
-    const content = await dataSource.downloadFile(file);
-    const records = parseContent(content);
-    await sendToFluent(records);
-    return records.length;
-  });
-}
-```
-### Partial Failure Handling
-```typescript
-async function processBatchWithPartialFailure(
-  records: any[],
-  client: FluentClient,
-  batchSize: number = 100
-) {
-  const results = {
-    successful: 0,
-    failed: 0,
-    errors: [] as Array<{ batch: number; error: string; records: any[] }>,
-  };
-  for (let i = 0; i < records.length; i += batchSize) {
-    const batch = records.slice(i, i + batchSize);
-    const batchNumber = Math.floor(i / batchSize) + 1;
-    try {
-      await client.sendBatch(jobId, { entities: batch });
-      results.successful += batch.length;
-      logger.info(`Batch ${batchNumber} succeeded`, { recordCount: batch.length });
-    } catch (error) {
-      results.failed += batch.length;
-      results.errors.push({
-        batch: batchNumber,
-        error: (error as Error).message,
-        records: batch,
-      });
-      logger.error(`Batch ${batchNumber} failed`, error as Error, {
-        recordCount: batch.length,
-        startIndex: i,
-      });
-      // Continue processing remaining batches
-    }
-  }
-  logger.info('Batch processing complete', results);
-  // Save failed batches for manual review or retry
-  if (results.errors.length > 0) {
-    await saveFailedBatches(results.errors);
-  }
-  return results;
-}
-async function saveFailedBatches(errors: any[]) {
-  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
-  const failedRecords = errors.flatMap(e => e.records);
-  await s3Target.writeFile(
-    `s3://failed-batches/failed-${timestamp}.json`,
-    JSON.stringify({ errors, records: failedRecords }, null, 2)
-  );
-}
-```
----
-## Performance Optimization
-### Optimization Checklist
-```typescript
-// 1. Use appropriate batch sizes
-const optimalBatchSize = calculateOptimalBatchSize(records.length);
-// 2. Parallel processing with concurrency limit
-const maxConcurrency = 5;
-await processFilesInParallel(files, maxConcurrency);
-// 3. Minimize data source round trips
-const files = await dataSource.listFiles({ prefix: 'inventory/' });
-// Download all file metadata in one call
-// 4. Use streaming for large files
-if (fileSize > 10 * 1024 * 1024) {
-  await processWithStreaming(file);
-} else {
-  await processBuffered(file);
-}
-// 5. Cache parsed results
-const parseCache = new Map<string, any[]>();
-if (parseCache.has(file.path)) {
-  return parseCache.get(file.path);
-}
-// 6. Efficient state lookups
-const processedFiles = new Set(await stateService.getProcessedFiles());
-for (const file of files) {
-  if (processedFiles.has(file.path)) continue;
-  // Process file
-}
-// 7. Avoid redundant validations
-// Validate once before processing, not on every record
-```
----
-## Production Patterns
-### Complete Ingestion Pipeline
-```typescript
-import {
-  S3DataSource,
-  CSVParserService,
-  UniversalMapper,
-  StateService,
-  VersoriKVAdapter,
-  createClient,
-} from '@fluentcommerce/fc-connect-sdk';
-async function productionIngestionPipeline(ctx: any) {
-  // 1. Initialize services
-  const { log, openKv } = ctx;
-  const client = await createClient(ctx); // Auto-detects Versori context
-  const s3Source = new S3DataSource(
-    {
-      /* config */
-    },
-    log
-  );
-  const csvParser = new CSVParserService();
-  const mapper = new UniversalMapper(mappingConfig);
-  const stateService = new StateService(new VersoriKVAdapter(openKv(':project:')));
-  // 2. List files
-  const files = await s3Source.listFiles({ prefix: 'inventory/' });
-  log.info('Files found', { count: files.length });
-  // 3. Filter and validate
-  const validFiles = files.filter(file => {
-    // Size check
-    if (file.size > 100 * 1024 * 1024) {
-      log.warn('File too large, skipping', { file: file.name, size: file.size });
-      return false;
-    }
-    // Name pattern check
-    if (!file.name.endsWith('.csv')) {
-      return false;
-    }
-    return true;
-  });
-  // 4. Process files
-  for (const file of validFiles) {
-    const stateKey = `processed:${file.etag}`;
-    // Check state
-    if (await stateService.isFileProcessed(kvAdapter as any, stateKey)) {
-      log.info('File already processed', { file: file.name });
-      continue;
-    }
-    try {
-      // Download and parse
-      const content = await s3Source.downloadFile(file.path);
-      const records = await csvParser.parse(content as string);
-      // Map fields
-      const mappingResult = await mapper.map(records);
-      if (!mappingResult.success) {
-        throw new Error(`Mapping failed: ${JSON.stringify(mappingResult.errors)}`);
-      }
-      // Send to Fluent with retry
-      await processWithRetry(async () => {
-        await client.sendBatch(jobId, { entities: mappingResult.data });
-      });
-      // Mark as processed
-      await stateService.updateSyncState(
-        kvAdapter as any,
-        [
-          {
-            fileName: file.name,
-            recordCount: records.length,
-            processedAt: new Date().toISOString(),
-          },
-        ],
-        'file-processing'
-      );
-      // Archive file
-      await s3Source.moveFile(file.path, file.path.replace('inventory/', 'processed/'));
-      log.info('File processed successfully', { file: file.name, records: records.length });
-    } catch (error) {
-      log.error('Failed to process file', { file: file.path, error: (error as Error).message });
-      // Continue with next file
-    }
-  }
-}
-```
----
-## Next Steps
-**Explore advanced topics** → [Module 5: Advanced Topics](./data-sources-05-advanced-topics.md)
-**Quick reference** → [Quick Reference](../data-sources-quick-reference.md)
----
-[← Back to SFTP Operations](./data-sources-03-sftp-operations.md) | [Next: Advanced Topics →](./data-sources-05-advanced-topics.md) | [↑ Back to Guide](../data-sources-readme.md)
+# Module 4: File Processing Patterns
+Master file processing strategies - streaming vs buffering, batch processing, state management, validation, and error recovery.
+## Table of Contents
+- [Streaming vs Buffering](#streaming-vs-buffering)
+- [Batch Processing](#batch-processing)
+- [State Management](#state-management)
+- [Format-Specific Processing](#format-specific-processing)
+- [Validation Strategies](#validation-strategies)
+- [Error Recovery](#error-recovery)
+- [Performance Optimization](#performance-optimization)
+- [Production Patterns](#production-patterns)
+---
+## Streaming vs Buffering
+### When to Use Each Strategy
+| Strategy              | Best For                        | File Size | Memory Usage                 | Processing Speed |
+| --------------------- | ------------------------------- | --------- | ---------------------------- | ---------------- |
+| **Buffering**         | Small files, simple processing  | < 10MB    | High (entire file in memory) | Fastest          |
+| **Streaming**         | Large files, memory-constrained | > 10MB    | Low (chunks processed)       | Moderate         |
+| **Chunked Streaming** | Very large files                | > 100MB   | Very low (fixed chunk size)  | Slowest          |
+### Buffering Pattern (Default)
+```typescript
+// Load entire file into memory (simple, fast for small files)
+const csvContent = await s3Source.downloadFile('inventory.csv');
+// content is entire file in memory (string or Buffer)
+const records = await csvParser.parse(csvContent);
+// Process all records at once
+for (const record of records) {
+  await processRecord(record);
+}
+```
+**Pros**:
+- ✅ Simple API
+- ✅ Fastest for small files (<10MB)
+- ✅ Easy to debug (full content available)
+**Cons**:
+- ❌ High memory usage for large files
+- ❌ Can cause OOM (Out of Memory) errors
+- ❌ Slow initial load for large files
+### Chunked Processing Pattern
+```typescript
+// Process large file in chunks to minimize memory usage
+async function processLargeFile(dataSource: S3DataSource, key: string) {
+  // Download file
+  const content = await dataSource.downloadFile(key, { encoding: 'binary' });
+  // Process in 1MB chunks
+  const chunkSize = 1024 * 1024; // 1MB
+  let offset = 0;
+  while (offset < (content as Buffer).length) {
+    const chunk = (content as Buffer).slice(offset, offset + chunkSize);
+    // Process chunk
+    await processChunk(chunk);
+    offset += chunkSize;
+    // Optional: Log progress
+    const progress = ((offset / (content as Buffer).length) * 100).toFixed(2);
+    logger.info(`Processing... ${progress}%`);
+  }
+}
+async function processChunk(chunk: Buffer) {
+  // Your chunk processing logic
+  const chunkRecords = parseChunk(chunk);
+  await sendToFluent(chunkRecords);
+}
+```
+**Pros**:
+- ✅ Lower memory usage (fixed chunk size)
+- ✅ Progress tracking
+- ✅ Can handle larger files (up to 100MB)
+**Cons**:
+- ❌ More complex code
+- ❌ Still loads full file initially
+- ❌ Not suitable for very large files (GB+)
+### CSV Streaming Pattern
+```typescript
+import { CSVParserService } from '@fluentcommerce/fc-connect-sdk';
+import { Readable } from 'stream';
+// Stream large CSV file line-by-line
+async function streamLargeCSV(dataSource: S3DataSource, key: string) {
+  const csvContent = await dataSource.downloadFile(key);
+  const csvParser = new CSVParserService();
+  // Parse CSV (returns array of records)
+  const records = await csvParser.parse(csvContent as string);
+  // Process in batches
+  const batchSize = 100;
+  for (let i = 0; i < records.length; i += batchSize) {
+    const batch = records.slice(i, i + batchSize);
+    await processBatch(batch);
+    // Optional: Add delay between batches to avoid rate limiting
+    await sleep(100);
+  }
+}
+function sleep(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+```
+**Pros**:
+- ✅ Batch processing for better throughput
+- ✅ Rate limiting control
+- ✅ Suitable for medium-large files
+**Decision Matrix**:
+```
+File Size Decision:
+< 10MB    → Buffer entire file (simplest)
+10-100MB  → Chunked processing (balanced)
+> 100MB   → Stream with batching (lowest memory)
+```
+---
+## Batch Processing
+### Batch Size Optimization
+```typescript
+// Find optimal batch size based on file size and record count
+function calculateOptimalBatchSize(totalRecords: number): number {
+  if (totalRecords < 100) return totalRecords; // Small: 1 batch
+  if (totalRecords < 1000) return 100; // Medium: 100 per batch
+  if (totalRecords < 10000) return 250; // Large: 250 per batch
+  return 500; // Very large: 500 per batch
+}
+// Process records in optimal batches
+async function processInBatches(records: any[], client: FluentClient) {
+  const batchSize = calculateOptimalBatchSize(records.length);
+  logger.info('Starting batch processing', {
+    totalRecords: records.length,
+    batchSize,
+    estimatedBatches: Math.ceil(records.length / batchSize),
+  });
+  for (let i = 0; i < records.length; i += batchSize) {
+    const batch = records.slice(i, i + batchSize);
+    const batchNumber = Math.floor(i / batchSize) + 1;
+    try {
+      await client.sendBatch(jobId, { entities: batch });
+      logger.info(`Batch ${batchNumber} sent successfully`, {
+        recordCount: batch.length,
+      });
+    } catch (error) {
+      logger.error(`Batch ${batchNumber} failed`, error as Error, {
+        recordCount: batch.length,
+        startIndex: i,
+      });
+      // Continue with next batch (or throw based on strategy)
+    }
+  }
+}
+```
+### Parallel Batch Processing
+```typescript
+// Process multiple files in parallel with concurrency limit
+async function processMultipleFiles(
+  dataSource: DataSource,
+  files: string[],
+  maxConcurrency: number = 5
+) {
+  const results = [];
+  // Process files with concurrency control
+  for (let i = 0; i < files.length; i += maxConcurrency) {
+    const batch = files.slice(i, i + maxConcurrency);
+    // Process batch in parallel
+    const batchResults = await Promise.allSettled(batch.map(file => processFile(dataSource, file)));
+    results.push(...batchResults);
+    // Log progress
+    const processed = i + batch.length;
+    const total = files.length;
+    logger.info(`Progress: ${processed}/${total} files processed`);
+  }
+  // Analyze results
+  const successful = results.filter(r => r.status === 'fulfilled').length;
+  const failed = results.filter(r => r.status === 'rejected').length;
+  logger.info('Batch processing complete', { successful, failed, total: files.length });
+  return results;
+}
+async function processFile(dataSource: any, file: string) {
+  const content = await dataSource.downloadFile(file);
+  const records = parseContent(content);
+  await sendToFluent(records);
+  return { file, recordCount: records.length };
+}
+```
+**Concurrency Guidelines**:
+- Local processing: 10-20 concurrent files
+- Cloud serverless (Lambda, Versori): 3-5 concurrent files
+- SFTP servers: 1-3 concurrent connections (avoid overwhelming vendor)
+- S3: 10+ (S3 scales automatically)
+---
+## State Management
+### Preventing Duplicate Processing
+```typescript
+import { StateService, VersoriKVAdapter } from '@fluentcommerce/fc-connect-sdk';
+async function processWithStateManagement(dataSource: DataSource, stateService: StateService) {
+  const files = await dataSource.listFiles({ prefix: 'inventory/' });
+  for (const file of files) {
+    // Create unique state key (file path + last modified timestamp)
+    const stateKey = `processed:${file.path}:${file.lastModified.toISOString()}`;
+    // Check if already processed
+    const processed = await stateService.isFileProcessed(kvAdapter as any, stateKey);
+    if (processed) {
+      logger.info('File already processed, skipping', {
+        file: file.name,
+        lastModified: file.lastModified,
+      });
+      continue;
+    }
+    try {
+      // Process file
+      const content = await dataSource.downloadFile(file.path);
+      const records = parseContent(content);
+      await sendToFluent(records);
+      // Mark as processed
+      await stateService.updateSyncState(
+        kvAdapter as any,
+        [
+          {
+            fileName: file.name,
+            filePath: file.path,
+            fileSize: file.size,
+            recordCount: records.length,
+            processedAt: new Date().toISOString(),
+            success: true,
+          },
+        ],
+        'file-processing'
+      );
+      logger.info('File processed successfully', { file: file.name });
+    } catch (error) {
+      logger.error('Failed to process file', error as Error, { file: file.path });
+      // Optional: Mark as failed (for retry logic)
+      await stateService.updateSyncState(
+        kvAdapter as any,
+        [
+          {
+            fileName: file.name,
+            processedAt: new Date().toISOString(),
+            success: false,
+            error: (error as Error).message,
+          },
+        ],
+        'file-processing'
+      );
+    }
+  }
+}
+```
+### State Key Strategies
+```typescript
+// Strategy 1: File path + modification time (recommended)
+const stateKey = `processed:${file.path}:${file.lastModified.toISOString()}`;
+// Pros: Handles file updates (new lastModified = new processing)
+// Cons: Same file content re-processed if touched
+// Strategy 2: File ETag (S3 only)
+const stateKey = `processed:${file.etag}`;
+// Pros: Content-based (same content = same ETag)
+// Cons: Only works with S3
+// Strategy 3: File name only (simple)
+const stateKey = `processed:${file.name}`;
+// Pros: Simplest
+// Cons: Doesn't handle file updates
+// Strategy 4: Content hash (most accurate)
+const contentHash = crypto.createHash('md5').update(content).digest('hex');
+const stateKey = `processed:${contentHash}`;
+// Pros: Truly content-based
+// Cons: Requires downloading file first (slow)
+```
+### Versori KV State Management
+```typescript
+import { VersoriKVAdapter, StateService } from '@fluentcommerce/fc-connect-sdk';
+// ✅ CORRECT: Access openKv from Versori context
+// import { openKv } from '@versori/run'; // ❌ WRONG - Not a direct export
+// In Versori workflow handler:
+const { openKv } = ctx;
+const kvStore = openKv(':project:');
+const kvAdapter = new VersoriKVAdapter(kvStore);
+const stateService = new StateService(kvAdapter);
+// Use in Versori webhook
+export const processInventory = webhook('inventory-sync', async (ctx) => {
+  const client = await createClient(ctx); // Auto-detects Versori context
+  const s3Source = new S3DataSource(
+    {
+      /* config */
+    },
+    log
+  );
+  const files = await s3Source.listFiles({ prefix: 'inventory/' });
+  for (const file of files) {
+    const stateKey = `processed:${file.etag}`;
+    if (await stateService.isFileProcessed(kvAdapter as any, stateKey)) {
+      log.info('File already processed', { file: file.name });
+      continue;
+    }
+    // Process file
+    const content = await s3Source.downloadFile(file.path);
+    await processContent(content, client);
+    // Mark as processed (persisted in Versori KV)
+    await stateService.updateSyncState(
+      kvAdapter as any,
+      [
+        {
+          fileName: file.name,
+          processedAt: new Date().toISOString(),
+        },
+      ],
+      'file-processing'
+    );
+  }
+});
+```
+---
+## Format-Specific Processing
+### CSV Processing
+```typescript
+import { CSVParserService } from '@fluentcommerce/fc-connect-sdk';
+async function processCSV(dataSource: DataSource, file: string) {
+  const csvParser = new CSVParserService();
+  // Download CSV
+  const content = await dataSource.downloadFile(file);
+  // Parse with validation
+  const records = await csvParser.parse(content as string, {
+    columns: true, // Use first row as headers
+    skip_empty_lines: true, // Skip empty rows
+    trim: true, // Trim whitespace
+    cast: true, // Auto-cast types (numbers, booleans)
+    cast_date: false, // Don't auto-cast dates (handle manually)
+    relax_column_count: false, // Enforce consistent column count
+    on_record: (record, context) => {
+      // Validate each record during parsing
+      if (!record.sku || !record.quantity) {
+        throw new Error(`Missing required fields at line ${context.lines}`);
+      }
+      return record;
+    },
+  });
+  logger.info('CSV parsed successfully', { recordCount: records.length });
+  return records;
+}
+```
+### JSON Processing
+```typescript
+async function processJSON(dataSource: DataSource, file: string) {
+  const content = await dataSource.downloadFile(file);
+  try {
+    const data = JSON.parse(content as string);
+    // Handle different JSON structures
+    let records: any[];
+    if (Array.isArray(data)) {
+      // Array of records
+      records = data;
+    } else if (data.records && Array.isArray(data.records)) {
+      // Wrapped array: { "records": [...] }
+      records = data.records;
+    } else if (data.data && Array.isArray(data.data)) {
+      // Wrapped array: { "data": [...] }
+      records = data.data;
+    } else {
+      // Single record
+      records = [data];
+    }
+    logger.info('JSON parsed successfully', { recordCount: records.length });
+    return records;
+  } catch (error) {
+    throw new Error(`Invalid JSON in file ${file}: ${(error as Error).message}`);
+  }
+}
+```
+### JSON Lines (JSONL) Processing
+```typescript
+async function processJSONL(dataSource: DataSource, file: string) {
+  const content = await dataSource.downloadFile(file);
+  const lines = (content as string).split('\n').filter(line => line.trim());
+  const records = [];
+  const errors = [];
+  for (let i = 0; i < lines.length; i++) {
+    try {
+      const record = JSON.parse(lines[i]);
+      records.push(record);
+    } catch (error) {
+      errors.push({
+        line: i + 1,
+        content: lines[i],
+        error: (error as Error).message,
+      });
+    }
+  }
+  if (errors.length > 0) {
+    logger.warn('JSONL parsing errors', { errorCount: errors.length, errors });
+  }
+  logger.info('JSONL parsed successfully', {
+    totalLines: lines.length,
+    recordCount: records.length,
+    errorCount: errors.length,
+  });
+  return records;
+}
+```
+### Parquet Processing
+```typescript
+import { ParquetParserService } from '@fluentcommerce/fc-connect-sdk';
+async function processParquet(dataSource: DataSource, file: string) {
+  const parquetParser = new ParquetParserService(logger);
+  // Download Parquet file
+  const buffer = await dataSource.downloadFile(file, { encoding: 'binary' });
+  // Parse Parquet
+  const records = await parquetParser.parse(buffer as Buffer);
+  logger.info('Parquet parsed successfully', {
+    recordCount: records.length,
+    fileSize: (buffer as Buffer).length,
+  });
+  return records;
+}
+```
+---
+## Validation Strategies
+### Pre-Processing Validation
+```typescript
+async function validateBeforeProcessing(file: FileMetadata): Promise<boolean> {
+  const validationRules = [
+    // Rule 1: File size check
+    () => {
+      const maxSize = 100 * 1024 * 1024; // 100MB
+      if (file.size > maxSize) {
+        logger.error('File too large', { size: file.size, maxSize });
+        return false;
+      }
+      return true;
+    },
+    // Rule 2: File name pattern check
+    () => {
+      const pattern = /^inventory_\d{8}\.csv$/;
+      if (!pattern.test(file.name)) {
+        logger.error('Invalid file name pattern', { name: file.name, expected: pattern });
+        return false;
+      }
+      return true;
+    },
+    // Rule 3: File age check (not too old)
+    () => {
+      const maxAge = 7 * 24 * 60 * 60 * 1000; // 7 days
+      const age = Date.now() - file.lastModified.getTime();
+      if (age > maxAge) {
+        logger.warn('File too old', {
+          name: file.name,
+          age: Math.floor(age / (24 * 60 * 60 * 1000)) + ' days',
+        });
+        return false;
+      }
+      return true;
+    },
+  ];
+  // Run all validation rules
+  for (const rule of validationRules) {
+    if (!rule()) {
+      return false;
+    }
+  }
+  return true;
+}
+// Use in processing pipeline
+async function processWithValidation(dataSource: DataSource) {
+  const files = await dataSource.listFiles();
+  for (const file of files) {
+    // Validate before downloading
+    if (!(await validateBeforeProcessing(file))) {
+      logger.warn('File failed validation, skipping', { file: file.name });
+      continue;
+    }
+    // Process validated file
+    await processFile(dataSource, file.path);
+  }
+}
+```
+### Record-Level Validation
+```typescript
+interface ValidationRule {
+  field: string;
+  rule: (value: any) => boolean;
+  message: string;
+}
+const validationRules: ValidationRule[] = [
+  {
+    field: 'sku',
+    rule: value => typeof value === 'string' && value.length > 0,
+    message: 'SKU must be non-empty string',
+  },
+  {
+    field: 'quantity',
+    rule: value => typeof value === 'number' && value >= 0,
+    message: 'Quantity must be non-negative number',
+  },
+  {
+    field: 'price',
+    rule: value => typeof value === 'number' && value > 0,
+    message: 'Price must be positive number',
+  },
+];
+function validateRecord(record: any, index: number): { valid: boolean; errors: string[] } {
+  const errors: string[] = [];
+  for (const rule of validationRules) {
+    if (!rule.rule(record[rule.field])) {
+      errors.push(`Record ${index}: ${rule.message} (value: ${record[rule.field]})`);
+    }
+  }
+  return { valid: errors.length === 0, errors };
+}
+// Use in processing
+async function processWithRecordValidation(records: any[]) {
+  const validRecords = [];
+  const invalidRecords = [];
+  for (let i = 0; i < records.length; i++) {
+    const validation = validateRecord(records[i], i + 1);
+    if (validation.valid) {
+      validRecords.push(records[i]);
+    } else {
+      invalidRecords.push({ record: records[i], errors: validation.errors });
+      logger.warn('Invalid record', { index: i + 1, errors: validation.errors });
+    }
+  }
+  logger.info('Validation complete', {
+    total: records.length,
+    valid: validRecords.length,
+    invalid: invalidRecords.length,
+  });
+  // Process only valid records
+  if (validRecords.length > 0) {
+    await sendToFluent(validRecords);
+  }
+  // Report invalid records
+  if (invalidRecords.length > 0) {
+    await reportInvalidRecords(invalidRecords);
+  }
+}
+```
+---
+## Error Recovery
+> **For comprehensive error handling patterns:** See [File Operations Error Handling Guide](../data-sources-file-operations-error-handling.md) for detailed coverage of connection errors, discovery errors, reading errors, parsing errors, troubleshooting flowcharts, and production checklists for S3 and SFTP operations.
+### Retry Strategy
+```typescript
+async function processWithRetry<T>(
+  fn: () => Promise<T>,
+  options = {
+    maxRetries: 3,
+    initialDelay: 1000,
+    maxDelay: 30000,
+    backoffMultiplier: 2,
+  }
+): Promise<T> {
+  let lastError: Error;
+  for (let attempt = 0; attempt <= options.maxRetries; attempt++) {
+    try {
+      return await fn();
+    } catch (error: any) {
+      lastError = error;
+      // Don't retry client errors (4xx)
+      if (error.statusCode && error.statusCode >= 400 && error.statusCode < 500) {
+        throw error;
+      }
+      if (attempt < options.maxRetries) {
+        const delay = Math.min(
+          options.initialDelay * Math.pow(options.backoffMultiplier, attempt),
+          options.maxDelay
+        );
+        logger.warn(`Attempt ${attempt + 1} failed, retrying in ${delay}ms`, {
+          error: error.message,
+          retriesLeft: options.maxRetries - attempt,
+        });
+        await sleep(delay);
+      }
+    }
+  }
+  throw new Error(
+    `Operation failed after ${options.maxRetries + 1} attempts: ${lastError!.message}`
+  );
+}
+// Use in file processing
+async function processFileWithRetry(dataSource: DataSource, file: string) {
+  return processWithRetry(async () => {
+    const content = await dataSource.downloadFile(file);
+    const records = parseContent(content);
+    await sendToFluent(records);
+    return records.length;
+  });
+}
+```
+### Partial Failure Handling
+```typescript
+async function processBatchWithPartialFailure(
+  records: any[],
+  client: FluentClient,
+  batchSize: number = 100
+) {
+  const results = {
+    successful: 0,
+    failed: 0,
+    errors: [] as Array<{ batch: number; error: string; records: any[] }>,
+  };
+  for (let i = 0; i < records.length; i += batchSize) {
+    const batch = records.slice(i, i + batchSize);
+    const batchNumber = Math.floor(i / batchSize) + 1;
+    try {
+      await client.sendBatch(jobId, { entities: batch });
+      results.successful += batch.length;
+      logger.info(`Batch ${batchNumber} succeeded`, { recordCount: batch.length });
+    } catch (error) {
+      results.failed += batch.length;
+      results.errors.push({
+        batch: batchNumber,
+        error: (error as Error).message,
+        records: batch,
+      });
+      logger.error(`Batch ${batchNumber} failed`, error as Error, {
+        recordCount: batch.length,
+        startIndex: i,
+      });
+      // Continue processing remaining batches
+    }
+  }
+  logger.info('Batch processing complete', results);
+  // Save failed batches for manual review or retry
+  if (results.errors.length > 0) {
+    await saveFailedBatches(results.errors);
+  }
+  return results;
+}
+async function saveFailedBatches(errors: any[]) {
+  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const failedRecords = errors.flatMap(e => e.records);
+  await s3Target.writeFile(
+    `s3://failed-batches/failed-${timestamp}.json`,
+    JSON.stringify({ errors, records: failedRecords }, null, 2)
+  );
+}
+```
+---
+## Performance Optimization
+### Optimization Checklist
+```typescript
+// 1. Use appropriate batch sizes
+const optimalBatchSize = calculateOptimalBatchSize(records.length);
+// 2. Parallel processing with concurrency limit
+const maxConcurrency = 5;
+await processFilesInParallel(files, maxConcurrency);
+// 3. Minimize data source round trips
+const files = await dataSource.listFiles({ prefix: 'inventory/' });
+// Download all file metadata in one call
+// 4. Use streaming for large files
+if (fileSize > 10 * 1024 * 1024) {
+  await processWithStreaming(file);
+} else {
+  await processBuffered(file);
+}
+// 5. Cache parsed results
+const parseCache = new Map<string, any[]>();
+if (parseCache.has(file.path)) {
+  return parseCache.get(file.path);
+}
+// 6. Efficient state lookups
+const processedFiles = new Set(await stateService.getProcessedFiles());
+for (const file of files) {
+  if (processedFiles.has(file.path)) continue;
+  // Process file
+}
+// 7. Avoid redundant validations
+// Validate once before processing, not on every record
+```
+---
+## Production Patterns
+### Complete Ingestion Pipeline
+```typescript
+import {
+  S3DataSource,
+  CSVParserService,
+  UniversalMapper,
+  StateService,
+  VersoriKVAdapter,
+  createClient,
+} from '@fluentcommerce/fc-connect-sdk';
+async function productionIngestionPipeline(ctx: any) {
+  // 1. Initialize services
+  const { log, openKv } = ctx;
+  const client = await createClient(ctx); // Auto-detects Versori context
+  const s3Source = new S3DataSource(
+    {
+      /* config */
+    },
+    log
+  );
+  const csvParser = new CSVParserService();
+  const mapper = new UniversalMapper(mappingConfig);
+  const stateService = new StateService(new VersoriKVAdapter(openKv(':project:')));
+  // 2. List files
+  const files = await s3Source.listFiles({ prefix: 'inventory/' });
+  log.info('Files found', { count: files.length });
+  // 3. Filter and validate
+  const validFiles = files.filter(file => {
+    // Size check
+    if (file.size > 100 * 1024 * 1024) {
+      log.warn('File too large, skipping', { file: file.name, size: file.size });
+      return false;
+    }
+    // Name pattern check
+    if (!file.name.endsWith('.csv')) {
+      return false;
+    }
+    return true;
+  });
+  // 4. Process files
+  for (const file of validFiles) {
+    const stateKey = `processed:${file.etag}`;
+    // Check state
+    if (await stateService.isFileProcessed(kvAdapter as any, stateKey)) {
+      log.info('File already processed', { file: file.name });
+      continue;
+    }
+    try {
+      // Download and parse
+      const content = await s3Source.downloadFile(file.path);
+      const records = await csvParser.parse(content as string);
+      // Map fields
+      const mappingResult = await mapper.map(records);
+      if (!mappingResult.success) {
+        throw new Error(`Mapping failed: ${JSON.stringify(mappingResult.errors)}`);
+      }
+      // Send to Fluent with retry
+      await processWithRetry(async () => {
+        await client.sendBatch(jobId, { entities: mappingResult.data });
+      });
+      // Mark as processed
+      await stateService.updateSyncState(
+        kvAdapter as any,
+        [
+          {
+            fileName: file.name,
+            recordCount: records.length,
+            processedAt: new Date().toISOString(),
+          },
+        ],
+        'file-processing'
+      );
+      // Archive file
+      await s3Source.moveFile(file.path, file.path.replace('inventory/', 'processed/'));
+      log.info('File processed successfully', { file: file.name, records: records.length });
+    } catch (error) {
+      log.error('Failed to process file', { file: file.path, error: (error as Error).message });
+      // Continue with next file
+    }
+  }
+}
+```
+---
+## Next Steps
+**Explore advanced topics** → [Module 5: Advanced Topics](./data-sources-05-advanced-topics.md)
+**Quick reference** → [Quick Reference](../data-sources-quick-reference.md)
+---
+[← Back to SFTP Operations](./data-sources-03-sftp-operations.md) | [Next: Advanced Topics →](./data-sources-05-advanced-topics.md) | [↑ Back to Guide](../data-sources-readme.md)