@fluentcommerce/fc-connect-sdk 0.1.54 → 0.1.55

package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-08-best-practices.md

@@ -1,712 +1,712 @@
-# Module 8: Best Practices
-
-[← Back to Parquet Guide](../parquet-readme.md)
-
-**Module 8 of 8** | **Level**: All Levels | **Time**: 20 minutes
-
----
-
-## Overview
-
-Learn production-ready patterns, error handling strategies, troubleshooting techniques, and best practices for reliable Parquet processing.
-
-## Learning Objectives
-
-By the end of this module, you will:
-
-- ✅ Implement robust error handling
-- ✅ Apply production-ready patterns
-- ✅ Troubleshoot common issues
-- ✅ Monitor and maintain Parquet workflows
-- ✅ Follow security and data quality best practices
-
----
-
-## Error Handling
-
-### Robust Error Recovery
-
-```typescript
-import { Buffer } from 'node:buffer'; // Required for Deno/Versori runtime
-
-class RobustParquetProcessor {
-  private parser: ParquetParserService;
-  private maxRetries = 3;
-
-  constructor() {
-    this.parser = new ParquetParserService(logger, {
-      batchSize: 2000,
-      enableStreaming: true,
-    });
-  }
-
-  /**
-   * Parse with automatic retry logic
-   */
-  async parseWithRetry(arrayBuffer: ArrayBuffer, fileName: string, retryCount = 0): Promise<any> {
-    try {
-      return await this.parser.parse(arrayBuffer, fileName);
-    } catch (error: any) {
-      // Retry on parsing errors
-      if (error.code === 'PARSING_ERROR' && retryCount < this.maxRetries) {
-        console.warn(`⚠️  Retry ${retryCount + 1}/${this.maxRetries} for ${fileName}`);
-
-        // Exponential backoff
-        await this.sleep(Math.pow(2, retryCount) * 1000);
-
-        return this.parseWithRetry(arrayBuffer, fileName, retryCount + 1);
-      }
-
-      // Log detailed error
-      console.error('❌ Parquet parsing failed:', {
-        fileName,
-        error: error.message,
-        code: error.code,
-        retryCount,
-      });
-
-      throw error;
-    }
-  }
-
-  /**
-   * Safe parsing with error containment
-   */
-  async safeParse(
-    arrayBuffer: ArrayBuffer,
-    fileName: string
-  ): Promise<{ success: boolean; data?: any; error?: any }> {
-    try {
-      const result = await this.parseWithRetry(arrayBuffer, fileName);
-      return { success: true, data: result };
-    } catch (error: any) {
-      return {
-        success: false,
-        error: {
-          message: error.message,
-          code: error.code,
-          fileName,
-        },
-      };
-    }
-  }
-
-  private sleep(ms: number): Promise<void> {
-    return new Promise(resolve => setTimeout(resolve, ms));
-  }
-}
-
-// Usage
-const processor = new RobustParquetProcessor();
-
-const result = await processor.safeParse(arrayBuffer, 'inventory.parquet');
-
-if (result.success) {
-  console.log(`✅ Parsed ${result.data.stats.totalRecords} records`);
-  // Process records
-} else {
-  console.error(`❌ Failed to parse:`, result.error);
-  // Handle error, notify monitoring
-}
-```
-
----
-
-## Validation Best Practices
-
-### Pre-Flight Validation
-
-```typescript
-/**
- * Validate before processing
- */
-async function validateAndProcess(arrayBuffer: ArrayBuffer, fileName: string): Promise<void> {
-  const parser = new ParquetParserService(logger);
-
-  // 1. Validate file
-  const validationResult = await parser.validateFile(arrayBuffer, fileName);
-
-  if (!validationResult.isValid) {
-    console.error('❌ File validation failed:');
-    validationResult.errors.forEach(err => {
-      console.error(`   ${err.field}: ${err.message}`);
-    });
-    throw new Error('Invalid Parquet file');
-  }
-
-  // 2. Log warnings
-  if (validationResult.warnings.length > 0) {
-    console.warn('⚠️  Validation warnings:');
-    validationResult.warnings.forEach(warn => {
-      console.warn(`   ${warn.field}: ${warn.message}`);
-    });
-  }
-
-  // 3. Parse with schema validation
-  const { records, stats } = await parser.parse(arrayBuffer, fileName, {
-    validation: {
-      enabled: true,
-      requiredColumns: ['ref', 'productRef', 'locationRef', 'qty'],
-      columnTypes: {
-        ref: 'string',
-        productRef: 'string',
-        locationRef: 'string',
-        qty: 'int32',
-      },
-    },
-  });
-
-  // 4. Data quality checks
-  if (stats.validation && !stats.validation.isValid) {
-    console.error('❌ Schema validation failed:');
-    stats.validation.errors.forEach(err => {
-      console.error(`   ${err.field}: ${err.message}`);
-    });
-    throw new Error('Schema validation failed');
-  }
-
-  // 5. Business logic validation
-  const invalidRecords = records.filter(r => r.qty < 0);
-  if (invalidRecords.length > 0) {
-    console.warn(`⚠️  Found ${invalidRecords.length} records with negative quantities`);
-    // Decide how to handle (skip, fix, fail)
-  }
-
-  console.log(`✅ Validation passed: ${stats.totalRecords} records`);
-}
-```
-
----
-
-## Production Patterns
-
-### Pattern 1: Idempotent Processing
-
-```typescript
-import { StateService, VersoriKVAdapter } from '@fluentcommerce/fc-connect-sdk';
-
-/**
- * Process files only once (idempotent)
- * Business use case: Prevent duplicate processing
- */
-class IdempotentParquetProcessor {
-  private parser: ParquetParserService;
-  private stateService: StateService;
-  private kvAdapter: VersoriKVAdapter;
-  private s3: S3DataSource;
-
-  constructor(kvAdapter: VersoriKVAdapter, s3Config: any, logger: Logger) {
-    this.parser = new ParquetParserService(logger);
-    this.stateService = new StateService(logger);
-    this.kvAdapter = kvAdapter;
-    this.s3 = new S3DataSource({ type: 'S3_CSV', s3Config }, logger);
-  }
-
-  async processNewFiles(prefix: string): Promise<void> {
-    // 1. List files
-    const files = await this.s3.listFiles({ prefix });
-    const parquetFiles = files.filter(f => f.path.endsWith('.parquet'));
-
-    console.log(`Found ${parquetFiles.length} Parquet files`);
-
-    // 2. Process only unprocessed files
-    for (const file of parquetFiles) {
-      const fileKey = `processed_${file.path}`;
-
-      // Check if already processed
-      const isProcessed = await this.stateService.isFileProcessed(this.kvAdapter, fileKey);
-
-      if (isProcessed) {
-        console.log(`⏭️  Skipping ${file.key} (already processed)`);
-        continue;
-      }
-
-      try {
-        console.log(`🔄 Processing ${file.key}...`);
-
-        // Process file
-        await this.processFile(file.path);
-
-        // Mark as processed
-        await this.stateService.markFileProcessed(this.kvAdapter, fileKey, {
-          timestamp: new Date().toISOString(),
-          fileKey: file.path,
-          bucket: (this.s3 as any).s3Config.bucket,
-        });
-
-        console.log(`✅ Completed ${file.key}`);
-      } catch (error) {
-        console.error(`❌ Failed ${file.key}:`, error);
-        // Don't mark as processed - will retry next run
-      }
-    }
-  }
-
-  private async processFile(key: string): Promise<void> {
-    const buffer = await this.s3.downloadFile(key, { encoding: 'binary' });
-    const arrayBuffer = buffer.buffer.slice(
-      buffer.byteOffset,
-      buffer.byteOffset + buffer.byteLength
-    );
-
-    const streamResult = await this.parser.parseStreaming(arrayBuffer, key);
-
-    for await (const batch of streamResult.records) {
-      await this.processBatch(batch);
-    }
-
-    const stats = await streamResult.stats;
-    console.log(`   ${stats.totalRecords} records processed`);
-  }
-
-  private async processBatch(batch: any[]): Promise<void> {
-    // Your business logic
-  }
-}
-
-// Usage example
-const logger = toStructuredLogger(createConsoleLogger(), { logLevel: 'info' });
-const processor = new IdempotentParquetProcessor(kvAdapter, s3Config, logger);
-await processor.processNewFiles('parquet/');
-```
-
-### Pattern 2: Dead Letter Queue
-
-```typescript
-/**
- * Failed file handling with DLQ pattern
- */
-class ParquetProcessorWithDLQ {
-  private parser: ParquetParserService;
-  private s3: S3DataSource;
-  private dlqBucket: string;
-
-  constructor(s3Config: any, dlqBucket: string, logger: Logger) {
-    this.parser = new ParquetParserService(logger);
-    this.s3 = new S3DataSource(
-      {
-        type: 'S3_CSV',
-        s3Config: {
-          bucket: process.env.S3_BUCKET!,
-          accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-          secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
-          region: process.env.AWS_REGION!,
-        },
-      },
-      logger
-    );
-    this.dlqBucket = dlqBucket;
-  }
-
-  async processWithDLQ(key: string): Promise<{ success: boolean; recordsProcessed?: number }> {
-    try {
-      const buffer = await this.s3.downloadFile(key, { encoding: 'binary' });
-      const arrayBuffer = buffer.buffer.slice(
-        buffer.byteOffset,
-        buffer.byteOffset + buffer.byteLength
-      );
-
-      const { records, stats } = await this.parser.parse(arrayBuffer, key);
-
-      // Process records
-      await this.processRecords(records);
-
-      return { success: true, recordsProcessed: stats.totalRecords };
-    } catch (error: any) {
-      console.error(`❌ Processing failed for ${key}:`, error.message);
-
-      // Move to DLQ
-      await this.moveToDLQ(key, error);
-
-      return { success: false };
-    }
-  }
-
-  private async moveToDLQ(sourceKey: string, error: Error): Promise<void> {
-    const dlqKey = `failed/${sourceKey}_${Date.now()}.parquet`;
-
-    console.log(`📦 Moving to DLQ: ${dlqKey}`);
-
-    // Create separate S3 instance for DLQ bucket
-    const dlqS3 = new S3DataSource(
-      {
-        type: 'S3_CSV',
-        s3Config: {
-          bucket: this.dlqBucket,
-          accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-          secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
-          region: process.env.AWS_REGION!,
-        },
-      },
-      toStructuredLogger(createConsoleLogger(), {
-        logLevel: 'info',
-      })
-    );
-
-    try {
-      // Copy file to DLQ
-      const buffer = await this.s3.downloadFile(sourceKey, { encoding: 'binary' });
-      await dlqS3.uploadFile(dlqKey, buffer, {
-        contentType: 'application/octet-stream',
-      });
-
-      // Write error metadata
-      const errorMetadata = {
-        originalKey: sourceKey,
-        error: error.message,
-        timestamp: new Date().toISOString(),
-      };
-
-      await dlqS3.uploadFile(
-        `${dlqKey}.error.json`,
-        Buffer.from(JSON.stringify(errorMetadata, null, 2)),
-        { contentType: 'application/json' }
-      );
-
-      console.log(`✅ Moved to DLQ with metadata`);
-    } catch (dlqError) {
-      console.error(`❌ Failed to move to DLQ:`, dlqError);
-      // Alert monitoring system
-    }
-  }
-
-  private async processRecords(records: any[]): Promise<void> {
-    // Your processing logic
-  }
-}
-```
-
----
-
-## Troubleshooting
-
-### Common Issues and Solutions
-
-#### Issue 1: Out of Memory
-
-**Symptoms**: Node.js crashes with `FATAL ERROR: Reached heap limit`
-
-**Solutions**:
-
-```typescript
-// 1. Enable streaming
-const parser = new ParquetParserService(logger, {
-  enableStreaming: true, // ✅ Always enable for large files
-  batchSize: 500, // ✅ Smaller batches
-});
-
-// 2. Process in smaller chunks
-const streamResult = await parser.parseStreaming(arrayBuffer, fileName);
-
-for await (const batch of streamResult.records) {
-  await processBatch(batch);
-  // Batch is released after processing
-}
-
-// 3. Increase Node.js heap size
-// node --max-old-space-size=4096 script.js
-```
-
-#### Issue 2: Slow Parsing
-
-**Symptoms**: Processing takes much longer than expected
-
-**Diagnostics**:
-
-```typescript
-// Check performance metrics
-const { records, stats } = await parser.parse(arrayBuffer, fileName);
-
-console.log(`Records/sec: ${stats.recordsPerSecond}`);
-console.log(`Processing time: ${stats.processingTimeMs}ms`);
-console.log(`File size: ${(stats.fileSizeBytes / 1024 / 1024).toFixed(2)}MB`);
-
-// Expected: 50,000-70,000 records/sec for typical files
-// If much slower, check:
-// - File compression (GZIP/BROTLI slower than SNAPPY)
-// - Validation enabled (disable for speed)
-// - Batch size too small (increase to 2000-5000)
-```
-
-**Solutions**:
-
-```typescript
-// 1. Disable validation for trusted sources
-const { records } = await parser.parse(arrayBuffer, fileName, {
-  validation: { enabled: false }, // ✅ Faster
-});
-
-// 2. Increase batch size
-const parser = new ParquetParserService(logger, {
-  batchSize: 5000, // ✅ Larger batches for throughput
-});
-
-// 3. Use parallel processing for multiple files
-// See Module 7: Performance Optimization
-```
-
-#### Issue 3: Schema Mismatch
-
-**Symptoms**: `VALIDATION_ERROR: Required column 'xyz' not found`
-
-**Diagnosis**:
-
-```typescript
-// Check column statistics
-const { stats } = await parser.parse(arrayBuffer, fileName);
-
-console.log('Available columns:');
-Object.keys(stats.columnStats).forEach(col => {
-  console.log(`  - ${col} (${stats.columnStats[col].type})`);
-});
-```
-
-**Solution**:
-
-```typescript
-// 1. Update validation config to match actual schema
-const { records } = await parser.parse(arrayBuffer, fileName, {
-  validation: {
-    enabled: true,
-    requiredColumns: ['ref', 'productRef', 'locationRef', 'qty'], // ✅ Correct columns
-    allowAdditionalColumns: true, // ✅ Allow extra columns
-  },
-});
-
-// 2. Or disable validation temporarily
-```
-
-#### Issue 4: Corrupted Parquet File
-
-**Symptoms**: `PARSING_ERROR: Invalid Parquet magic number`
-
-**Solutions**:
-
-```typescript
-// 1. Validate file before parsing
-const validationResult = await parser.validateFile(arrayBuffer, fileName);
-
-if (!validationResult.isValid) {
-  console.error('File is corrupted or not Parquet format');
-  // Move to DLQ or alert
-  return;
-}
-
-// 2. Check file integrity
-const fileHash = crypto.createHash('md5').update(buffer).digest('hex');
-console.log(`File hash: ${fileHash}`);
-// Compare with expected hash if available
-```
-
----
-
-## Monitoring and Logging
-
-### Comprehensive Logging
-
-```typescript
-class ParquetProcessorWithLogging {
-  private parser: ParquetParserService;
-  private logger: Logger;
-
-  constructor() {
-    this.logger = toStructuredLogger(createConsoleLogger(), {
-      logLevel: 'info',
-    });
-    this.parser = new ParquetParserService(this.logger, {
-      batchSize: 2000,
-      enableStreaming: true,
-    });
-  }
-
-  async processWithLogging(arrayBuffer: ArrayBuffer, fileName: string): Promise<void> {
-    const startTime = Date.now();
-    const fileSizeMB = (arrayBuffer.byteLength / 1024 / 1024).toFixed(2);
-
-    this.logger.info(`📥 Starting: ${fileName} (${fileSizeMB}MB)`);
-
-    try {
-      // Parse
-      const { records, stats } = await this.parser.parse(arrayBuffer, fileName);
-
-      const duration = Date.now() - startTime;
-
-      // Log success
-      this.logger.info(`✅ Success: ${fileName}`, {
-        records: stats.totalRecords,
-        processingTimeMs: stats.processingTimeMs,
-        throughput: stats.recordsPerSecond,
-        batches: stats.batchesProcessed,
-        totalDuration: duration,
-        peakMemoryMB: (stats.peakMemoryUsage / 1024 / 1024).toFixed(2),
-      });
-
-      // Log warnings if any
-      if (stats.validation && stats.validation.warnings.length > 0) {
-        this.logger.warn(`⚠️  Validation warnings for ${fileName}:`, {
-          warnings: stats.validation.warnings,
-        });
-      }
-    } catch (error: any) {
-      const duration = Date.now() - startTime;
-
-      this.logger.error(`❌ Failed: ${fileName}`, {
-        error: error.message,
-        code: error.code,
-        duration,
-      });
-
-      throw error;
-    }
-  }
-}
-```
-
----
-
-## Security Best Practices
-
-### 1. Validate File Sources
-
-```typescript
-// Only process files from trusted S3 buckets
-const ALLOWED_BUCKETS = ['data-lake-prod', 'data-lake-staging'];
-
-async function processSecure(bucket: string, key: string): Promise<void> {
-  if (!ALLOWED_BUCKETS.includes(bucket)) {
-    throw new Error(`Unauthorized bucket: ${bucket}`);
-  }
-
-  // Process file
-}
-```
-
-### 2. Sanitize File Paths
-
-```typescript
-import * as path from 'path';
-
-function sanitizePath(filePath: string): string {
-  // Prevent directory traversal
-  const normalized = path.normalize(filePath);
-
-  if (normalized.includes('..')) {
-    throw new Error('Invalid file path: directory traversal detected');
-  }
-
-  return normalized;
-}
-```
-
-### 3. Limit File Size
-
-```typescript
-const MAX_FILE_SIZE = 500 * 1024 * 1024; // 500 MB
-
-async function processWithSizeLimit(arrayBuffer: ArrayBuffer, fileName: string): Promise<void> {
-  if (arrayBuffer.byteLength > MAX_FILE_SIZE) {
-    throw new Error(
-      `File too large: ${(arrayBuffer.byteLength / 1024 / 1024).toFixed(2)}MB > ${MAX_FILE_SIZE / 1024 / 1024}MB`
-    );
-  }
-
-  // Process file
-}
-```
-
----
-
-## Data Quality Checks
-
-```typescript
-/**
- * Validate data quality after parsing
- */
-function validateDataQuality(
-  records: any[],
-  stats: ParquetParsingStats
-): { isValid: boolean; issues: string[] } {
-  const issues: string[] = [];
-
-  // Check for sufficient records
-  if (records.length === 0) {
-    issues.push('No records found in file');
-  }
-
-  // Check column completeness
-  for (const [colName, colStats] of Object.entries(stats.columnStats)) {
-    const completeness = 1 - colStats.nullCount / stats.totalRecords;
-
-    if (completeness < 0.9) {
-      issues.push(`Low completeness for ${colName}: ${(completeness * 100).toFixed(1)}%`);
-    }
-  }
-
-  // Check for duplicate keys
-  const refs = records.map(r => r.ref).filter(Boolean);
-  const uniqueRefs = new Set(refs);
-
-  if (refs.length !== uniqueRefs.size) {
-    issues.push(`Duplicate refs found: ${refs.length - uniqueRefs.size} duplicates`);
-  }
-
-  // Business logic checks
-  const negativeQty = records.filter(r => r.qty < 0).length;
-  if (negativeQty > 0) {
-    issues.push(`${negativeQty} records with negative quantities`);
-  }
-
-  return {
-    isValid: issues.length === 0,
-    issues,
-  };
-}
-
-// Usage
-const { records, stats } = await parser.parse(arrayBuffer, fileName);
-const qualityCheck = validateDataQuality(records, stats);
-
-if (!qualityCheck.isValid) {
-  console.warn('⚠️  Data quality issues:');
-  qualityCheck.issues.forEach(issue => console.warn(`   - ${issue}`));
-  // Decide whether to proceed or reject
-}
-```
-
----
-
-## Key Takeaways
-
-- 🎯 **Retry logic** - Handle transient failures gracefully
-- 🎯 **Validation first** - Check schema and data quality before processing
-- 🎯 **Idempotent processing** - Use state management to prevent duplicates
-- 🎯 **Dead letter queue** - Capture failed files for manual review
-- 🎯 **Comprehensive logging** - Track success, failures, and performance
-- 🎯 **Security checks** - Validate sources, sanitize paths, limit sizes
-- 🎯 **Monitor metrics** - Track throughput, errors, memory usage
-
----
-
-## Summary
-
-You've completed the Parquet Processing guide! You now know how to:
-
-✅ Understand Parquet format and when to use it
-✅ Parse Parquet files with ParquetParserService
-✅ Handle large files with streaming
-✅ Write Parquet files using parquetjs
-✅ Extract from Fluent Commerce GraphQL to Parquet
-✅ Integrate with S3 for data lake operations
-✅ Optimize performance for large-scale processing
-✅ Apply production-ready error handling and monitoring
-
----
-
-[← Previous: Module 7 - Performance Optimization](./03-pattern-guides-parquet-07-performance-optimization.md) | [Back to Guide](../parquet-readme.md)
-
-## Related Documentation
-
-- [Data Extraction Guide](../../extraction/) - Complete extraction workflows
-- [S3 Complete Guide](../../../02-CORE-GUIDES/data-sources/data-sources-readme.md) - S3 operations
-- [Universal Mapping Guide](../../../02-CORE-GUIDES/mapping/) - Data transformations
-- [Auto-Pagination Guide](../../../02-CORE-GUIDES/auto-pagination/) - GraphQL pagination patterns
+# Module 8: Best Practices
+
+[← Back to Parquet Guide](../parquet-readme.md)
+
+**Module 8 of 8** | **Level**: All Levels | **Time**: 20 minutes
+
+---
+
+## Overview
+
+Learn production-ready patterns, error handling strategies, troubleshooting techniques, and best practices for reliable Parquet processing.
+
+## Learning Objectives
+
+By the end of this module, you will:
+
+- ✅ Implement robust error handling
+- ✅ Apply production-ready patterns
+- ✅ Troubleshoot common issues
+- ✅ Monitor and maintain Parquet workflows
+- ✅ Follow security and data quality best practices
+
+---
+
+## Error Handling
+
+### Robust Error Recovery
+
+```typescript
+import { Buffer } from 'node:buffer'; // Required for Deno/Versori runtime
+
+class RobustParquetProcessor {
+  private parser: ParquetParserService;
+  private maxRetries = 3;
+
+  constructor() {
+    this.parser = new ParquetParserService(logger, {
+      batchSize: 2000,
+      enableStreaming: true,
+    });
+  }
+
+  /**
+   * Parse with automatic retry logic
+   */
+  async parseWithRetry(arrayBuffer: ArrayBuffer, fileName: string, retryCount = 0): Promise<any> {
+    try {
+      return await this.parser.parse(arrayBuffer, fileName);
+    } catch (error: any) {
+      // Retry on parsing errors
+      if (error.code === 'PARSING_ERROR' && retryCount < this.maxRetries) {
+        console.warn(`⚠️  Retry ${retryCount + 1}/${this.maxRetries} for ${fileName}`);
+
+        // Exponential backoff
+        await this.sleep(Math.pow(2, retryCount) * 1000);
+
+        return this.parseWithRetry(arrayBuffer, fileName, retryCount + 1);
+      }
+
+      // Log detailed error
+      console.error('❌ Parquet parsing failed:', {
+        fileName,
+        error: error.message,
+        code: error.code,
+        retryCount,
+      });
+
+      throw error;
+    }
+  }
+
+  /**
+   * Safe parsing with error containment
+   */
+  async safeParse(
+    arrayBuffer: ArrayBuffer,
+    fileName: string
+  ): Promise<{ success: boolean; data?: any; error?: any }> {
+    try {
+      const result = await this.parseWithRetry(arrayBuffer, fileName);
+      return { success: true, data: result };
+    } catch (error: any) {
+      return {
+        success: false,
+        error: {
+          message: error.message,
+          code: error.code,
+          fileName,
+        },
+      };
+    }
+  }
+
+  private sleep(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+}
+
+// Usage
+const processor = new RobustParquetProcessor();
+
+const result = await processor.safeParse(arrayBuffer, 'inventory.parquet');
+
+if (result.success) {
+  console.log(`✅ Parsed ${result.data.stats.totalRecords} records`);
+  // Process records
+} else {
+  console.error(`❌ Failed to parse:`, result.error);
+  // Handle error, notify monitoring
+}
+```
+
+---
+
+## Validation Best Practices
+
+### Pre-Flight Validation
+
+```typescript
+/**
+ * Validate before processing
+ */
+async function validateAndProcess(arrayBuffer: ArrayBuffer, fileName: string): Promise<void> {
+  const parser = new ParquetParserService(logger);
+
+  // 1. Validate file
+  const validationResult = await parser.validateFile(arrayBuffer, fileName);
+
+  if (!validationResult.isValid) {
+    console.error('❌ File validation failed:');
+    validationResult.errors.forEach(err => {
+      console.error(`   ${err.field}: ${err.message}`);
+    });
+    throw new Error('Invalid Parquet file');
+  }
+
+  // 2. Log warnings
+  if (validationResult.warnings.length > 0) {
+    console.warn('⚠️  Validation warnings:');
+    validationResult.warnings.forEach(warn => {
+      console.warn(`   ${warn.field}: ${warn.message}`);
+    });
+  }
+
+  // 3. Parse with schema validation
+  const { records, stats } = await parser.parse(arrayBuffer, fileName, {
+    validation: {
+      enabled: true,
+      requiredColumns: ['ref', 'productRef', 'locationRef', 'qty'],
+      columnTypes: {
+        ref: 'string',
+        productRef: 'string',
+        locationRef: 'string',
+        qty: 'int32',
+      },
+    },
+  });
+
+  // 4. Data quality checks
+  if (stats.validation && !stats.validation.isValid) {
+    console.error('❌ Schema validation failed:');
+    stats.validation.errors.forEach(err => {
+      console.error(`   ${err.field}: ${err.message}`);
+    });
+    throw new Error('Schema validation failed');
+  }
+
+  // 5. Business logic validation
+  const invalidRecords = records.filter(r => r.qty < 0);
+  if (invalidRecords.length > 0) {
+    console.warn(`⚠️  Found ${invalidRecords.length} records with negative quantities`);
+    // Decide how to handle (skip, fix, fail)
+  }
+
+  console.log(`✅ Validation passed: ${stats.totalRecords} records`);
+}
+```
+
+---
+
+## Production Patterns
+
+### Pattern 1: Idempotent Processing
+
+```typescript
+import { StateService, VersoriKVAdapter } from '@fluentcommerce/fc-connect-sdk';
+
+/**
+ * Process files only once (idempotent)
+ * Business use case: Prevent duplicate processing
+ */
+class IdempotentParquetProcessor {
+  private parser: ParquetParserService;
+  private stateService: StateService;
+  private kvAdapter: VersoriKVAdapter;
+  private s3: S3DataSource;
+
+  constructor(kvAdapter: VersoriKVAdapter, s3Config: any, logger: Logger) {
+    this.parser = new ParquetParserService(logger);
+    this.stateService = new StateService(logger);
+    this.kvAdapter = kvAdapter;
+    this.s3 = new S3DataSource({ type: 'S3_CSV', s3Config }, logger);
+  }
+
+  async processNewFiles(prefix: string): Promise<void> {
+    // 1. List files
+    const files = await this.s3.listFiles({ prefix });
+    const parquetFiles = files.filter(f => f.path.endsWith('.parquet'));
+
+    console.log(`Found ${parquetFiles.length} Parquet files`);
+
+    // 2. Process only unprocessed files
+    for (const file of parquetFiles) {
+      const fileKey = `processed_${file.path}`;
+
+      // Check if already processed
+      const isProcessed = await this.stateService.isFileProcessed(this.kvAdapter, fileKey);
+
+      if (isProcessed) {
+        console.log(`⏭️  Skipping ${file.key} (already processed)`);
+        continue;
+      }
+
+      try {
+        console.log(`🔄 Processing ${file.key}...`);
+
+        // Process file
+        await this.processFile(file.path);
+
+        // Mark as processed
+        await this.stateService.markFileProcessed(this.kvAdapter, fileKey, {
+          timestamp: new Date().toISOString(),
+          fileKey: file.path,
+          bucket: (this.s3 as any).s3Config.bucket,
+        });
+
+        console.log(`✅ Completed ${file.key}`);
+      } catch (error) {
+        console.error(`❌ Failed ${file.key}:`, error);
+        // Don't mark as processed - will retry next run
+      }
+    }
+  }
+
+  private async processFile(key: string): Promise<void> {
+    const buffer = await this.s3.downloadFile(key, { encoding: 'binary' });
+    const arrayBuffer = buffer.buffer.slice(
+      buffer.byteOffset,
+      buffer.byteOffset + buffer.byteLength
+    );
+
+    const streamResult = await this.parser.parseStreaming(arrayBuffer, key);
+
+    for await (const batch of streamResult.records) {
+      await this.processBatch(batch);
+    }
+
+    const stats = await streamResult.stats;
+    console.log(`   ${stats.totalRecords} records processed`);
+  }
+
+  private async processBatch(batch: any[]): Promise<void> {
+    // Your business logic
+  }
+}
+
+// Usage example
+const logger = toStructuredLogger(createConsoleLogger(), { logLevel: 'info' });
+const processor = new IdempotentParquetProcessor(kvAdapter, s3Config, logger);
+await processor.processNewFiles('parquet/');
+```
+
+### Pattern 2: Dead Letter Queue
+
+```typescript
+/**
+ * Failed file handling with DLQ pattern
+ */
+class ParquetProcessorWithDLQ {
+  private parser: ParquetParserService;
+  private s3: S3DataSource;
+  private dlqBucket: string;
+
+  constructor(s3Config: any, dlqBucket: string, logger: Logger) {
+    this.parser = new ParquetParserService(logger);
+    this.s3 = new S3DataSource(
+      {
+        type: 'S3_CSV',
+        s3Config: {
+          bucket: process.env.S3_BUCKET!,
+          accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+          secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
+          region: process.env.AWS_REGION!,
+        },
+      },
+      logger
+    );
+    this.dlqBucket = dlqBucket;
+  }
+
+  async processWithDLQ(key: string): Promise<{ success: boolean; recordsProcessed?: number }> {
+    try {
+      const buffer = await this.s3.downloadFile(key, { encoding: 'binary' });
+      const arrayBuffer = buffer.buffer.slice(
+        buffer.byteOffset,
+        buffer.byteOffset + buffer.byteLength
+      );
+
+      const { records, stats } = await this.parser.parse(arrayBuffer, key);
+
+      // Process records
+      await this.processRecords(records);
+
+      return { success: true, recordsProcessed: stats.totalRecords };
+    } catch (error: any) {
+      console.error(`❌ Processing failed for ${key}:`, error.message);
+
+      // Move to DLQ
+      await this.moveToDLQ(key, error);
+
+      return { success: false };
+    }
+  }
+
+  private async moveToDLQ(sourceKey: string, error: Error): Promise<void> {
+    const dlqKey = `failed/${sourceKey}_${Date.now()}.parquet`;
+
+    console.log(`📦 Moving to DLQ: ${dlqKey}`);
+
+    // Create separate S3 instance for DLQ bucket
+    const dlqS3 = new S3DataSource(
+      {
+        type: 'S3_CSV',
+        s3Config: {
+          bucket: this.dlqBucket,
+          accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+          secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
+          region: process.env.AWS_REGION!,
+        },
+      },
+      toStructuredLogger(createConsoleLogger(), {
+        logLevel: 'info',
+      })
+    );
+
+    try {
+      // Copy file to DLQ
+      const buffer = await this.s3.downloadFile(sourceKey, { encoding: 'binary' });
+      await dlqS3.uploadFile(dlqKey, buffer, {
+        contentType: 'application/octet-stream',
+      });
+
+      // Write error metadata
+      const errorMetadata = {
+        originalKey: sourceKey,
+        error: error.message,
+        timestamp: new Date().toISOString(),
+      };
+
+      await dlqS3.uploadFile(
+        `${dlqKey}.error.json`,
+        Buffer.from(JSON.stringify(errorMetadata, null, 2)),
+        { contentType: 'application/json' }
+      );
+
+      console.log(`✅ Moved to DLQ with metadata`);
+    } catch (dlqError) {
+      console.error(`❌ Failed to move to DLQ:`, dlqError);
+      // Alert monitoring system
+    }
+  }
+
+  private async processRecords(records: any[]): Promise<void> {
+    // Your processing logic
+  }
+}
+```
+
+---
+
+## Troubleshooting
+
+### Common Issues and Solutions
+
+#### Issue 1: Out of Memory
+
+**Symptoms**: Node.js crashes with `FATAL ERROR: Reached heap limit`
+
+**Solutions**:
+
+```typescript
+// 1. Enable streaming
+const parser = new ParquetParserService(logger, {
+  enableStreaming: true, // ✅ Always enable for large files
+  batchSize: 500, // ✅ Smaller batches
+});
+
+// 2. Process in smaller chunks
+const streamResult = await parser.parseStreaming(arrayBuffer, fileName);
+
+for await (const batch of streamResult.records) {
+  await processBatch(batch);
+  // Batch is released after processing
+}
+
+// 3. Increase Node.js heap size
+// node --max-old-space-size=4096 script.js
+```
+
+#### Issue 2: Slow Parsing
+
+**Symptoms**: Processing takes much longer than expected
+
+**Diagnostics**:
+
+```typescript
+// Check performance metrics
+const { records, stats } = await parser.parse(arrayBuffer, fileName);
+
+console.log(`Records/sec: ${stats.recordsPerSecond}`);
+console.log(`Processing time: ${stats.processingTimeMs}ms`);
+console.log(`File size: ${(stats.fileSizeBytes / 1024 / 1024).toFixed(2)}MB`);
+
+// Expected: 50,000-70,000 records/sec for typical files
+// If much slower, check:
+// - File compression (GZIP/BROTLI slower than SNAPPY)
+// - Validation enabled (disable for speed)
+// - Batch size too small (increase to 2000-5000)
+```
+
+**Solutions**:
+
+```typescript
+// 1. Disable validation for trusted sources
+const { records } = await parser.parse(arrayBuffer, fileName, {
+  validation: { enabled: false }, // ✅ Faster
+});
+
+// 2. Increase batch size
+const parser = new ParquetParserService(logger, {
+  batchSize: 5000, // ✅ Larger batches for throughput
+});
+
+// 3. Use parallel processing for multiple files
+// See Module 7: Performance Optimization
+```
+
+#### Issue 3: Schema Mismatch
+
+**Symptoms**: `VALIDATION_ERROR: Required column 'xyz' not found`
+
+**Diagnosis**:
+
+```typescript
+// Check column statistics
+const { stats } = await parser.parse(arrayBuffer, fileName);
+
+console.log('Available columns:');
+Object.keys(stats.columnStats).forEach(col => {
+  console.log(`  - ${col} (${stats.columnStats[col].type})`);
+});
+```
+
+**Solution**:
+
+```typescript
+// 1. Update validation config to match actual schema
+const { records } = await parser.parse(arrayBuffer, fileName, {
+  validation: {
+    enabled: true,
+    requiredColumns: ['ref', 'productRef', 'locationRef', 'qty'], // ✅ Correct columns
+    allowAdditionalColumns: true, // ✅ Allow extra columns
+  },
+});
+
+// 2. Or disable validation temporarily
+```
+
+#### Issue 4: Corrupted Parquet File
+
+**Symptoms**: `PARSING_ERROR: Invalid Parquet magic number`
+
+**Solutions**:
+
+```typescript
+// 1. Validate file before parsing
+const validationResult = await parser.validateFile(arrayBuffer, fileName);
+
+if (!validationResult.isValid) {
+  console.error('File is corrupted or not Parquet format');
+  // Move to DLQ or alert
+  return;
+}
+
+// 2. Check file integrity
+const fileHash = crypto.createHash('md5').update(buffer).digest('hex');
+console.log(`File hash: ${fileHash}`);
+// Compare with expected hash if available
+```
+
+---
+
+## Monitoring and Logging
+
+### Comprehensive Logging
+
+```typescript
+class ParquetProcessorWithLogging {
+  private parser: ParquetParserService;
+  private logger: Logger;
+
+  constructor() {
+    this.logger = toStructuredLogger(createConsoleLogger(), {
+      logLevel: 'info',
+    });
+    this.parser = new ParquetParserService(this.logger, {
+      batchSize: 2000,
+      enableStreaming: true,
+    });
+  }
+
+  async processWithLogging(arrayBuffer: ArrayBuffer, fileName: string): Promise<void> {
+    const startTime = Date.now();
+    const fileSizeMB = (arrayBuffer.byteLength / 1024 / 1024).toFixed(2);
+
+    this.logger.info(`📥 Starting: ${fileName} (${fileSizeMB}MB)`);
+
+    try {
+      // Parse
+      const { records, stats } = await this.parser.parse(arrayBuffer, fileName);
+
+      const duration = Date.now() - startTime;
+
+      // Log success
+      this.logger.info(`✅ Success: ${fileName}`, {
+        records: stats.totalRecords,
+        processingTimeMs: stats.processingTimeMs,
+        throughput: stats.recordsPerSecond,
+        batches: stats.batchesProcessed,
+        totalDuration: duration,
+        peakMemoryMB: (stats.peakMemoryUsage / 1024 / 1024).toFixed(2),
+      });
+
+      // Log warnings if any
+      if (stats.validation && stats.validation.warnings.length > 0) {
+        this.logger.warn(`⚠️  Validation warnings for ${fileName}:`, {
+          warnings: stats.validation.warnings,
+        });
+      }
+    } catch (error: any) {
+      const duration = Date.now() - startTime;
+
+      this.logger.error(`❌ Failed: ${fileName}`, {
+        error: error.message,
+        code: error.code,
+        duration,
+      });
+
+      throw error;
+    }
+  }
+}
+```
+
+---
+
+## Security Best Practices
+
+### 1. Validate File Sources
+
+```typescript
+// Only process files from trusted S3 buckets
+const ALLOWED_BUCKETS = ['data-lake-prod', 'data-lake-staging'];
+
+async function processSecure(bucket: string, key: string): Promise<void> {
+  if (!ALLOWED_BUCKETS.includes(bucket)) {
+    throw new Error(`Unauthorized bucket: ${bucket}`);
+  }
+
+  // Process file
+}
+```
+
+### 2. Sanitize File Paths
+
+```typescript
+import * as path from 'path';
+
+function sanitizePath(filePath: string): string {
+  // Prevent directory traversal
+  const normalized = path.normalize(filePath);
+
+  if (normalized.includes('..')) {
+    throw new Error('Invalid file path: directory traversal detected');
+  }
+
+  return normalized;
+}
+```
+
+### 3. Limit File Size
+
+```typescript
+const MAX_FILE_SIZE = 500 * 1024 * 1024; // 500 MB
+
+async function processWithSizeLimit(arrayBuffer: ArrayBuffer, fileName: string): Promise<void> {
+  if (arrayBuffer.byteLength > MAX_FILE_SIZE) {
+    throw new Error(
+      `File too large: ${(arrayBuffer.byteLength / 1024 / 1024).toFixed(2)}MB > ${MAX_FILE_SIZE / 1024 / 1024}MB`
+    );
+  }
+
+  // Process file
+}
+```
+
+---
+
+## Data Quality Checks
+
+```typescript
+/**
+ * Validate data quality after parsing
+ */
+function validateDataQuality(
+  records: any[],
+  stats: ParquetParsingStats
+): { isValid: boolean; issues: string[] } {
+  const issues: string[] = [];
+
+  // Check for sufficient records
+  if (records.length === 0) {
+    issues.push('No records found in file');
+  }
+
+  // Check column completeness
+  for (const [colName, colStats] of Object.entries(stats.columnStats)) {
+    const completeness = 1 - colStats.nullCount / stats.totalRecords;
+
+    if (completeness < 0.9) {
+      issues.push(`Low completeness for ${colName}: ${(completeness * 100).toFixed(1)}%`);
+    }
+  }
+
+  // Check for duplicate keys
+  const refs = records.map(r => r.ref).filter(Boolean);
+  const uniqueRefs = new Set(refs);
+
+  if (refs.length !== uniqueRefs.size) {
+    issues.push(`Duplicate refs found: ${refs.length - uniqueRefs.size} duplicates`);
+  }
+
+  // Business logic checks
+  const negativeQty = records.filter(r => r.qty < 0).length;
+  if (negativeQty > 0) {
+    issues.push(`${negativeQty} records with negative quantities`);
+  }
+
+  return {
+    isValid: issues.length === 0,
+    issues,
+  };
+}
+
+// Usage
+const { records, stats } = await parser.parse(arrayBuffer, fileName);
+const qualityCheck = validateDataQuality(records, stats);
+
+if (!qualityCheck.isValid) {
+  console.warn('⚠️  Data quality issues:');
+  qualityCheck.issues.forEach(issue => console.warn(`   - ${issue}`));
+  // Decide whether to proceed or reject
+}
+```
+
+---
+
+## Key Takeaways
+
+- 🎯 **Retry logic** - Handle transient failures gracefully
+- 🎯 **Validation first** - Check schema and data quality before processing
+- 🎯 **Idempotent processing** - Use state management to prevent duplicates
+- 🎯 **Dead letter queue** - Capture failed files for manual review
+- 🎯 **Comprehensive logging** - Track success, failures, and performance
+- 🎯 **Security checks** - Validate sources, sanitize paths, limit sizes
+- 🎯 **Monitor metrics** - Track throughput, errors, memory usage
+
+---
+
+## Summary
+
+You've completed the Parquet Processing guide! You now know how to:
+
+✅ Understand Parquet format and when to use it
+✅ Parse Parquet files with ParquetParserService
+✅ Handle large files with streaming
+✅ Write Parquet files using parquetjs
+✅ Extract from Fluent Commerce GraphQL to Parquet
+✅ Integrate with S3 for data lake operations
+✅ Optimize performance for large-scale processing
+✅ Apply production-ready error handling and monitoring
+
+---
+
+[← Previous: Module 7 - Performance Optimization](./03-pattern-guides-parquet-07-performance-optimization.md) | [Back to Guide](../parquet-readme.md)
+
+## Related Documentation
+
+- [Data Extraction Guide](../../extraction/) - Complete extraction workflows
+- [S3 Complete Guide](../../../02-CORE-GUIDES/data-sources/data-sources-readme.md) - S3 operations
+- [Universal Mapping Guide](../../../02-CORE-GUIDES/mapping/) - Data transformations
+- [Auto-Pagination Guide](../../../02-CORE-GUIDES/auto-pagination/) - GraphQL pagination patterns