@fluentcommerce/fc-connect-sdk 0.1.54 → 0.1.56

package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-03-reading-parquet.md

@@ -1,525 +1,525 @@
-# Module 3: Reading Parquet Files
-
-[← Back to Parquet Guide](../parquet-readme.md)
-
-**Module 3 of 8** | **Level**: Intermediate | **Time**: 20 minutes
-
----
-
-## Overview
-
-Learn advanced Parquet reading techniques including streaming, filtering, column selection, and memory-efficient processing for large datasets.
-
-## Learning Objectives
-
-By the end of this module, you will:
-- ✅ Use streaming mode for memory-efficient processing
-- ✅ Apply validation rules and schema checking
-- ✅ Process files of any size without memory issues
-- ✅ Monitor performance metrics and optimize throughput
-- ✅ Handle errors gracefully with retry logic
-
----
-
-## Streaming Support
-
-### When to Use Streaming
-
-**Use streaming for**:
-- Files > 50 MB
-- Memory-constrained environments
-- Real-time processing requirements
-- Unknown file sizes
-
-**Use regular parsing for**:
-- Files < 10 MB
-- When you need all records in memory
-- Simple one-time processing
-
-### Basic Streaming Example
-
-```typescript
-import {
-  ParquetParserService,
-  createConsoleLogger,
-  toStructuredLogger
-} from '@fluentcommerce/fc-connect-sdk';
-
-async function streamLargeFile(arrayBuffer: ArrayBuffer, fileName: string) {
-  const logger = toStructuredLogger(createConsoleLogger(), {
-  logLevel: 'info'
-});
-  const parser = new ParquetParserService(logger, {
-    batchSize: 1000,  // Process 1000 records at a time
-    enableStreaming: true
-  });
-
-  // Start streaming parse
-  const streamResult = await parser.parseStreaming(arrayBuffer, fileName);
-
-  let totalProcessed = 0;
-
-  // Process batches as they stream
-  for await (const batch of streamResult.records) {
-    console.log(`Processing batch of ${batch.length} records`);
-
-    // Process each record
-    for (const record of batch) {
-      // Your processing logic
-      await processRecord(record);
-    }
-
-    totalProcessed += batch.length;
-
-    // Progress logging
-    if (totalProcessed % 10000 === 0) {
-      console.log(`Processed ${totalProcessed} records...`);
-    }
-  }
-
-  // Get final statistics
-  const stats = await streamResult.stats;
-  console.log(`Complete: ${stats.totalRecords} records in ${stats.processingTimeMs}ms`);
-}
-
-async function processRecord(record: any): Promise<void> {
-  // Your business logic
-  console.log(`Processing: ${record.productRef} at ${record.locationRef}`);
-}
-```
-
-### Streaming with Cancellation
-
-```typescript
-async function cancelableStreamingParse(
-  arrayBuffer: ArrayBuffer,
-  fileName: string,
-  maxRecords: number
-) {
-  const parser = new ParquetParserService(logger);
-  const streamResult = await parser.parseStreaming(arrayBuffer, fileName);
-
-  let processedCount = 0;
-
-  try {
-    for await (const batch of streamResult.records) {
-      for (const record of batch) {
-        await processRecord(record);
-        processedCount++;
-
-        // Cancel after reaching limit
-        if (processedCount >= maxRecords) {
-          streamResult.cancel();
-          break;
-        }
-      }
-    }
-  } catch (error) {
-    if (error.message.includes('cancelled')) {
-      console.log(`Streaming cancelled after ${processedCount} records`);
-    } else {
-      throw error;
-    }
-  }
-
-  console.log(`Processed ${processedCount} records`);
-}
-```
-
----
-
-## Memory-Efficient Processing
-
-### Pattern 1: Minimal Memory Usage
-
-```typescript
-class MemoryEfficientProcessor {
-  private parser: ParquetParserService;
-
-  constructor() {
-    this.parser = new ParquetParserService(logger, {
-      batchSize: 500,  // Smaller batches for lower memory
-      enableStreaming: true
-    });
-  }
-
-  async processWithMinimalMemory(
-    arrayBuffer: ArrayBuffer,
-    fileName: string,
-    processor: (record: any) => Promise<void>
-  ): Promise<void> {
-    const streamResult = await this.parser.parseStreaming(
-      arrayBuffer,
-      fileName
-    );
-
-    let processedCount = 0;
-
-    for await (const batch of streamResult.records) {
-      // Process and immediately release each batch
-      for (const record of batch) {
-        await processor(record);
-        processedCount++;
-
-        // Periodic garbage collection hint
-        if (processedCount % 10000 === 0) {
-          console.log(`Processed ${processedCount} records`);
-
-          // Force GC if available (run with --expose-gc)
-          if (global.gc) {
-            global.gc();
-          }
-        }
-      }
-
-      // Batch is now out of scope and can be garbage collected
-    }
-
-    const stats = await streamResult.stats;
-    console.log(`Total: ${processedCount} records`);
-    console.log(`Peak memory: ${(stats.peakMemoryUsage / 1024 / 1024).toFixed(2)} MB`);
-  }
-}
-
-// Usage
-const processor = new MemoryEfficientProcessor();
-await processor.processWithMinimalMemory(
-  arrayBuffer,
-  'large-file.parquet',
-  async (record) => {
-    // Your processing logic - don't store records
-    await sendToDatabase(record);
-  }
-);
-```
-
-### Pattern 2: Batched Database Inserts
-
-```typescript
-async function streamToDatabase(
-  arrayBuffer: ArrayBuffer,
-  fileName: string,
-  db: Database
-) {
-  const parser = new ParquetParserService(logger);
-  const streamResult = await parser.parseStreaming(arrayBuffer, fileName);
-
-  const insertBatch: any[] = [];
-  const BATCH_SIZE = 1000;
-
-  for await (const batch of streamResult.records) {
-    insertBatch.push(...batch);
-
-    // Insert when batch is full
-    if (insertBatch.length >= BATCH_SIZE) {
-      await db.insertMany('inventory', insertBatch);
-      console.log(`Inserted ${insertBatch.length} records`);
-
-      // Clear batch to free memory
-      insertBatch.length = 0;
-    }
-  }
-
-  // Insert remaining records
-  if (insertBatch.length > 0) {
-    await db.insertMany('inventory', insertBatch);
-    console.log(`Inserted final ${insertBatch.length} records`);
-  }
-
-  const stats = await streamResult.stats;
-  console.log(`Complete: ${stats.totalRecords} records processed`);
-}
-```
-
----
-
-## Validation
-
-### File Validation
-
-```typescript
-async function validateParquetFile(
-  arrayBuffer: ArrayBuffer,
-  fileName: string
-): Promise<boolean> {
-  const parser = new ParquetParserService(logger);
-
-  const validationResult = await parser.validateFile(
-    arrayBuffer,
-    fileName
-  );
-
-  if (!validationResult.isValid) {
-    console.error('❌ Validation failed:');
-    validationResult.errors.forEach(error => {
-      console.error(`   ${error.field}: ${error.message}`);
-    });
-    return false;
-  }
-
-  if (validationResult.warnings.length > 0) {
-    console.warn('⚠️  Validation warnings:');
-    validationResult.warnings.forEach(warning => {
-      console.warn(`   ${warning.field}: ${warning.message}`);
-    });
-  }
-
-  console.log('✅ Validation passed');
-  return true;
-}
-```
-
-### Schema Validation
-
-```typescript
-async function parseWithSchemaValidation(
-  arrayBuffer: ArrayBuffer,
-  fileName: string
-) {
-  const parser = new ParquetParserService(logger);
-
-  const { records, stats } = await parser.parse(
-    arrayBuffer,
-    fileName,
-    {
-      validation: {
-        enabled: true,
-        requiredColumns: [
-          'ref',
-          'productRef',
-          'locationRef',
-          'qty',
-          'status'
-        ],
-        columnTypes: {
-          ref: 'string',
-          productRef: 'string',
-          locationRef: 'string',
-          qty: 'int32',
-          status: 'string'
-        },
-        allowAdditionalColumns: true  // Allow extra columns
-      }
-    }
-  );
-
-  // Check validation results
-  if (stats.validation) {
-    if (!stats.validation.isValid) {
-      console.error('Schema validation failed:');
-      stats.validation.errors.forEach(error => {
-        console.error(`  ${error.field}: ${error.message}`);
-      });
-      throw new Error('Invalid schema');
-    }
-
-    if (stats.validation.warnings.length > 0) {
-      console.warn('Schema warnings:');
-      stats.validation.warnings.forEach(warning => {
-        console.warn(`  ${warning.field}: ${warning.message}`);
-      });
-    }
-  }
-
-  return records;
-}
-```
-
----
-
-## Column Statistics
-
-### Accessing Column Statistics
-
-```typescript
-async function analyzeColumns(arrayBuffer: ArrayBuffer, fileName: string) {
-  const parser = new ParquetParserService(logger);
-  const { records, stats } = await parser.parse(arrayBuffer, fileName);
-
-  console.log('Column Analysis:');
-  console.log('================\n');
-
-  for (const [columnName, columnStats] of Object.entries(stats.columnStats)) {
-    console.log(`Column: ${columnName}`);
-    console.log(`  Type: ${columnStats.type}`);
-    console.log(`  Total Values: ${stats.totalRecords}`);
-    console.log(`  Null Count: ${columnStats.nullCount}`);
-    console.log(`  Unique Values: ${columnStats.uniqueValues}`);
-    console.log(`  Completeness: ${((1 - columnStats.nullCount / stats.totalRecords) * 100).toFixed(2)}%`);
-
-    if (columnStats.minValue !== undefined) {
-      console.log(`  Min Value: ${columnStats.minValue}`);
-      console.log(`  Max Value: ${columnStats.maxValue}`);
-    }
-
-    console.log('');
-  }
-}
-```
-
-### Data Quality Profiling
-
-```typescript
-function profileDataQuality(stats: ParquetParsingStats) {
-  const profile = {
-    totalRecords: stats.totalRecords,
-    columns: {} as any,
-    dataQuality: {
-      completeness: 0,
-      uniqueness: 0,
-      issues: [] as string[]
-    }
-  };
-
-  let totalCompleteness = 0;
-  let totalUniqueness = 0;
-  const columnCount = Object.keys(stats.columnStats).length;
-
-  for (const [name, colStats] of Object.entries(stats.columnStats)) {
-    const completeness = 1 - (colStats.nullCount / stats.totalRecords);
-    const uniqueness = colStats.uniqueValues / stats.totalRecords;
-
-    profile.columns[name] = {
-      type: colStats.type,
-      completeness: `${(completeness * 100).toFixed(2)}%`,
-      uniqueness: `${(uniqueness * 100).toFixed(2)}%`,
-      nulls: colStats.nullCount,
-      distinct: colStats.uniqueValues
-    };
-
-    // Check for data quality issues
-    if (completeness < 0.9) {
-      profile.dataQuality.issues.push(
-        `${name}: Low completeness (${(completeness * 100).toFixed(1)}%)`
-      );
-    }
-
-    if (colStats.uniqueValues === 1 && stats.totalRecords > 100) {
-      profile.dataQuality.issues.push(
-        `${name}: All values are identical (potential data issue)`
-      );
-    }
-
-    totalCompleteness += completeness;
-    totalUniqueness += uniqueness;
-  }
-
-  profile.dataQuality.completeness =
-    `${(totalCompleteness / columnCount * 100).toFixed(2)}%`;
-  profile.dataQuality.uniqueness =
-    `${(totalUniqueness / columnCount * 100).toFixed(2)}%`;
-
-  return profile;
-}
-
-// Usage
-const { stats } = await parser.parse(arrayBuffer, fileName);
-const qualityProfile = profileDataQuality(stats);
-console.log('Data Quality Profile:', JSON.stringify(qualityProfile, null, 2));
-```
-
----
-
-## Performance Monitoring
-
-### Tracking Performance Metrics
-
-```typescript
-class ParquetPerformanceMonitor {
-  private parser: ParquetParserService;
-
-  constructor() {
-    this.parser = new ParquetParserService(logger, {
-      batchSize: 2000,
-      enableStreaming: true
-    });
-  }
-
-  async parseWithMetrics(
-    arrayBuffer: ArrayBuffer,
-    fileName: string
-  ): Promise<any> {
-    const startMemory = process.memoryUsage();
-    const startTime = Date.now();
-
-    const { records, stats } = await this.parser.parse(
-      arrayBuffer,
-      fileName
-    );
-
-    const endMemory = process.memoryUsage();
-    const endTime = Date.now();
-
-    // Calculate metrics
-    const metrics = {
-      fileName,
-      fileSize: stats.fileSizeBytes,
-      fileSizeMB: (stats.fileSizeBytes / 1024 / 1024).toFixed(2),
-      recordCount: stats.totalRecords,
-      processingTime: stats.processingTimeMs,
-      totalTime: endTime - startTime,
-      recordsPerSecond: stats.recordsPerSecond,
-      memoryUsed: endMemory.heapUsed - startMemory.heapUsed,
-      memoryUsedMB: ((endMemory.heapUsed - startMemory.heapUsed) / 1024 / 1024).toFixed(2),
-      peakMemory: stats.peakMemoryUsage,
-      peakMemoryMB: (stats.peakMemoryUsage / 1024 / 1024).toFixed(2),
-      batchesProcessed: stats.batchesProcessed
-    };
-
-    console.log('📊 Performance Metrics:');
-    console.log(`   File: ${metrics.fileName}`);
-    console.log(`   Size: ${metrics.fileSizeMB} MB`);
-    console.log(`   Records: ${metrics.recordCount.toLocaleString()}`);
-    console.log(`   Processing Time: ${metrics.processingTime}ms`);
-    console.log(`   Throughput: ${metrics.recordsPerSecond.toLocaleString()} rec/sec`);
-    console.log(`   Memory Used: ${metrics.memoryUsedMB} MB`);
-    console.log(`   Peak Memory: ${metrics.peakMemoryMB} MB`);
-    console.log(`   Batches: ${metrics.batchesProcessed}`);
-
-    return metrics;
-  }
-
-  getParserMetrics() {
-    const metrics = this.parser.getPerformanceMetrics();
-
-    console.log('🔧 Parser Statistics:');
-    console.log(`   Total Requests: ${metrics.totalRequests}`);
-    console.log(`   Successful: ${metrics.successfulRequests}`);
-    console.log(`   Failed: ${metrics.failedRequests}`);
-    console.log(`   Success Rate: ${(metrics.successfulRequests / metrics.totalRequests * 100).toFixed(2)}%`);
-    console.log(`   Avg Response Time: ${metrics.averageResponseTime.toFixed(2)}ms`);
-
-    return metrics;
-  }
-
-  resetMetrics(): void {
-    this.parser.resetMetrics();
-    console.log('✅ Performance metrics reset');
-  }
-}
-```
-
----
-
-## Key Takeaways
-
-- 🎯 **Use streaming** for files > 50 MB to avoid memory issues
-- 🎯 **Validate schema** to catch data quality issues early
-- 🎯 **Monitor statistics** to track performance and data quality
-- 🎯 **Process in batches** for memory-efficient operations
-- 🎯 **Profile columns** to understand data characteristics
-
----
-
-## Next Steps
-
-Continue to [Module 4: Writing Parquet](./03-pattern-guides-parquet-04-writing-parquet.md) to learn how to create Parquet files (using external libraries).
-
----
-
-[← Previous: Module 2 - Quick Start](.././modules/03-pattern-guides-parquet-02-quick-start.md) | [Back to Guide](../parquet-readme.md) | [Next: Module 4 - Writing Parquet →](./03-pattern-guides-parquet-04-writing-parquet.md)
-
-## Related Documentation
-
-- [Module 7: Performance Optimization](./03-pattern-guides-parquet-07-performance-optimization.md) - Advanced performance patterns
-- [Module 8: Best Practices](./03-pattern-guides-parquet-08-best-practices.md) - Production patterns and error handling
+# Module 3: Reading Parquet Files
+
+[← Back to Parquet Guide](../parquet-readme.md)
+
+**Module 3 of 8** | **Level**: Intermediate | **Time**: 20 minutes
+
+---
+
+## Overview
+
+Learn advanced Parquet reading techniques including streaming, filtering, column selection, and memory-efficient processing for large datasets.
+
+## Learning Objectives
+
+By the end of this module, you will:
+- ✅ Use streaming mode for memory-efficient processing
+- ✅ Apply validation rules and schema checking
+- ✅ Process files of any size without memory issues
+- ✅ Monitor performance metrics and optimize throughput
+- ✅ Handle errors gracefully with retry logic
+
+---
+
+## Streaming Support
+
+### When to Use Streaming
+
+**Use streaming for**:
+- Files > 50 MB
+- Memory-constrained environments
+- Real-time processing requirements
+- Unknown file sizes
+
+**Use regular parsing for**:
+- Files < 10 MB
+- When you need all records in memory
+- Simple one-time processing
+
+### Basic Streaming Example
+
+```typescript
+import {
+  ParquetParserService,
+  createConsoleLogger,
+  toStructuredLogger
+} from '@fluentcommerce/fc-connect-sdk';
+
+async function streamLargeFile(arrayBuffer: ArrayBuffer, fileName: string) {
+  const logger = toStructuredLogger(createConsoleLogger(), {
+  logLevel: 'info'
+});
+  const parser = new ParquetParserService(logger, {
+    batchSize: 1000,  // Process 1000 records at a time
+    enableStreaming: true
+  });
+
+  // Start streaming parse
+  const streamResult = await parser.parseStreaming(arrayBuffer, fileName);
+
+  let totalProcessed = 0;
+
+  // Process batches as they stream
+  for await (const batch of streamResult.records) {
+    console.log(`Processing batch of ${batch.length} records`);
+
+    // Process each record
+    for (const record of batch) {
+      // Your processing logic
+      await processRecord(record);
+    }
+
+    totalProcessed += batch.length;
+
+    // Progress logging
+    if (totalProcessed % 10000 === 0) {
+      console.log(`Processed ${totalProcessed} records...`);
+    }
+  }
+
+  // Get final statistics
+  const stats = await streamResult.stats;
+  console.log(`Complete: ${stats.totalRecords} records in ${stats.processingTimeMs}ms`);
+}
+
+async function processRecord(record: any): Promise<void> {
+  // Your business logic
+  console.log(`Processing: ${record.productRef} at ${record.locationRef}`);
+}
+```
+
+### Streaming with Cancellation
+
+```typescript
+async function cancelableStreamingParse(
+  arrayBuffer: ArrayBuffer,
+  fileName: string,
+  maxRecords: number
+) {
+  const parser = new ParquetParserService(logger);
+  const streamResult = await parser.parseStreaming(arrayBuffer, fileName);
+
+  let processedCount = 0;
+
+  try {
+    for await (const batch of streamResult.records) {
+      for (const record of batch) {
+        await processRecord(record);
+        processedCount++;
+
+        // Cancel after reaching limit
+        if (processedCount >= maxRecords) {
+          streamResult.cancel();
+          break;
+        }
+      }
+    }
+  } catch (error) {
+    if (error.message.includes('cancelled')) {
+      console.log(`Streaming cancelled after ${processedCount} records`);
+    } else {
+      throw error;
+    }
+  }
+
+  console.log(`Processed ${processedCount} records`);
+}
+```
+
+---
+
+## Memory-Efficient Processing
+
+### Pattern 1: Minimal Memory Usage
+
+```typescript
+class MemoryEfficientProcessor {
+  private parser: ParquetParserService;
+
+  constructor() {
+    this.parser = new ParquetParserService(logger, {
+      batchSize: 500,  // Smaller batches for lower memory
+      enableStreaming: true
+    });
+  }
+
+  async processWithMinimalMemory(
+    arrayBuffer: ArrayBuffer,
+    fileName: string,
+    processor: (record: any) => Promise<void>
+  ): Promise<void> {
+    const streamResult = await this.parser.parseStreaming(
+      arrayBuffer,
+      fileName
+    );
+
+    let processedCount = 0;
+
+    for await (const batch of streamResult.records) {
+      // Process and immediately release each batch
+      for (const record of batch) {
+        await processor(record);
+        processedCount++;
+
+        // Periodic garbage collection hint
+        if (processedCount % 10000 === 0) {
+          console.log(`Processed ${processedCount} records`);
+
+          // Force GC if available (run with --expose-gc)
+          if (global.gc) {
+            global.gc();
+          }
+        }
+      }
+
+      // Batch is now out of scope and can be garbage collected
+    }
+
+    const stats = await streamResult.stats;
+    console.log(`Total: ${processedCount} records`);
+    console.log(`Peak memory: ${(stats.peakMemoryUsage / 1024 / 1024).toFixed(2)} MB`);
+  }
+}
+
+// Usage
+const processor = new MemoryEfficientProcessor();
+await processor.processWithMinimalMemory(
+  arrayBuffer,
+  'large-file.parquet',
+  async (record) => {
+    // Your processing logic - don't store records
+    await sendToDatabase(record);
+  }
+);
+```
+
+### Pattern 2: Batched Database Inserts
+
+```typescript
+async function streamToDatabase(
+  arrayBuffer: ArrayBuffer,
+  fileName: string,
+  db: Database
+) {
+  const parser = new ParquetParserService(logger);
+  const streamResult = await parser.parseStreaming(arrayBuffer, fileName);
+
+  const insertBatch: any[] = [];
+  const BATCH_SIZE = 1000;
+
+  for await (const batch of streamResult.records) {
+    insertBatch.push(...batch);
+
+    // Insert when batch is full
+    if (insertBatch.length >= BATCH_SIZE) {
+      await db.insertMany('inventory', insertBatch);
+      console.log(`Inserted ${insertBatch.length} records`);
+
+      // Clear batch to free memory
+      insertBatch.length = 0;
+    }
+  }
+
+  // Insert remaining records
+  if (insertBatch.length > 0) {
+    await db.insertMany('inventory', insertBatch);
+    console.log(`Inserted final ${insertBatch.length} records`);
+  }
+
+  const stats = await streamResult.stats;
+  console.log(`Complete: ${stats.totalRecords} records processed`);
+}
+```
+
+---
+
+## Validation
+
+### File Validation
+
+```typescript
+async function validateParquetFile(
+  arrayBuffer: ArrayBuffer,
+  fileName: string
+): Promise<boolean> {
+  const parser = new ParquetParserService(logger);
+
+  const validationResult = await parser.validateFile(
+    arrayBuffer,
+    fileName
+  );
+
+  if (!validationResult.isValid) {
+    console.error('❌ Validation failed:');
+    validationResult.errors.forEach(error => {
+      console.error(`   ${error.field}: ${error.message}`);
+    });
+    return false;
+  }
+
+  if (validationResult.warnings.length > 0) {
+    console.warn('⚠️  Validation warnings:');
+    validationResult.warnings.forEach(warning => {
+      console.warn(`   ${warning.field}: ${warning.message}`);
+    });
+  }
+
+  console.log('✅ Validation passed');
+  return true;
+}
+```
+
+### Schema Validation
+
+```typescript
+async function parseWithSchemaValidation(
+  arrayBuffer: ArrayBuffer,
+  fileName: string
+) {
+  const parser = new ParquetParserService(logger);
+
+  const { records, stats } = await parser.parse(
+    arrayBuffer,
+    fileName,
+    {
+      validation: {
+        enabled: true,
+        requiredColumns: [
+          'ref',
+          'productRef',
+          'locationRef',
+          'qty',
+          'status'
+        ],
+        columnTypes: {
+          ref: 'string',
+          productRef: 'string',
+          locationRef: 'string',
+          qty: 'int32',
+          status: 'string'
+        },
+        allowAdditionalColumns: true  // Allow extra columns
+      }
+    }
+  );
+
+  // Check validation results
+  if (stats.validation) {
+    if (!stats.validation.isValid) {
+      console.error('Schema validation failed:');
+      stats.validation.errors.forEach(error => {
+        console.error(`  ${error.field}: ${error.message}`);
+      });
+      throw new Error('Invalid schema');
+    }
+
+    if (stats.validation.warnings.length > 0) {
+      console.warn('Schema warnings:');
+      stats.validation.warnings.forEach(warning => {
+        console.warn(`  ${warning.field}: ${warning.message}`);
+      });
+    }
+  }
+
+  return records;
+}
+```
+
+---
+
+## Column Statistics
+
+### Accessing Column Statistics
+
+```typescript
+async function analyzeColumns(arrayBuffer: ArrayBuffer, fileName: string) {
+  const parser = new ParquetParserService(logger);
+  const { records, stats } = await parser.parse(arrayBuffer, fileName);
+
+  console.log('Column Analysis:');
+  console.log('================\n');
+
+  for (const [columnName, columnStats] of Object.entries(stats.columnStats)) {
+    console.log(`Column: ${columnName}`);
+    console.log(`  Type: ${columnStats.type}`);
+    console.log(`  Total Values: ${stats.totalRecords}`);
+    console.log(`  Null Count: ${columnStats.nullCount}`);
+    console.log(`  Unique Values: ${columnStats.uniqueValues}`);
+    console.log(`  Completeness: ${((1 - columnStats.nullCount / stats.totalRecords) * 100).toFixed(2)}%`);
+
+    if (columnStats.minValue !== undefined) {
+      console.log(`  Min Value: ${columnStats.minValue}`);
+      console.log(`  Max Value: ${columnStats.maxValue}`);
+    }
+
+    console.log('');
+  }
+}
+```
+
+### Data Quality Profiling
+
+```typescript
+function profileDataQuality(stats: ParquetParsingStats) {
+  const profile = {
+    totalRecords: stats.totalRecords,
+    columns: {} as any,
+    dataQuality: {
+      completeness: 0,
+      uniqueness: 0,
+      issues: [] as string[]
+    }
+  };
+
+  let totalCompleteness = 0;
+  let totalUniqueness = 0;
+  const columnCount = Object.keys(stats.columnStats).length;
+
+  for (const [name, colStats] of Object.entries(stats.columnStats)) {
+    const completeness = 1 - (colStats.nullCount / stats.totalRecords);
+    const uniqueness = colStats.uniqueValues / stats.totalRecords;
+
+    profile.columns[name] = {
+      type: colStats.type,
+      completeness: `${(completeness * 100).toFixed(2)}%`,
+      uniqueness: `${(uniqueness * 100).toFixed(2)}%`,
+      nulls: colStats.nullCount,
+      distinct: colStats.uniqueValues
+    };
+
+    // Check for data quality issues
+    if (completeness < 0.9) {
+      profile.dataQuality.issues.push(
+        `${name}: Low completeness (${(completeness * 100).toFixed(1)}%)`
+      );
+    }
+
+    if (colStats.uniqueValues === 1 && stats.totalRecords > 100) {
+      profile.dataQuality.issues.push(
+        `${name}: All values are identical (potential data issue)`
+      );
+    }
+
+    totalCompleteness += completeness;
+    totalUniqueness += uniqueness;
+  }
+
+  profile.dataQuality.completeness =
+    `${(totalCompleteness / columnCount * 100).toFixed(2)}%`;
+  profile.dataQuality.uniqueness =
+    `${(totalUniqueness / columnCount * 100).toFixed(2)}%`;
+
+  return profile;
+}
+
+// Usage
+const { stats } = await parser.parse(arrayBuffer, fileName);
+const qualityProfile = profileDataQuality(stats);
+console.log('Data Quality Profile:', JSON.stringify(qualityProfile, null, 2));
+```
+
+---
+
+## Performance Monitoring
+
+### Tracking Performance Metrics
+
+```typescript
+class ParquetPerformanceMonitor {
+  private parser: ParquetParserService;
+
+  constructor() {
+    this.parser = new ParquetParserService(logger, {
+      batchSize: 2000,
+      enableStreaming: true
+    });
+  }
+
+  async parseWithMetrics(
+    arrayBuffer: ArrayBuffer,
+    fileName: string
+  ): Promise<any> {
+    const startMemory = process.memoryUsage();
+    const startTime = Date.now();
+
+    const { records, stats } = await this.parser.parse(
+      arrayBuffer,
+      fileName
+    );
+
+    const endMemory = process.memoryUsage();
+    const endTime = Date.now();
+
+    // Calculate metrics
+    const metrics = {
+      fileName,
+      fileSize: stats.fileSizeBytes,
+      fileSizeMB: (stats.fileSizeBytes / 1024 / 1024).toFixed(2),
+      recordCount: stats.totalRecords,
+      processingTime: stats.processingTimeMs,
+      totalTime: endTime - startTime,
+      recordsPerSecond: stats.recordsPerSecond,
+      memoryUsed: endMemory.heapUsed - startMemory.heapUsed,
+      memoryUsedMB: ((endMemory.heapUsed - startMemory.heapUsed) / 1024 / 1024).toFixed(2),
+      peakMemory: stats.peakMemoryUsage,
+      peakMemoryMB: (stats.peakMemoryUsage / 1024 / 1024).toFixed(2),
+      batchesProcessed: stats.batchesProcessed
+    };
+
+    console.log('📊 Performance Metrics:');
+    console.log(`   File: ${metrics.fileName}`);
+    console.log(`   Size: ${metrics.fileSizeMB} MB`);
+    console.log(`   Records: ${metrics.recordCount.toLocaleString()}`);
+    console.log(`   Processing Time: ${metrics.processingTime}ms`);
+    console.log(`   Throughput: ${metrics.recordsPerSecond.toLocaleString()} rec/sec`);
+    console.log(`   Memory Used: ${metrics.memoryUsedMB} MB`);
+    console.log(`   Peak Memory: ${metrics.peakMemoryMB} MB`);
+    console.log(`   Batches: ${metrics.batchesProcessed}`);
+
+    return metrics;
+  }
+
+  getParserMetrics() {
+    const metrics = this.parser.getPerformanceMetrics();
+
+    console.log('🔧 Parser Statistics:');
+    console.log(`   Total Requests: ${metrics.totalRequests}`);
+    console.log(`   Successful: ${metrics.successfulRequests}`);
+    console.log(`   Failed: ${metrics.failedRequests}`);
+    console.log(`   Success Rate: ${(metrics.successfulRequests / metrics.totalRequests * 100).toFixed(2)}%`);
+    console.log(`   Avg Response Time: ${metrics.averageResponseTime.toFixed(2)}ms`);
+
+    return metrics;
+  }
+
+  resetMetrics(): void {
+    this.parser.resetMetrics();
+    console.log('✅ Performance metrics reset');
+  }
+}
+```
+
+---
+
+## Key Takeaways
+
+- 🎯 **Use streaming** for files > 50 MB to avoid memory issues
+- 🎯 **Validate schema** to catch data quality issues early
+- 🎯 **Monitor statistics** to track performance and data quality
+- 🎯 **Process in batches** for memory-efficient operations
+- 🎯 **Profile columns** to understand data characteristics
+
+---
+
+## Next Steps
+
+Continue to [Module 4: Writing Parquet](./03-pattern-guides-parquet-04-writing-parquet.md) to learn how to create Parquet files (using external libraries).
+
+---
+
+[← Previous: Module 2 - Quick Start](.././modules/03-pattern-guides-parquet-02-quick-start.md) | [Back to Guide](../parquet-readme.md) | [Next: Module 4 - Writing Parquet →](./03-pattern-guides-parquet-04-writing-parquet.md)
+
+## Related Documentation
+
+- [Module 7: Performance Optimization](./03-pattern-guides-parquet-07-performance-optimization.md) - Advanced performance patterns
+- [Module 8: Best Practices](./03-pattern-guides-parquet-08-best-practices.md) - Production patterns and error handling