@fluentcommerce/fc-connect-sdk 0.1.54 → 0.1.56

package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-02-quick-start.md

@@ -1,572 +1,572 @@
-# Module 2: Quick Start
-
-[← Back to Parquet Guide](../parquet-readme.md)
-
-**Module 2 of 8** | **Level**: Beginner | **Time**: 15 minutes
-
----
-
-## Overview
-
-This module gets you up and running with ParquetParserService quickly. You'll parse your first Parquet file and understand basic usage patterns.
-
-## Learning Objectives
-
-By the end of this module, you will:
-- ✅ Install the SDK and required dependencies
-- ✅ Parse a simple Parquet file
-- ✅ Understand basic configuration options
-- ✅ View parsing statistics and performance metrics
-- ✅ Process Parquet records
-
----
-
-## Installation
-
-The ParquetParserService is included in the FC Connect SDK with all dependencies:
-
-```bash
-npm install @fluentcommerce/fc-connect-sdk
-```
-
-**What's Included**:
-- `ParquetParserService` - Main parser service
-- `hyparquet` - High-performance Parquet library (dependency)
-- `Logger` interface - Logging support (use `createConsoleLogger()` or pass your own)
-
-No additional packages required for basic Parquet parsing.
-
----
-
-## Your First Parquet Parse
-
-### Step 1: Import the Service
-
-```typescript
-import {
-  ParquetParserService,
-  createConsoleLogger,
-  toStructuredLogger
-} from '@fluentcommerce/fc-connect-sdk';
-```
-
-### Step 2: Initialize the Parser
-
-```typescript
-import {
-  ParquetParserService,
-  createConsoleLogger,
-  toStructuredLogger
-} from '@fluentcommerce/fc-connect-sdk';
-
-// Create a logger (optional but recommended)
-const logger = toStructuredLogger(createConsoleLogger(), {
-  logLevel: 'info'
-});
-
-// Create parser instance
-const parser = new ParquetParserService(logger);
-```
-
-### Step 3: Parse a Parquet File
-
-```typescript
-import * as fs from 'fs/promises';
-
-async function parseParquetFile(filePath: string) {
-  // Read file into buffer
-  const buffer = await fs.readFile(filePath);
-
-  // Convert Node.js Buffer to ArrayBuffer
-  const arrayBuffer = buffer.buffer.slice(
-    buffer.byteOffset,
-    buffer.byteOffset + buffer.byteLength
-  );
-
-  // Parse the Parquet file
-  const { records, stats } = await parser.parse(
-    arrayBuffer,
-    'inventory.parquet'  // File name for logging
-  );
-
-  // Display results
-  console.log(`✅ Parsed ${stats.totalRecords} records`);
-  console.log(`⏱️  Processing time: ${stats.processingTimeMs}ms`);
-  console.log(`🚀 Throughput: ${stats.recordsPerSecond} records/sec`);
-
-  // Access records
-  records.forEach((record, index) => {
-    console.log(`Record ${index + 1}:`, record);
-  });
-
-  return records;
-}
-
-// Usage
-parseParquetFile('./data/inventory.parquet')
-  .catch(console.error);
-```
-
-### Expected Output
-
-```
-✅ Parsed 10000 records
-⏱️  Processing time: 285ms
-🚀 Throughput: 35087 records/sec
-Record 1: { ref: 'IP-001', productRef: 'SKU-001', locationRef: 'WH-001', qty: 100 }
-Record 2: { ref: 'IP-002', productRef: 'SKU-002', locationRef: 'WH-001', qty: 50 }
-...
-```
-
----
-
-## Complete Working Example
-
-Here's a complete, runnable script:
-
-```typescript
-import {
-  ParquetParserService,
-  createConsoleLogger,
-  toStructuredLogger
-} from '@fluentcommerce/fc-connect-sdk';
-import * as fs from 'fs/promises';
-import * as path from 'path';
-
-/**
- * Simple Parquet file processor
- * Business use case: Parse inventory Parquet files from S3
- */
-async function main() {
-  console.log('🚀 Starting Parquet parser...\n');
-
-  // 1. Initialize services
-  const logger = toStructuredLogger(createConsoleLogger(), {
-    logLevel: 'info'
-  });
-  const parser = new ParquetParserService(logger);
-
-  // 2. Read Parquet file
-  const filePath = path.join(__dirname, 'data', 'inventory.parquet');
-  const buffer = await fs.readFile(filePath);
-  const arrayBuffer = buffer.buffer.slice(
-    buffer.byteOffset,
-    buffer.byteOffset + buffer.byteLength
-  );
-
-  console.log(`📂 File: ${filePath}`);
-  console.log(`📦 Size: ${(buffer.length / 1024).toFixed(2)} KB\n`);
-
-  // 3. Parse with basic options
-  const result = await parser.parse(
-    arrayBuffer,
-    'inventory.parquet',
-    {
-      batchSize: 1000  // Process in batches of 1000
-    }
-  );
-
-  // 4. Display statistics
-  console.log('📊 Parsing Statistics:');
-  console.log(`   Total Records: ${result.stats.totalRecords}`);
-  console.log(`   Processing Time: ${result.stats.processingTimeMs}ms`);
-  console.log(`   Throughput: ${result.stats.recordsPerSecond.toLocaleString()} rec/sec`);
-  console.log(`   Batches Processed: ${result.stats.batchesProcessed}`);
-  console.log(`   Peak Memory: ${(result.stats.peakMemoryUsage / 1024 / 1024).toFixed(2)} MB\n`);
-
-  // 5. Examine column statistics
-  console.log('📋 Column Statistics:');
-  for (const [columnName, columnStats] of Object.entries(result.stats.columnStats)) {
-    console.log(`   ${columnName}:`);
-    console.log(`      Type: ${columnStats.type}`);
-    console.log(`      Nulls: ${columnStats.nullCount}`);
-    console.log(`      Unique Values: ${columnStats.uniqueValues}`);
-
-    if (columnStats.minValue !== undefined) {
-      console.log(`      Min: ${columnStats.minValue}`);
-      console.log(`      Max: ${columnStats.maxValue}`);
-    }
-  }
-
-  // 6. Process first few records
-  console.log('\n📝 Sample Records (first 5):');
-  result.records.slice(0, 5).forEach((record, index) => {
-    console.log(`   ${index + 1}. ${JSON.stringify(record)}`);
-  });
-
-  console.log('\n✅ Complete!');
-}
-
-// Run the script
-main().catch((error) => {
-  console.error('❌ Error:', error.message);
-  process.exit(1);
-});
-```
-
----
-
-## Configuration Options
-
-### Basic Configuration
-
-```typescript
-const parser = new ParquetParserService(logger, {
-  batchSize: 1000,                      // Records per batch
-  maxFileSize: 100 * 1024 * 1024,       // 100 MB max file size
-  enableStreaming: true                  // Enable streaming mode
-});
-```
-
-### Parsing Options
-
-```typescript
-const { records, stats } = await parser.parse(
-  arrayBuffer,
-  'inventory.parquet',
-  {
-    // Batch size for this specific parse operation
-    batchSize: 500,
-
-    // Validation options
-    validation: {
-      enabled: true,
-      requiredColumns: ['productRef', 'locationRef', 'qty'],
-      columnTypes: {
-        productRef: 'string',
-        locationRef: 'string',
-        qty: 'int32'
-      },
-      allowAdditionalColumns: true
-    }
-  }
-);
-```
-
-### Configuration Reference
-
-| Option | Type | Default | Description |
-|--------|------|---------|-------------|
-| `batchSize` | number | 1000 | Records processed per batch |
-| `maxFileSize` | number | 100 MB | Maximum file size allowed |
-| `enableStreaming` | boolean | true | Enable streaming mode |
-| `validation.enabled` | boolean | false | Enable validation |
-| `validation.requiredColumns` | string[] | [] | Required column names |
-| `validation.columnTypes` | object | {} | Expected column types |
-| `validation.allowAdditionalColumns` | boolean | true | Allow extra columns |
-
----
-
-## Understanding Parse Results
-
-### Result Structure
-
-```typescript
-interface ParquetParseResult {
-  records: any[];           // Parsed records
-  stats: ParsingStats;      // Statistics and metrics
-}
-
-interface ParsingStats {
-  // Core metrics
-  totalRecords: number;
-  processingTimeMs: number;
-  recordsPerSecond: number;
-  fileSizeBytes: number;
-
-  // Processing info
-  batchesProcessed: number;
-  peakMemoryUsage: number;
-
-  // Column statistics
-  columnStats: {
-    [columnName: string]: {
-      type: string;
-      nullCount: number;
-      uniqueValues: number;
-      minValue?: any;
-      maxValue?: any;
-    }
-  };
-
-  // Validation results (if enabled)
-  validation?: {
-    isValid: boolean;
-    errors: Array<{ field: string; message: string }>;
-    warnings: Array<{ field: string; message: string }>;
-  };
-}
-```
-
-### Accessing Statistics
-
-```typescript
-const { records, stats } = await parser.parse(arrayBuffer, fileName);
-
-// Performance metrics
-console.log(`Processed ${stats.totalRecords} records in ${stats.processingTimeMs}ms`);
-console.log(`Throughput: ${stats.recordsPerSecond} records/sec`);
-
-// Memory usage
-console.log(`Peak memory: ${stats.peakMemoryUsage / 1024 / 1024}MB`);
-
-// Column information
-Object.keys(stats.columnStats).forEach(columnName => {
-  const col = stats.columnStats[columnName];
-  console.log(`${columnName}: ${col.type}, ${col.nullCount} nulls`);
-});
-```
-
----
-
-## Working with Test Files
-
-The SDK includes test files for development:
-
-### Available Test Files
-
-Located in `fc-connect-sdk/tests/fixtures/parquet/`:
-
-```typescript
-import * as path from 'path';
-
-// Test file patterns
-const testFiles = {
-  // Small test files
-  tiny: 'single-record.parquet',           // 1 record, 1.8 KB
-  small: 'inventory-position-small.parquet', // 50 records, 3-10 KB
-
-  // Medium test files
-  medium: 'inventory-position-medium.parquet', // 1,000 records, 170 KB
-
-  // Large test files
-  large: 'inventory-position-large.parquet',   // 10,000 records, 1.7 MB
-  xlarge: 'inventory-position-50k.parquet',    // 50,000 records, 8.5 MB
-  xxlarge: 'inventory-position-100k.parquet',  // 100,000 records, 17 MB
-
-  // Performance test files
-  stress: 'inventory-position-500k.parquet',   // 500,000 records, 85 MB
-  max: 'mixed-positions-1m.parquet'            // 1,000,000 records, 169 MB
-};
-
-// Load a test file
-const filePath = path.join(
-  'node_modules/@fluentcommerce/fc-connect-sdk/tests/fixtures/parquet',
-  testFiles.medium
-);
-```
-
-### Test File Structure
-
-All test files contain inventory-like data:
-
-```typescript
-// Record structure in test files
-{
-  ref: string;          // Position reference (e.g., "IP-001")
-  productRef: string;   // Product SKU (e.g., "SKU-001")
-  locationRef: string;  // Location code (e.g., "WH-001")
-  qty: number;          // Quantity (integer)
-  status?: string;      // Status (optional)
-}
-```
-
----
-
-## Common Patterns
-
-### Pattern 1: Basic File Processing
-
-```typescript
-async function processParquetFile(filePath: string) {
-  const parser = new ParquetParserService(logger);
-
-  const buffer = await fs.readFile(filePath);
-  const arrayBuffer = buffer.buffer.slice(
-    buffer.byteOffset,
-    buffer.byteOffset + buffer.byteLength
-  );
-
-  const { records, stats } = await parser.parse(arrayBuffer, filePath);
-
-  console.log(`Processed ${stats.totalRecords} records`);
-  return records;
-}
-```
-
-### Pattern 2: Processing with Validation
-
-```typescript
-async function processWithValidation(filePath: string) {
-  const parser = new ParquetParserService(logger);
-
-  const buffer = await fs.readFile(filePath);
-  const arrayBuffer = buffer.buffer.slice(
-    buffer.byteOffset,
-    buffer.byteOffset + buffer.byteLength
-  );
-
-  const { records, stats } = await parser.parse(
-    arrayBuffer,
-    filePath,
-    {
-      validation: {
-        enabled: true,
-        requiredColumns: ['productRef', 'locationRef', 'qty'],
-        columnTypes: {
-          productRef: 'string',
-          locationRef: 'string',
-          qty: 'int32'
-        }
-      }
-    }
-  );
-
-  // Check validation
-  if (stats.validation && !stats.validation.isValid) {
-    console.error('Validation failed:');
-    stats.validation.errors.forEach(err => {
-      console.error(`  ${err.field}: ${err.message}`);
-    });
-    throw new Error('Parquet validation failed');
-  }
-
-  return records;
-}
-```
-
-### Pattern 3: Batch Processing
-
-```typescript
-async function processBatched(filePath: string, callback: (batch: any[]) => Promise<void>) {
-  const parser = new ParquetParserService(logger, {
-    batchSize: 500  // Small batches for controlled processing
-  });
-
-  const buffer = await fs.readFile(filePath);
-  const arrayBuffer = buffer.buffer.slice(
-    buffer.byteOffset,
-    buffer.byteOffset + buffer.byteLength
-  );
-
-  // Use streaming mode for large files
-  const streamResult = await parser.parseStreaming(arrayBuffer, filePath);
-
-  for await (const batch of streamResult.records) {
-    console.log(`Processing batch of ${batch.length} records`);
-    await callback(batch);
-  }
-
-  const stats = await streamResult.stats;
-  console.log(`Total: ${stats.totalRecords} records processed`);
-}
-```
-
----
-
-## Error Handling
-
-### Basic Error Handling
-
-```typescript
-async function safeParseParquet(filePath: string) {
-  const parser = new ParquetParserService(logger);
-
-  try {
-    const buffer = await fs.readFile(filePath);
-    const arrayBuffer = buffer.buffer.slice(
-      buffer.byteOffset,
-      buffer.byteOffset + buffer.byteLength
-    );
-
-    const { records, stats } = await parser.parse(arrayBuffer, filePath);
-
-    console.log(`✅ Success: ${stats.totalRecords} records`);
-    return records;
-
-  } catch (error: any) {
-    console.error(`❌ Parse failed:`, error.message);
-
-    // Check error type
-    if (error.code === 'PARSING_ERROR') {
-      console.error('Invalid Parquet file format');
-    } else if (error.code === 'VALIDATION_ERROR') {
-      console.error('Schema validation failed');
-    } else {
-      console.error('Unexpected error:', error);
-    }
-
-    throw error;
-  }
-}
-```
-
----
-
-## Performance Tips
-
-### 1. Choose Appropriate Batch Size
-
-```typescript
-// Small files (< 10 MB) - larger batches
-const parser = new ParquetParserService(logger, {
-  batchSize: 5000
-});
-
-// Large files (> 100 MB) - smaller batches
-const parser = new ParquetParserService(logger, {
-  batchSize: 500
-});
-```
-
-### 2. Enable Streaming for Large Files
-
-```typescript
-// For files > 50 MB, use streaming
-if (fileSize > 50 * 1024 * 1024) {
-  const streamResult = await parser.parseStreaming(arrayBuffer, fileName);
-
-  for await (const batch of streamResult.records) {
-    // Process batch
-  }
-}
-```
-
-### 3. Disable Validation for Performance
-
-```typescript
-// Skip validation for speed (production with trusted sources)
-const { records, stats } = await parser.parse(
-  arrayBuffer,
-  fileName,
-  {
-    validation: { enabled: false }  // Faster processing
-  }
-);
-```
-
----
-
-## Key Takeaways
-
-- 🎯 **Simple API** - Just `parse(arrayBuffer, fileName)`
-- 🎯 **Rich statistics** - Performance metrics, column stats, validation results
-- 🎯 **Flexible configuration** - Batch size, validation, streaming
-- 🎯 **Error handling** - Built-in error codes for different failure types
-- 🎯 **Test files available** - Use SDK test fixtures for development
-
----
-
-## Next Steps
-
-Continue to [Module 3: Reading Parquet](./03-pattern-guides-parquet-03-reading-parquet.md) to learn advanced parsing techniques including streaming, filtering, and memory optimization.
-
----
-
-[← Previous: Module 1 - Introduction](.././modules/03-pattern-guides-parquet-01-introduction.md) | [Back to Guide](../parquet-readme.md) | [Next: Module 3 - Reading Parquet →](./03-pattern-guides-parquet-03-reading-parquet.md)
-
-## Related Documentation
-
-- [Module 3: Reading Parquet](./03-pattern-guides-parquet-03-reading-parquet.md) - Advanced reading patterns
-- [Module 8: Best Practices](./03-pattern-guides-parquet-08-best-practices.md) - Production patterns
-- [Data Sources Guide](../../../02-CORE-GUIDES/data-sources/data-sources-readme.md) - S3 integration
+# Module 2: Quick Start
+
+[← Back to Parquet Guide](../parquet-readme.md)
+
+**Module 2 of 8** | **Level**: Beginner | **Time**: 15 minutes
+
+---
+
+## Overview
+
+This module gets you up and running with ParquetParserService quickly. You'll parse your first Parquet file and understand basic usage patterns.
+
+## Learning Objectives
+
+By the end of this module, you will:
+- ✅ Install the SDK and required dependencies
+- ✅ Parse a simple Parquet file
+- ✅ Understand basic configuration options
+- ✅ View parsing statistics and performance metrics
+- ✅ Process Parquet records
+
+---
+
+## Installation
+
+The ParquetParserService is included in the FC Connect SDK with all dependencies:
+
+```bash
+npm install @fluentcommerce/fc-connect-sdk
+```
+
+**What's Included**:
+- `ParquetParserService` - Main parser service
+- `hyparquet` - High-performance Parquet library (dependency)
+- `Logger` interface - Logging support (use `createConsoleLogger()` or pass your own)
+
+No additional packages required for basic Parquet parsing.
+
+---
+
+## Your First Parquet Parse
+
+### Step 1: Import the Service
+
+```typescript
+import {
+  ParquetParserService,
+  createConsoleLogger,
+  toStructuredLogger
+} from '@fluentcommerce/fc-connect-sdk';
+```
+
+### Step 2: Initialize the Parser
+
+```typescript
+import {
+  ParquetParserService,
+  createConsoleLogger,
+  toStructuredLogger
+} from '@fluentcommerce/fc-connect-sdk';
+
+// Create a logger (optional but recommended)
+const logger = toStructuredLogger(createConsoleLogger(), {
+  logLevel: 'info'
+});
+
+// Create parser instance
+const parser = new ParquetParserService(logger);
+```
+
+### Step 3: Parse a Parquet File
+
+```typescript
+import * as fs from 'fs/promises';
+
+async function parseParquetFile(filePath: string) {
+  // Read file into buffer
+  const buffer = await fs.readFile(filePath);
+
+  // Convert Node.js Buffer to ArrayBuffer
+  const arrayBuffer = buffer.buffer.slice(
+    buffer.byteOffset,
+    buffer.byteOffset + buffer.byteLength
+  );
+
+  // Parse the Parquet file
+  const { records, stats } = await parser.parse(
+    arrayBuffer,
+    'inventory.parquet'  // File name for logging
+  );
+
+  // Display results
+  console.log(`✅ Parsed ${stats.totalRecords} records`);
+  console.log(`⏱️  Processing time: ${stats.processingTimeMs}ms`);
+  console.log(`🚀 Throughput: ${stats.recordsPerSecond} records/sec`);
+
+  // Access records
+  records.forEach((record, index) => {
+    console.log(`Record ${index + 1}:`, record);
+  });
+
+  return records;
+}
+
+// Usage
+parseParquetFile('./data/inventory.parquet')
+  .catch(console.error);
+```
+
+### Expected Output
+
+```
+✅ Parsed 10000 records
+⏱️  Processing time: 285ms
+🚀 Throughput: 35087 records/sec
+Record 1: { ref: 'IP-001', productRef: 'SKU-001', locationRef: 'WH-001', qty: 100 }
+Record 2: { ref: 'IP-002', productRef: 'SKU-002', locationRef: 'WH-001', qty: 50 }
+...
+```
+
+---
+
+## Complete Working Example
+
+Here's a complete, runnable script:
+
+```typescript
+import {
+  ParquetParserService,
+  createConsoleLogger,
+  toStructuredLogger
+} from '@fluentcommerce/fc-connect-sdk';
+import * as fs from 'fs/promises';
+import * as path from 'path';
+
+/**
+ * Simple Parquet file processor
+ * Business use case: Parse inventory Parquet files from S3
+ */
+async function main() {
+  console.log('🚀 Starting Parquet parser...\n');
+
+  // 1. Initialize services
+  const logger = toStructuredLogger(createConsoleLogger(), {
+    logLevel: 'info'
+  });
+  const parser = new ParquetParserService(logger);
+
+  // 2. Read Parquet file
+  const filePath = path.join(__dirname, 'data', 'inventory.parquet');
+  const buffer = await fs.readFile(filePath);
+  const arrayBuffer = buffer.buffer.slice(
+    buffer.byteOffset,
+    buffer.byteOffset + buffer.byteLength
+  );
+
+  console.log(`📂 File: ${filePath}`);
+  console.log(`📦 Size: ${(buffer.length / 1024).toFixed(2)} KB\n`);
+
+  // 3. Parse with basic options
+  const result = await parser.parse(
+    arrayBuffer,
+    'inventory.parquet',
+    {
+      batchSize: 1000  // Process in batches of 1000
+    }
+  );
+
+  // 4. Display statistics
+  console.log('📊 Parsing Statistics:');
+  console.log(`   Total Records: ${result.stats.totalRecords}`);
+  console.log(`   Processing Time: ${result.stats.processingTimeMs}ms`);
+  console.log(`   Throughput: ${result.stats.recordsPerSecond.toLocaleString()} rec/sec`);
+  console.log(`   Batches Processed: ${result.stats.batchesProcessed}`);
+  console.log(`   Peak Memory: ${(result.stats.peakMemoryUsage / 1024 / 1024).toFixed(2)} MB\n`);
+
+  // 5. Examine column statistics
+  console.log('📋 Column Statistics:');
+  for (const [columnName, columnStats] of Object.entries(result.stats.columnStats)) {
+    console.log(`   ${columnName}:`);
+    console.log(`      Type: ${columnStats.type}`);
+    console.log(`      Nulls: ${columnStats.nullCount}`);
+    console.log(`      Unique Values: ${columnStats.uniqueValues}`);
+
+    if (columnStats.minValue !== undefined) {
+      console.log(`      Min: ${columnStats.minValue}`);
+      console.log(`      Max: ${columnStats.maxValue}`);
+    }
+  }
+
+  // 6. Process first few records
+  console.log('\n📝 Sample Records (first 5):');
+  result.records.slice(0, 5).forEach((record, index) => {
+    console.log(`   ${index + 1}. ${JSON.stringify(record)}`);
+  });
+
+  console.log('\n✅ Complete!');
+}
+
+// Run the script
+main().catch((error) => {
+  console.error('❌ Error:', error.message);
+  process.exit(1);
+});
+```
+
+---
+
+## Configuration Options
+
+### Basic Configuration
+
+```typescript
+const parser = new ParquetParserService(logger, {
+  batchSize: 1000,                      // Records per batch
+  maxFileSize: 100 * 1024 * 1024,       // 100 MB max file size
+  enableStreaming: true                  // Enable streaming mode
+});
+```
+
+### Parsing Options
+
+```typescript
+const { records, stats } = await parser.parse(
+  arrayBuffer,
+  'inventory.parquet',
+  {
+    // Batch size for this specific parse operation
+    batchSize: 500,
+
+    // Validation options
+    validation: {
+      enabled: true,
+      requiredColumns: ['productRef', 'locationRef', 'qty'],
+      columnTypes: {
+        productRef: 'string',
+        locationRef: 'string',
+        qty: 'int32'
+      },
+      allowAdditionalColumns: true
+    }
+  }
+);
+```
+
+### Configuration Reference
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `batchSize` | number | 1000 | Records processed per batch |
+| `maxFileSize` | number | 100 MB | Maximum file size allowed |
+| `enableStreaming` | boolean | true | Enable streaming mode |
+| `validation.enabled` | boolean | false | Enable validation |
+| `validation.requiredColumns` | string[] | [] | Required column names |
+| `validation.columnTypes` | object | {} | Expected column types |
+| `validation.allowAdditionalColumns` | boolean | true | Allow extra columns |
+
+---
+
+## Understanding Parse Results
+
+### Result Structure
+
+```typescript
+interface ParquetParseResult {
+  records: any[];           // Parsed records
+  stats: ParsingStats;      // Statistics and metrics
+}
+
+interface ParsingStats {
+  // Core metrics
+  totalRecords: number;
+  processingTimeMs: number;
+  recordsPerSecond: number;
+  fileSizeBytes: number;
+
+  // Processing info
+  batchesProcessed: number;
+  peakMemoryUsage: number;
+
+  // Column statistics
+  columnStats: {
+    [columnName: string]: {
+      type: string;
+      nullCount: number;
+      uniqueValues: number;
+      minValue?: any;
+      maxValue?: any;
+    }
+  };
+
+  // Validation results (if enabled)
+  validation?: {
+    isValid: boolean;
+    errors: Array<{ field: string; message: string }>;
+    warnings: Array<{ field: string; message: string }>;
+  };
+}
+```
+
+### Accessing Statistics
+
+```typescript
+const { records, stats } = await parser.parse(arrayBuffer, fileName);
+
+// Performance metrics
+console.log(`Processed ${stats.totalRecords} records in ${stats.processingTimeMs}ms`);
+console.log(`Throughput: ${stats.recordsPerSecond} records/sec`);
+
+// Memory usage
+console.log(`Peak memory: ${stats.peakMemoryUsage / 1024 / 1024}MB`);
+
+// Column information
+Object.keys(stats.columnStats).forEach(columnName => {
+  const col = stats.columnStats[columnName];
+  console.log(`${columnName}: ${col.type}, ${col.nullCount} nulls`);
+});
+```
+
+---
+
+## Working with Test Files
+
+The SDK includes test files for development:
+
+### Available Test Files
+
+Located in `fc-connect-sdk/tests/fixtures/parquet/`:
+
+```typescript
+import * as path from 'path';
+
+// Test file patterns
+const testFiles = {
+  // Small test files
+  tiny: 'single-record.parquet',           // 1 record, 1.8 KB
+  small: 'inventory-position-small.parquet', // 50 records, 3-10 KB
+
+  // Medium test files
+  medium: 'inventory-position-medium.parquet', // 1,000 records, 170 KB
+
+  // Large test files
+  large: 'inventory-position-large.parquet',   // 10,000 records, 1.7 MB
+  xlarge: 'inventory-position-50k.parquet',    // 50,000 records, 8.5 MB
+  xxlarge: 'inventory-position-100k.parquet',  // 100,000 records, 17 MB
+
+  // Performance test files
+  stress: 'inventory-position-500k.parquet',   // 500,000 records, 85 MB
+  max: 'mixed-positions-1m.parquet'            // 1,000,000 records, 169 MB
+};
+
+// Load a test file
+const filePath = path.join(
+  'node_modules/@fluentcommerce/fc-connect-sdk/tests/fixtures/parquet',
+  testFiles.medium
+);
+```
+
+### Test File Structure
+
+All test files contain inventory-like data:
+
+```typescript
+// Record structure in test files
+{
+  ref: string;          // Position reference (e.g., "IP-001")
+  productRef: string;   // Product SKU (e.g., "SKU-001")
+  locationRef: string;  // Location code (e.g., "WH-001")
+  qty: number;          // Quantity (integer)
+  status?: string;      // Status (optional)
+}
+```
+
+---
+
+## Common Patterns
+
+### Pattern 1: Basic File Processing
+
+```typescript
+async function processParquetFile(filePath: string) {
+  const parser = new ParquetParserService(logger);
+
+  const buffer = await fs.readFile(filePath);
+  const arrayBuffer = buffer.buffer.slice(
+    buffer.byteOffset,
+    buffer.byteOffset + buffer.byteLength
+  );
+
+  const { records, stats } = await parser.parse(arrayBuffer, filePath);
+
+  console.log(`Processed ${stats.totalRecords} records`);
+  return records;
+}
+```
+
+### Pattern 2: Processing with Validation
+
+```typescript
+async function processWithValidation(filePath: string) {
+  const parser = new ParquetParserService(logger);
+
+  const buffer = await fs.readFile(filePath);
+  const arrayBuffer = buffer.buffer.slice(
+    buffer.byteOffset,
+    buffer.byteOffset + buffer.byteLength
+  );
+
+  const { records, stats } = await parser.parse(
+    arrayBuffer,
+    filePath,
+    {
+      validation: {
+        enabled: true,
+        requiredColumns: ['productRef', 'locationRef', 'qty'],
+        columnTypes: {
+          productRef: 'string',
+          locationRef: 'string',
+          qty: 'int32'
+        }
+      }
+    }
+  );
+
+  // Check validation
+  if (stats.validation && !stats.validation.isValid) {
+    console.error('Validation failed:');
+    stats.validation.errors.forEach(err => {
+      console.error(`  ${err.field}: ${err.message}`);
+    });
+    throw new Error('Parquet validation failed');
+  }
+
+  return records;
+}
+```
+
+### Pattern 3: Batch Processing
+
+```typescript
+async function processBatched(filePath: string, callback: (batch: any[]) => Promise<void>) {
+  const parser = new ParquetParserService(logger, {
+    batchSize: 500  // Small batches for controlled processing
+  });
+
+  const buffer = await fs.readFile(filePath);
+  const arrayBuffer = buffer.buffer.slice(
+    buffer.byteOffset,
+    buffer.byteOffset + buffer.byteLength
+  );
+
+  // Use streaming mode for large files
+  const streamResult = await parser.parseStreaming(arrayBuffer, filePath);
+
+  for await (const batch of streamResult.records) {
+    console.log(`Processing batch of ${batch.length} records`);
+    await callback(batch);
+  }
+
+  const stats = await streamResult.stats;
+  console.log(`Total: ${stats.totalRecords} records processed`);
+}
+```
+
+---
+
+## Error Handling
+
+### Basic Error Handling
+
+```typescript
+async function safeParseParquet(filePath: string) {
+  const parser = new ParquetParserService(logger);
+
+  try {
+    const buffer = await fs.readFile(filePath);
+    const arrayBuffer = buffer.buffer.slice(
+      buffer.byteOffset,
+      buffer.byteOffset + buffer.byteLength
+    );
+
+    const { records, stats } = await parser.parse(arrayBuffer, filePath);
+
+    console.log(`✅ Success: ${stats.totalRecords} records`);
+    return records;
+
+  } catch (error: any) {
+    console.error(`❌ Parse failed:`, error.message);
+
+    // Check error type
+    if (error.code === 'PARSING_ERROR') {
+      console.error('Invalid Parquet file format');
+    } else if (error.code === 'VALIDATION_ERROR') {
+      console.error('Schema validation failed');
+    } else {
+      console.error('Unexpected error:', error);
+    }
+
+    throw error;
+  }
+}
+```
+
+---
+
+## Performance Tips
+
+### 1. Choose Appropriate Batch Size
+
+```typescript
+// Small files (< 10 MB) - larger batches
+const parser = new ParquetParserService(logger, {
+  batchSize: 5000
+});
+
+// Large files (> 100 MB) - smaller batches
+const parser = new ParquetParserService(logger, {
+  batchSize: 500
+});
+```
+
+### 2. Enable Streaming for Large Files
+
+```typescript
+// For files > 50 MB, use streaming
+if (fileSize > 50 * 1024 * 1024) {
+  const streamResult = await parser.parseStreaming(arrayBuffer, fileName);
+
+  for await (const batch of streamResult.records) {
+    // Process batch
+  }
+}
+```
+
+### 3. Disable Validation for Performance
+
+```typescript
+// Skip validation for speed (production with trusted sources)
+const { records, stats } = await parser.parse(
+  arrayBuffer,
+  fileName,
+  {
+    validation: { enabled: false }  // Faster processing
+  }
+);
+```
+
+---
+
+## Key Takeaways
+
+- 🎯 **Simple API** - Just `parse(arrayBuffer, fileName)`
+- 🎯 **Rich statistics** - Performance metrics, column stats, validation results
+- 🎯 **Flexible configuration** - Batch size, validation, streaming
+- 🎯 **Error handling** - Built-in error codes for different failure types
+- 🎯 **Test files available** - Use SDK test fixtures for development
+
+---
+
+## Next Steps
+
+Continue to [Module 3: Reading Parquet](./03-pattern-guides-parquet-03-reading-parquet.md) to learn advanced parsing techniques including streaming, filtering, and memory optimization.
+
+---
+
+[← Previous: Module 1 - Introduction](.././modules/03-pattern-guides-parquet-01-introduction.md) | [Back to Guide](../parquet-readme.md) | [Next: Module 3 - Reading Parquet →](./03-pattern-guides-parquet-03-reading-parquet.md)
+
+## Related Documentation
+
+- [Module 3: Reading Parquet](./03-pattern-guides-parquet-03-reading-parquet.md) - Advanced reading patterns
+- [Module 8: Best Practices](./03-pattern-guides-parquet-08-best-practices.md) - Production patterns
+- [Data Sources Guide](../../../02-CORE-GUIDES/data-sources/data-sources-readme.md) - S3 integration