npm - @fluentcommerce/fc-connect-sdk - Versions diffs - 0.1.54 → 0.1.56 - Mend

@fluentcommerce/fc-connect-sdk 0.1.54 → 0.1.56

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (476) hide show

package/docs/03-PATTERN-GUIDES/parquet/modules/03-pattern-guides-parquet-07-performance-optimization.md CHANGED Viewed

@@ -1,525 +1,525 @@
-# Module 7: Performance Optimization
-[← Back to Parquet Guide](../parquet-readme.md)
-**Module 7 of 8** | **Level**: Advanced | **Time**: 25 minutes
----
-## Overview
-Learn advanced performance optimization techniques for processing large-scale Parquet datasets efficiently.
-## Learning Objectives
-By the end of this module, you will:
-- ✅ Optimize batch sizes for different file sizes
-- ✅ Implement parallel processing strategies
-- ✅ Use partitioning for efficient queries
-- ✅ Monitor and track performance metrics
-- ✅ Tune compression and row group settings
----
-## Performance Benchmarks
-### SDK Parser Performance
-Based on test files in `fc-connect-sdk/tests/fixtures/parquet/`:
-| File Size | Records | Processing Time | Throughput | Memory Usage |
-|-----------|---------|-----------------|------------|--------------|
-| **10 KB** | 50 | < 10 ms | 5,000+ rec/sec | ~10 MB |
-| **170 KB** | 1,000 | 50-100 ms | 20,000 rec/sec | ~20 MB |
-| **1.7 MB** | 10,000 | 200-300 ms | 35,000 rec/sec | ~30 MB |
-| **8.5 MB** | 50,000 | 800ms-1s | 50,000 rec/sec | ~50 MB |
-| **17 MB** | 100,000 | 1.5-2 sec | 55,000 rec/sec | ~70 MB |
-| **85 MB** | 500,000 | 6-8 sec | 65,000 rec/sec | ~150 MB |
-| **169 MB** | 1,000,000 | 12-15 sec | 70,000 rec/sec | ~200 MB |
-**Hardware**: Node.js runtime, 8GB RAM, standard SSD
----
-## Batch Size Optimization
-### Choosing the Right Batch Size
-```typescript
-/**
- * Dynamic batch size based on file size
- */
-function getOptimalBatchSize(fileSizeBytes: number): number {
-  const sizeMB = fileSizeBytes / 1024 / 1024;
-  if (sizeMB < 1) {
-    return 5000;   // Small files: larger batches
-  } else if (sizeMB < 10) {
-    return 2000;   // Medium files: balanced
-  } else if (sizeMB < 50) {
-    return 1000;   // Large files: standard
-  } else {
-    return 500;    // Very large files: smaller batches
-  }
-}
-async function parseWithOptimalBatchSize(
-  arrayBuffer: ArrayBuffer,
-  fileName: string,
-  fileSizeBytes: number
-) {
-  const batchSize = getOptimalBatchSize(fileSizeBytes);
-  const parser = new ParquetParserService(logger, {
-    batchSize,
-    enableStreaming: fileSizeBytes > 50 * 1024 * 1024  // Stream if > 50MB
-  });
-  console.log(`Using batch size: ${batchSize}`);
-  if (parser.enableStreaming) {
-    const streamResult = await parser.parseStreaming(arrayBuffer, fileName);
-    for await (const batch of streamResult.records) {
-      await processBatch(batch);
-    }
-    return await streamResult.stats;
-  } else {
-    return await parser.parse(arrayBuffer, fileName);
-  }
-}
-```
----
-## Parallel Processing
-### Process Multiple Files Concurrently
-```typescript
-/**
- * Process multiple Parquet files in parallel
- * Business use case: Batch processing of daily extracts
- */
-class ParallelParquetProcessor {
-  private parser: ParquetParserService;
-  private concurrency: number;
-  constructor(concurrency: number = 3) {
-    this.parser = new ParquetParserService(logger, {
-      batchSize: 2000,
-      enableStreaming: true
-    });
-    this.concurrency = concurrency;
-  }
-  async processFiles(
-    files: Array<{ bucket: string; key: string }>
-  ): Promise<void> {
-    const queue = [...files];
-    const workers: Promise<void>[] = [];
-    console.log(`Processing ${files.length} files with ${this.concurrency} workers`);
-    // Start workers
-    for (let i = 0; i < this.concurrency; i++) {
-      workers.push(this.worker(queue, i + 1));
-    }
-    // Wait for all workers to complete
-    await Promise.all(workers);
-    // Get final metrics
-    const metrics = this.parser.getPerformanceMetrics();
-    console.log(`\n✅ Batch processing complete:`);
-    console.log(`   Total files: ${files.length}`);
-    console.log(`   Success rate: ${(metrics.successfulRequests / metrics.totalRequests * 100).toFixed(2)}%`);
-    console.log(`   Avg processing time: ${metrics.averageResponseTime.toFixed(2)}ms`);
-  }
-  private async worker(
-    queue: Array<{ bucket: string; key: string }>,
-    workerId: number
-  ): Promise<void> {
-    while (queue.length > 0) {
-      const file = queue.shift();
-      if (!file) break;
-      try {
-        console.log(`[Worker ${workerId}] Processing ${file.key}...`);
-        const buffer = await s3.downloadFile(file.key, { encoding: 'binary' });
-        const arrayBuffer = buffer.buffer.slice(
-          buffer.byteOffset,
-          buffer.byteOffset + buffer.byteLength
-        );
-        const startTime = Date.now();
-        const streamResult = await this.parser.parseStreaming(
-          arrayBuffer,
-          file.key,
-          { batchSize: 5000 }
-        );
-        for await (const batch of streamResult.records) {
-          await this.processBatch(batch);
-        }
-        const stats = await streamResult.stats;
-        const duration = Date.now() - startTime;
-        console.log(`[Worker ${workerId}] ✅ ${file.key}: ${stats.totalRecords} records in ${duration}ms`);
-      } catch (error) {
-        console.error(`[Worker ${workerId}] ❌ Failed ${file.key}:`, error.message);
-      }
-    }
-    console.log(`[Worker ${workerId}] Finished`);
-  }
-  private async processBatch(batch: any[]): Promise<void> {
-    // Your processing logic
-    // Parallel batch processing within each worker
-    await Promise.all(
-      batch.map(record => this.processRecord(record))
-    );
-  }
-  private async processRecord(record: any): Promise<void> {
-    // Your business logic
-  }
-}
-// Usage
-const processor = new ParallelParquetProcessor(3);  // 3 concurrent workers
-await processor.processFiles([
-  { bucket: 'data-lake', key: 'inventory/2025-10-18.parquet' },
-  { bucket: 'data-lake', key: 'inventory/2025-10-19.parquet' },
-  { bucket: 'data-lake', key: 'inventory/2025-10-20.parquet' }
-]);
-```
----
-## Partitioning Strategies
-### Time-Based Partitioning
-```typescript
-import { Buffer } from 'node:buffer'; // Required for Deno/Versori
-/**
- * Generate time-based partitions for efficient queries
- * Business use case: Daily snapshots for analytics
- */
-class PartitionedExtractor {
-  async extractWithTimePartitioning(
-    outputDir: string,
-    startDate: Date,
-    endDate: Date
-  ): Promise<void> {
-    const dates = this.getDateRange(startDate, endDate);
-    for (const date of dates) {
-      // Hive-style partitioning: year=YYYY/month=MM/day=DD
-      const partition = `year=${date.year}/month=${date.month}/day=${date.day}`;
-      const outputPath = `${outputDir}/${partition}/data.parquet`;
-      console.log(`\nExtracting partition: ${partition}`);
-      const schema = new parquet.ParquetSchema({
-        ref: { type: 'UTF8' },
-        productRef: { type: 'UTF8' },
-        locationRef: { type: 'UTF8' },
-        qty: { type: 'INT32' },
-        updatedOn: { type: 'TIMESTAMP_MILLIS' }
-      });
-      const writer = await parquet.ParquetWriter.openFile(schema, outputPath, {
-        compression: 'SNAPPY',
-        rowGroupSize: 10000
-      });
-      // Extract data for this partition
-      let hasNextPage = true;
-      let cursor: string | null = null;
-      let partitionRecords = 0;
-      while (hasNextPage) {
-        const result = await client.graphql({
-          query: `
-            query ExtractPartition($from: DateTime!, $to: DateTime!, $after: String, $first: Int!) {
-              inventoryPositions(
-                after: $after,
-                first: $first,
-                updatedOn: { from: $from, to: $to }
-              ) {
-                pageInfo { hasNextPage }
-                edges {
-                  cursor
-                  node { ref productRef locationRef qty updatedOn }
-                }
-              }
-            }
-          `,
-          variables: {
-            from: date.start.toISOString(),
-            to: date.end.toISOString(),
-            first: 1000,
-            after: cursor
-          }
-        });
-        const { edges, pageInfo } = result.data.inventoryPositions;
-        for (const edge of edges) {
-          await writer.appendRow({
-            ...edge.node,
-            updatedOn: edge.node.updatedOn ? new Date(edge.node.updatedOn) : null
-          });
-          partitionRecords++;
-        }
-        hasNextPage = pageInfo.hasNextPage;
-        cursor = edges.length > 0 ? edges[edges.length - 1].cursor : null;
-      }
-      await writer.close();
-      console.log(`✅ ${partition}: ${partitionRecords} records`);
-    }
-  }
-  private getDateRange(start: Date, end: Date): any[] {
-    const dates = [];
-    const current = new Date(start);
-    while (current <= end) {
-      dates.push({
-        year: current.getFullYear(),
-        month: String(current.getMonth() + 1).padStart(2, '0'),
-        day: String(current.getDate()).padStart(2, '0'),
-        start: new Date(current.setHours(0, 0, 0, 0)),
-        end: new Date(current.setHours(23, 59, 59, 999))
-      });
-      current.setDate(current.getDate() + 1);
-    }
-    return dates;
-  }
-}
-```
-### Category-Based Partitioning
-```typescript
-import { Buffer } from 'node:buffer'; // Required for Deno/Versori
-/**
- * Partition by business category
- * Business use case: Separate high-value vs low-value inventory
- */
-async function extractWithCategoryPartitioning(outputDir: string): Promise<void> {
-  const categories = [
-    { name: 'high-value', filter: { minQty: 1000 } },
-    { name: 'medium-value', filter: { minQty: 100, maxQty: 999 } },
-    { name: 'low-value', filter: { maxQty: 99 } }
-  ];
-  for (const category of categories) {
-    const outputPath = `${outputDir}/category=${category.name}/data.parquet`;
-    console.log(`Extracting category: ${category.name}`);
-    // Extract and write for this category
-    // (Implementation similar to time-based partitioning)
-  }
-}
-```
----
-## Compression Optimization
-### Compression Trade-offs
-```typescript
-import { Buffer } from 'node:buffer'; // Required for Deno/Versori
-import * as parquet from 'parquetjs';
-import * as fs from 'fs/promises';
-/**
- * Test different compression algorithms
- */
-async function benchmarkCompression(
-  records: any[],
-  outputDir: string
-): Promise<void> {
-  const compressions = ['UNCOMPRESSED', 'SNAPPY', 'GZIP', 'BROTLI'];
-  const results = [];
-  for (const compression of compressions) {
-    const outputFile = `${outputDir}/test_${compression.toLowerCase()}.parquet`;
-    const startTime = Date.now();
-    const writer = await parquet.ParquetWriter.openFile(schema, outputFile, {
-      compression: compression as any,
-      rowGroupSize: 10000
-    });
-    for (const record of records) {
-      await writer.appendRow(record);
-    }
-    await writer.close();
-    const duration = Date.now() - startTime;
-    const fileStats = await fs.stat(outputFile);
-    results.push({
-      compression,
-      writeTime: duration,
-      fileSize: fileStats.size,
-      fileSizeMB: (fileStats.size / 1024 / 1024).toFixed(2)
-    });
-    console.log(`${compression}: ${duration}ms, ${(fileStats.size / 1024 / 1024).toFixed(2)}MB`);
-  }
-  console.table(results);
-}
-```
-**Typical Results** (100K records):
-| Compression | Write Time | File Size | Compression Ratio |
-|-------------|------------|-----------|-------------------|
-| UNCOMPRESSED | 1.2 sec | 20 MB | 1x |
-| SNAPPY | 1.5 sec | 7 MB | 2.9x |
-| GZIP | 3.2 sec | 5 MB | 4x |
-| BROTLI | 8.5 sec | 4 MB | 5x |
----
-## Memory Management
-### Garbage Collection Hints
-```typescript
-import { Buffer } from 'node:buffer'; // Required for Deno/Versori
-/**
- * Process with explicit memory management
- */
-async function processWithMemoryManagement(
-  arrayBuffer: ArrayBuffer,
-  fileName: string
-): Promise<void> {
-  const parser = new ParquetParserService(logger, {
-    batchSize: 500,  // Small batches
-    enableStreaming: true
-  });
-  const streamResult = await parser.parseStreaming(arrayBuffer, fileName);
-  let processedCount = 0;
-  for await (const batch of streamResult.records) {
-    // Process batch
-    await processBatch(batch);
-    processedCount += batch.length;
-    // Periodic GC hint
-    if (processedCount % 50000 === 0) {
-      console.log(`Processed ${processedCount} records`);
-      // Force GC if available (run with --expose-gc)
-      if (global.gc) {
-        global.gc();
-        const mem = process.memoryUsage();
-        console.log(`  Heap: ${(mem.heapUsed / 1024 / 1024).toFixed(2)}MB`);
-      }
-    }
-  }
-  const stats = await streamResult.stats;
-  console.log(`Peak memory: ${(stats.peakMemoryUsage / 1024 / 1024).toFixed(2)}MB`);
-}
-```
----
-## Performance Monitoring
-### Comprehensive Metrics
-```typescript
-import { Buffer } from 'node:buffer'; // Required for Deno/Versori
-class PerformanceTracker {
-  private startTime: number;
-  private checkpoints: Map<string, number>;
-  constructor() {
-    this.startTime = Date.now();
-    this.checkpoints = new Map();
-  }
-  checkpoint(name: string): void {
-    this.checkpoints.set(name, Date.now() - this.startTime);
-    console.log(`✓ ${name}: ${this.checkpoints.get(name)}ms`);
-  }
-  summary(): void {
-    console.log('\n📊 Performance Summary:');
-    for (const [name, duration] of this.checkpoints) {
-      console.log(`   ${name}: ${duration}ms`);
-    }
-    console.log(`   Total: ${Date.now() - this.startTime}ms`);
-  }
-}
-// Usage
-const tracker = new PerformanceTracker();
-tracker.checkpoint('Download from S3');
-// ... download code
-tracker.checkpoint('Parse Parquet');
-// ... parsing code
-tracker.checkpoint('Transform data');
-// ... transformation code
-tracker.checkpoint('Upload to target');
-// ... upload code
-tracker.summary();
-```
----
-## Key Takeaways
-- 🎯 **Dynamic batch sizes** - Adjust based on file size
-- 🎯 **Parallel processing** - Use worker pools for multiple files
-- 🎯 **Partitioning** - Time/category-based for efficient queries
-- 🎯 **SNAPPY compression** - Best balance for most cases
-- 🎯 **Monitor metrics** - Track and optimize performance
----
-## Next Steps
-Continue to [Module 8: Best Practices](./03-pattern-guides-parquet-08-best-practices.md) to learn production patterns, error handling, and troubleshooting.
----
-[← Previous: Module 6 - S3 Integration](./03-pattern-guides-parquet-06-s3-integration.md) | [Back to Guide](../parquet-readme.md) | [Next: Module 8 - Best Practices →](./03-pattern-guides-parquet-08-best-practices.md)
-## Related Documentation
-- [Module 3: Reading Parquet](./03-pattern-guides-parquet-03-reading-parquet.md) - Streaming and validation
-- [Module 8: Best Practices](./03-pattern-guides-parquet-08-best-practices.md) - Production patterns
+# Module 7: Performance Optimization
+[← Back to Parquet Guide](../parquet-readme.md)
+**Module 7 of 8** | **Level**: Advanced | **Time**: 25 minutes
+---
+## Overview
+Learn advanced performance optimization techniques for processing large-scale Parquet datasets efficiently.
+## Learning Objectives
+By the end of this module, you will:
+- ✅ Optimize batch sizes for different file sizes
+- ✅ Implement parallel processing strategies
+- ✅ Use partitioning for efficient queries
+- ✅ Monitor and track performance metrics
+- ✅ Tune compression and row group settings
+---
+## Performance Benchmarks
+### SDK Parser Performance
+Based on test files in `fc-connect-sdk/tests/fixtures/parquet/`:
+| File Size | Records | Processing Time | Throughput | Memory Usage |
+|-----------|---------|-----------------|------------|--------------|
+| **10 KB** | 50 | < 10 ms | 5,000+ rec/sec | ~10 MB |
+| **170 KB** | 1,000 | 50-100 ms | 20,000 rec/sec | ~20 MB |
+| **1.7 MB** | 10,000 | 200-300 ms | 35,000 rec/sec | ~30 MB |
+| **8.5 MB** | 50,000 | 800ms-1s | 50,000 rec/sec | ~50 MB |
+| **17 MB** | 100,000 | 1.5-2 sec | 55,000 rec/sec | ~70 MB |
+| **85 MB** | 500,000 | 6-8 sec | 65,000 rec/sec | ~150 MB |
+| **169 MB** | 1,000,000 | 12-15 sec | 70,000 rec/sec | ~200 MB |
+**Hardware**: Node.js runtime, 8GB RAM, standard SSD
+---
+## Batch Size Optimization
+### Choosing the Right Batch Size
+```typescript
+/**
+ * Dynamic batch size based on file size
+ */
+function getOptimalBatchSize(fileSizeBytes: number): number {
+  const sizeMB = fileSizeBytes / 1024 / 1024;
+  if (sizeMB < 1) {
+    return 5000;   // Small files: larger batches
+  } else if (sizeMB < 10) {
+    return 2000;   // Medium files: balanced
+  } else if (sizeMB < 50) {
+    return 1000;   // Large files: standard
+  } else {
+    return 500;    // Very large files: smaller batches
+  }
+}
+async function parseWithOptimalBatchSize(
+  arrayBuffer: ArrayBuffer,
+  fileName: string,
+  fileSizeBytes: number
+) {
+  const batchSize = getOptimalBatchSize(fileSizeBytes);
+  const parser = new ParquetParserService(logger, {
+    batchSize,
+    enableStreaming: fileSizeBytes > 50 * 1024 * 1024  // Stream if > 50MB
+  });
+  console.log(`Using batch size: ${batchSize}`);
+  if (parser.enableStreaming) {
+    const streamResult = await parser.parseStreaming(arrayBuffer, fileName);
+    for await (const batch of streamResult.records) {
+      await processBatch(batch);
+    }
+    return await streamResult.stats;
+  } else {
+    return await parser.parse(arrayBuffer, fileName);
+  }
+}
+```
+---
+## Parallel Processing
+### Process Multiple Files Concurrently
+```typescript
+/**
+ * Process multiple Parquet files in parallel
+ * Business use case: Batch processing of daily extracts
+ */
+class ParallelParquetProcessor {
+  private parser: ParquetParserService;
+  private concurrency: number;
+  constructor(concurrency: number = 3) {
+    this.parser = new ParquetParserService(logger, {
+      batchSize: 2000,
+      enableStreaming: true
+    });
+    this.concurrency = concurrency;
+  }
+  async processFiles(
+    files: Array<{ bucket: string; key: string }>
+  ): Promise<void> {
+    const queue = [...files];
+    const workers: Promise<void>[] = [];
+    console.log(`Processing ${files.length} files with ${this.concurrency} workers`);
+    // Start workers
+    for (let i = 0; i < this.concurrency; i++) {
+      workers.push(this.worker(queue, i + 1));
+    }
+    // Wait for all workers to complete
+    await Promise.all(workers);
+    // Get final metrics
+    const metrics = this.parser.getPerformanceMetrics();
+    console.log(`\n✅ Batch processing complete:`);
+    console.log(`   Total files: ${files.length}`);
+    console.log(`   Success rate: ${(metrics.successfulRequests / metrics.totalRequests * 100).toFixed(2)}%`);
+    console.log(`   Avg processing time: ${metrics.averageResponseTime.toFixed(2)}ms`);
+  }
+  private async worker(
+    queue: Array<{ bucket: string; key: string }>,
+    workerId: number
+  ): Promise<void> {
+    while (queue.length > 0) {
+      const file = queue.shift();
+      if (!file) break;
+      try {
+        console.log(`[Worker ${workerId}] Processing ${file.key}...`);
+        const buffer = await s3.downloadFile(file.key, { encoding: 'binary' });
+        const arrayBuffer = buffer.buffer.slice(
+          buffer.byteOffset,
+          buffer.byteOffset + buffer.byteLength
+        );
+        const startTime = Date.now();
+        const streamResult = await this.parser.parseStreaming(
+          arrayBuffer,
+          file.key,
+          { batchSize: 5000 }
+        );
+        for await (const batch of streamResult.records) {
+          await this.processBatch(batch);
+        }
+        const stats = await streamResult.stats;
+        const duration = Date.now() - startTime;
+        console.log(`[Worker ${workerId}] ✅ ${file.key}: ${stats.totalRecords} records in ${duration}ms`);
+      } catch (error) {
+        console.error(`[Worker ${workerId}] ❌ Failed ${file.key}:`, error.message);
+      }
+    }
+    console.log(`[Worker ${workerId}] Finished`);
+  }
+  private async processBatch(batch: any[]): Promise<void> {
+    // Your processing logic
+    // Parallel batch processing within each worker
+    await Promise.all(
+      batch.map(record => this.processRecord(record))
+    );
+  }
+  private async processRecord(record: any): Promise<void> {
+    // Your business logic
+  }
+}
+// Usage
+const processor = new ParallelParquetProcessor(3);  // 3 concurrent workers
+await processor.processFiles([
+  { bucket: 'data-lake', key: 'inventory/2025-10-18.parquet' },
+  { bucket: 'data-lake', key: 'inventory/2025-10-19.parquet' },
+  { bucket: 'data-lake', key: 'inventory/2025-10-20.parquet' }
+]);
+```
+---
+## Partitioning Strategies
+### Time-Based Partitioning
+```typescript
+import { Buffer } from 'node:buffer'; // Required for Deno/Versori
+/**
+ * Generate time-based partitions for efficient queries
+ * Business use case: Daily snapshots for analytics
+ */
+class PartitionedExtractor {
+  async extractWithTimePartitioning(
+    outputDir: string,
+    startDate: Date,
+    endDate: Date
+  ): Promise<void> {
+    const dates = this.getDateRange(startDate, endDate);
+    for (const date of dates) {
+      // Hive-style partitioning: year=YYYY/month=MM/day=DD
+      const partition = `year=${date.year}/month=${date.month}/day=${date.day}`;
+      const outputPath = `${outputDir}/${partition}/data.parquet`;
+      console.log(`\nExtracting partition: ${partition}`);
+      const schema = new parquet.ParquetSchema({
+        ref: { type: 'UTF8' },
+        productRef: { type: 'UTF8' },
+        locationRef: { type: 'UTF8' },
+        qty: { type: 'INT32' },
+        updatedOn: { type: 'TIMESTAMP_MILLIS' }
+      });
+      const writer = await parquet.ParquetWriter.openFile(schema, outputPath, {
+        compression: 'SNAPPY',
+        rowGroupSize: 10000
+      });
+      // Extract data for this partition
+      let hasNextPage = true;
+      let cursor: string | null = null;
+      let partitionRecords = 0;
+      while (hasNextPage) {
+        const result = await client.graphql({
+          query: `
+            query ExtractPartition($from: DateTime!, $to: DateTime!, $after: String, $first: Int!) {
+              inventoryPositions(
+                after: $after,
+                first: $first,
+                updatedOn: { from: $from, to: $to }
+              ) {
+                pageInfo { hasNextPage }
+                edges {
+                  cursor
+                  node { ref productRef locationRef qty updatedOn }
+                }
+              }
+            }
+          `,
+          variables: {
+            from: date.start.toISOString(),
+            to: date.end.toISOString(),
+            first: 1000,
+            after: cursor
+          }
+        });
+        const { edges, pageInfo } = result.data.inventoryPositions;
+        for (const edge of edges) {
+          await writer.appendRow({
+            ...edge.node,
+            updatedOn: edge.node.updatedOn ? new Date(edge.node.updatedOn) : null
+          });
+          partitionRecords++;
+        }
+        hasNextPage = pageInfo.hasNextPage;
+        cursor = edges.length > 0 ? edges[edges.length - 1].cursor : null;
+      }
+      await writer.close();
+      console.log(`✅ ${partition}: ${partitionRecords} records`);
+    }
+  }
+  private getDateRange(start: Date, end: Date): any[] {
+    const dates = [];
+    const current = new Date(start);
+    while (current <= end) {
+      dates.push({
+        year: current.getFullYear(),
+        month: String(current.getMonth() + 1).padStart(2, '0'),
+        day: String(current.getDate()).padStart(2, '0'),
+        start: new Date(current.setHours(0, 0, 0, 0)),
+        end: new Date(current.setHours(23, 59, 59, 999))
+      });
+      current.setDate(current.getDate() + 1);
+    }
+    return dates;
+  }
+}
+```
+### Category-Based Partitioning
+```typescript
+import { Buffer } from 'node:buffer'; // Required for Deno/Versori
+/**
+ * Partition by business category
+ * Business use case: Separate high-value vs low-value inventory
+ */
+async function extractWithCategoryPartitioning(outputDir: string): Promise<void> {
+  const categories = [
+    { name: 'high-value', filter: { minQty: 1000 } },
+    { name: 'medium-value', filter: { minQty: 100, maxQty: 999 } },
+    { name: 'low-value', filter: { maxQty: 99 } }
+  ];
+  for (const category of categories) {
+    const outputPath = `${outputDir}/category=${category.name}/data.parquet`;
+    console.log(`Extracting category: ${category.name}`);
+    // Extract and write for this category
+    // (Implementation similar to time-based partitioning)
+  }
+}
+```
+---
+## Compression Optimization
+### Compression Trade-offs
+```typescript
+import { Buffer } from 'node:buffer'; // Required for Deno/Versori
+import * as parquet from 'parquetjs';
+import * as fs from 'fs/promises';
+/**
+ * Test different compression algorithms
+ */
+async function benchmarkCompression(
+  records: any[],
+  outputDir: string
+): Promise<void> {
+  const compressions = ['UNCOMPRESSED', 'SNAPPY', 'GZIP', 'BROTLI'];
+  const results = [];
+  for (const compression of compressions) {
+    const outputFile = `${outputDir}/test_${compression.toLowerCase()}.parquet`;
+    const startTime = Date.now();
+    const writer = await parquet.ParquetWriter.openFile(schema, outputFile, {
+      compression: compression as any,
+      rowGroupSize: 10000
+    });
+    for (const record of records) {
+      await writer.appendRow(record);
+    }
+    await writer.close();
+    const duration = Date.now() - startTime;
+    const fileStats = await fs.stat(outputFile);
+    results.push({
+      compression,
+      writeTime: duration,
+      fileSize: fileStats.size,
+      fileSizeMB: (fileStats.size / 1024 / 1024).toFixed(2)
+    });
+    console.log(`${compression}: ${duration}ms, ${(fileStats.size / 1024 / 1024).toFixed(2)}MB`);
+  }
+  console.table(results);
+}
+```
+**Typical Results** (100K records):
+| Compression | Write Time | File Size | Compression Ratio |
+|-------------|------------|-----------|-------------------|
+| UNCOMPRESSED | 1.2 sec | 20 MB | 1x |
+| SNAPPY | 1.5 sec | 7 MB | 2.9x |
+| GZIP | 3.2 sec | 5 MB | 4x |
+| BROTLI | 8.5 sec | 4 MB | 5x |
+---
+## Memory Management
+### Garbage Collection Hints
+```typescript
+import { Buffer } from 'node:buffer'; // Required for Deno/Versori
+/**
+ * Process with explicit memory management
+ */
+async function processWithMemoryManagement(
+  arrayBuffer: ArrayBuffer,
+  fileName: string
+): Promise<void> {
+  const parser = new ParquetParserService(logger, {
+    batchSize: 500,  // Small batches
+    enableStreaming: true
+  });
+  const streamResult = await parser.parseStreaming(arrayBuffer, fileName);
+  let processedCount = 0;
+  for await (const batch of streamResult.records) {
+    // Process batch
+    await processBatch(batch);
+    processedCount += batch.length;
+    // Periodic GC hint
+    if (processedCount % 50000 === 0) {
+      console.log(`Processed ${processedCount} records`);
+      // Force GC if available (run with --expose-gc)
+      if (global.gc) {
+        global.gc();
+        const mem = process.memoryUsage();
+        console.log(`  Heap: ${(mem.heapUsed / 1024 / 1024).toFixed(2)}MB`);
+      }
+    }
+  }
+  const stats = await streamResult.stats;
+  console.log(`Peak memory: ${(stats.peakMemoryUsage / 1024 / 1024).toFixed(2)}MB`);
+}
+```
+---
+## Performance Monitoring
+### Comprehensive Metrics
+```typescript
+import { Buffer } from 'node:buffer'; // Required for Deno/Versori
+class PerformanceTracker {
+  private startTime: number;
+  private checkpoints: Map<string, number>;
+  constructor() {
+    this.startTime = Date.now();
+    this.checkpoints = new Map();
+  }
+  checkpoint(name: string): void {
+    this.checkpoints.set(name, Date.now() - this.startTime);
+    console.log(`✓ ${name}: ${this.checkpoints.get(name)}ms`);
+  }
+  summary(): void {
+    console.log('\n📊 Performance Summary:');
+    for (const [name, duration] of this.checkpoints) {
+      console.log(`   ${name}: ${duration}ms`);
+    }
+    console.log(`   Total: ${Date.now() - this.startTime}ms`);
+  }
+}
+// Usage
+const tracker = new PerformanceTracker();
+tracker.checkpoint('Download from S3');
+// ... download code
+tracker.checkpoint('Parse Parquet');
+// ... parsing code
+tracker.checkpoint('Transform data');
+// ... transformation code
+tracker.checkpoint('Upload to target');
+// ... upload code
+tracker.summary();
+```
+---
+## Key Takeaways
+- 🎯 **Dynamic batch sizes** - Adjust based on file size
+- 🎯 **Parallel processing** - Use worker pools for multiple files
+- 🎯 **Partitioning** - Time/category-based for efficient queries
+- 🎯 **SNAPPY compression** - Best balance for most cases
+- 🎯 **Monitor metrics** - Track and optimize performance
+---
+## Next Steps
+Continue to [Module 8: Best Practices](./03-pattern-guides-parquet-08-best-practices.md) to learn production patterns, error handling, and troubleshooting.
+---
+[← Previous: Module 6 - S3 Integration](./03-pattern-guides-parquet-06-s3-integration.md) | [Back to Guide](../parquet-readme.md) | [Next: Module 8 - Best Practices →](./03-pattern-guides-parquet-08-best-practices.md)
+## Related Documentation
+- [Module 3: Reading Parquet](./03-pattern-guides-parquet-03-reading-parquet.md) - Streaming and validation
+- [Module 8: Best Practices](./03-pattern-guides-parquet-08-best-practices.md) - Production patterns