@fluentcommerce/fc-connect-sdk 0.1.54 → 0.1.56

package/docs/03-PATTERN-GUIDES/file-operations/modules/file-operations-05-streaming-performance.md

@@ -1,503 +1,503 @@
-# Module 5: Performance Optimization & Large Files
-
-**Level:** Advanced
-**Estimated Time:** 20 minutes
-
-## Overview
-
-Learn how to handle large files efficiently and optimize file operation performance using batch processing, memory management, and parallel operations.
-
-## Learning Objectives
-
-By the end of this module, you will:
-- ✅ Understand memory-efficient file processing strategies
-- ✅ Implement batch processing for multiple files
-- ✅ Handle large files without running out of memory
-- ✅ Optimize parallel file operations
-- ✅ Monitor performance metrics
-
-## Important Note About Streaming
-
-⚠️ **The SDK does not currently provide built-in streaming methods.**
-
-Both `S3DataSource` and `SftpDataSource` use `downloadFile()` which returns the entire file content as a `string` or `Buffer`. For large files, you need to implement your own processing strategies using the techniques below.
-
-## File Size Guidelines
-
-| File Size | Strategy | Memory Impact | Recommendation |
-|-----------|----------|---------------|----------------|
-| < 10 MB | Direct download | ~File size | ✅ Use `downloadFile()` directly |
-| 10-100 MB | Download + process in chunks | ~File size | ⚠️ Process in batches, clear memory |
-| 100-500 MB | Download with memory management | ~File size | ⚠️ Use Buffer, process in chunks, monitor memory |
-| > 500 MB | Avoid direct download | Very high | 🔴 Split files at source if possible |
-
-## Memory-Efficient Processing Strategies
-
-### Strategy 1: Process Downloaded Files in Chunks
-
-```typescript
-import { S3DataSource } from '@fluentcommerce/fc-connect-sdk';
-
-async function processLargeFileInChunks(s3: S3DataSource, key: string) {
-  // Download as Buffer to handle binary/large files (S3 uses encoding: 'binary')
-  const fileBuffer = await s3.downloadFile(key, { encoding: 'binary' });
-
-  const chunkSize = 1024 * 1024; // 1MB chunks
-  let offset = 0;
-  let recordCount = 0;
-
-  while (offset < fileBuffer.length) {
-    const chunk = fileBuffer.slice(offset, offset + chunkSize);
-    const chunkStr = chunk.toString('utf8');
-
-    // Process chunk line-by-line
-    const lines = chunkStr.split('\n');
-
-    for (const line of lines) {
-      if (line.trim()) {
-        await processRecord(line);
-        recordCount++;
-      }
-    }
-
-    offset += chunkSize;
-
-    // Log progress
-    if (recordCount % 10000 === 0) {
-      console.log(`Processed ${recordCount} records...`);
-    }
-  }
-
-  console.log(`Total records: ${recordCount}`);
-}
-
-async function processRecord(line: string): Promise<void> {
-  // Your processing logic
-  const data = JSON.parse(line);
-  // ... handle data
-}
-```
-
-### Strategy 2: Download with Size Limit
-
-```typescript
-// SFTP supports maxSize option to prevent large downloads
-async function safeDownload(sftp: SftpDataSource, path: string) {
-  try {
-    const content = await sftp.downloadFile(path, {
-      maxSize: 100 * 1024 * 1024  // 100 MB limit
-    });
-
-    return content;
-
-  } catch (error: any) {
-    if (error.message.includes('exceeds maximum')) {
-      console.error(`File ${path} is too large (> 100MB)`);
-      // Handle oversized file (skip, split, etc.)
-      return null;
-    }
-    throw error;
-  }
-}
-```
-
-### Strategy 3: Line-by-Line Processing from Buffer
-
-```typescript
-async function processCSVFromBuffer(content: string | Buffer) {
-  const text = typeof content === 'string' ? content : content.toString('utf8');
-  const lines = text.split('\n');
-
-  const results: any[] = [];
-
-  for (let i = 0; i < lines.length; i++) {
-    const line = lines[i].trim();
-
-    if (!line) continue; // Skip empty lines
-
-    const record = parseCSVLine(line);
-    results.push(record);
-
-    // Clear memory periodically
-    if (results.length >= 10000) {
-      await processBatch(results);
-      results.length = 0; // Clear array
-    }
-  }
-
-  // Process remaining records
-  if (results.length > 0) {
-    await processBatch(results);
-  }
-}
-
-function parseCSVLine(line: string): any {
-  const fields = line.split(',');
-  return {
-    sku: fields[0],
-    qty: parseInt(fields[1], 10),
-    location: fields[2]
-  };
-}
-
-async function processBatch(records: any[]): Promise<void> {
-  console.log(`Processing batch of ${records.length} records...`);
-  // Your batch processing logic
-}
-```
-
-## Batch File Processing
-
-### Parallel Processing with Controlled Concurrency
-
-```typescript
-import { S3DataSource } from '@fluentcommerce/fc-connect-sdk';
-
-async function processManyFilesInParallel(s3: S3DataSource, prefix: string) {
-  const files = await s3.listFiles({ prefix });
-
-  const concurrency = 10; // Process 10 files at once
-
-  for (let i = 0; i < files.length; i += concurrency) {
-    const batch = files.slice(i, i + concurrency);
-
-    await Promise.all(
-      batch.map(async (file) => {
-        try {
-          console.log(`Processing ${file.name}...`);
-
-          const content = await s3.downloadFile(file.name);
-          const result = await transformData(content);
-
-          // Upload result
-          await s3.uploadFile(
-            file.name.replace('incoming/', 'output/'),
-            JSON.stringify(result),
-            { contentType: 'application/json' }
-          );
-
-          // Archive original
-          await s3.moveFile(
-            file.name,
-            file.name.replace('incoming/', 'archive/'),
-            true // deleteSource
-          );
-
-          console.log(`✓ Completed ${file.name}`);
-
-        } catch (error: any) {
-          console.error(`✗ Failed ${file.name}:`, error.message);
-        }
-      })
-    );
-
-    console.log(`Completed batch ${Math.floor(i / concurrency) + 1}/${Math.ceil(files.length / concurrency)}`);
-  }
-}
-
-async function transformData(content: string): Promise<any> {
-  // Your transformation logic
-  return { processed: true, data: content };
-}
-```
-
-### Sequential Processing for Memory-Constrained Environments
-
-```typescript
-async function processFilesSequentially(s3: S3DataSource, prefix: string) {
-  const files = await s3.listFiles({ prefix });
-
-  let successCount = 0;
-  let failureCount = 0;
-
-  for (const file of files) {
-    try {
-      console.log(`Processing ${file.name}...`);
-
-      // Download and process one file at a time
-      const content = await s3.downloadFile(file.name);
-      await processFileContent(content, file.name);
-
-      // Move to archive
-      await s3.moveFile(
-        file.name,
-        file.name.replace('incoming/', 'archive/'),
-        true
-      );
-
-      successCount++;
-
-      // Force garbage collection hint (Node.js)
-      if (global.gc) {
-        global.gc();
-      }
-
-    } catch (error: any) {
-      console.error(`Failed ${file.name}:`, error.message);
-      failureCount++;
-    }
-  }
-
-  console.log(`\nResults: ${successCount} succeeded, ${failureCount} failed`);
-}
-
-async function processFileContent(content: string, fileName: string): Promise<void> {
-  // Process content line by line to manage memory
-  const lines = content.split('\n');
-  console.log(`  Processing ${lines.length} lines...`);
-
-  for (const line of lines) {
-    if (line.trim()) {
-      // Process each line
-      await handleLine(line);
-    }
-  }
-}
-
-async function handleLine(line: string): Promise<void> {
-  // Your line processing logic
-}
-```
-
-## SFTP Connection Pooling
-
-### Reuse Connections for Better Performance
-
-```typescript
-import { SftpDataSource } from '@fluentcommerce/fc-connect-sdk';
-
-// ✅ GOOD: Reuse connection for multiple operations
-async function processMultipleFilesSftpReuse(config: any, logger: any) {
-  const sftp = new SftpDataSource(config, logger);
-
-  try {
-    const files = await sftp.listFiles({ remotePath: '/incoming' });
-
-    for (const file of files) {
-      const content = await sftp.downloadFile(file.path);
-      await processData(content);
-
-      // Move to processed folder
-      await sftp.moveFile(
-        file.path,
-        file.path.replace('/incoming/', '/processed/'),
-        false // Don't overwrite
-      );
-    }
-
-  } finally {
-    await sftp.dispose(); // Close connection once
-  }
-}
-
-// ❌ BAD: Create new connection for each file
-async function processMultipleFilesSftpSlow(config: any, logger: any, files: any[]) {
-  for (const file of files) {
-    const sftp = new SftpDataSource(config, logger);
-
-    try {
-      const content = await sftp.downloadFile(file.path);
-      await processData(content);
-    } finally {
-      await sftp.dispose(); // Expensive connection overhead!
-    }
-  }
-}
-
-async function processData(content: string | Buffer): Promise<void> {
-  // Your processing logic
-}
-```
-
-## Performance Monitoring
-
-### Track Processing Metrics
-
-```typescript
-interface ProcessingMetrics {
-  filesProcessed: number;
-  filesSkipped: number;
-  filesFailed: number;
-  totalBytes: number;
-  totalRecords: number;
-  startTime: number;
-  endTime?: number;
-}
-
-async function processWithMetrics(s3: S3DataSource, prefix: string) {
-  const metrics: ProcessingMetrics = {
-    filesProcessed: 0,
-    filesSkipped: 0,
-    filesFailed: 0,
-    totalBytes: 0,
-    totalRecords: 0,
-    startTime: Date.now()
-  };
-
-  const files = await s3.listFiles({ prefix });
-  console.log(`Found ${files.length} files to process\n`);
-
-  for (const file of files) {
-    try {
-      // Skip non-CSV files
-      if (!file.name.endsWith('.csv')) {
-        metrics.filesSkipped++;
-        console.log(`⊘ Skipped ${file.name} (not CSV)`);
-        continue;
-      }
-
-      const content = await s3.downloadFile(file.name);
-      metrics.totalBytes += content.length;
-
-      const lines = content.split('\n').filter(l => l.trim());
-      metrics.totalRecords += lines.length;
-
-      await processFileData(content);
-
-      metrics.filesProcessed++;
-      console.log(`✓ Processed ${file.name} (${lines.length} records)`);
-
-    } catch (error: any) {
-      metrics.filesFailed++;
-      console.error(`✗ Failed ${file.name}:`, error.message);
-    }
-  }
-
-  metrics.endTime = Date.now();
-  printMetrics(metrics);
-}
-
-function printMetrics(metrics: ProcessingMetrics): void {
-  const duration = ((metrics.endTime! - metrics.startTime) / 1000).toFixed(2);
-  const throughputMBps = (metrics.totalBytes / (1024 * 1024) / parseFloat(duration)).toFixed(2);
-  const throughputRecSec = (metrics.totalRecords / parseFloat(duration)).toFixed(0);
-
-  console.log('\n' + '='.repeat(50));
-  console.log('Processing Metrics:');
-  console.log('='.repeat(50));
-  console.log(`Files Processed:  ${metrics.filesProcessed}`);
-  console.log(`Files Skipped:    ${metrics.filesSkipped}`);
-  console.log(`Files Failed:     ${metrics.filesFailed}`);
-  console.log(`Total Records:    ${metrics.totalRecords}`);
-  console.log(`Total Data:       ${(metrics.totalBytes / (1024 * 1024)).toFixed(2)} MB`);
-  console.log(`Duration:         ${duration}s`);
-  console.log(`Throughput:       ${throughputMBps} MB/s`);
-  console.log(`Records/Second:   ${throughputRecSec}`);
-  console.log('='.repeat(50) + '\n');
-}
-
-async function processFileData(content: string): Promise<void> {
-  // Your processing logic
-}
-```
-
-## Memory Management Tips
-
-### 1. Use Binary Mode for Large Files
-
-```typescript
-// For large files, use Buffer to avoid string conversion overhead
-// S3: use encoding: 'binary'
-const buffer = await s3.downloadFile(key, { encoding: 'binary' });
-
-// SFTP: use asBuffer: true
-const buffer = await sftp.downloadFile(path, { asBuffer: true });
-
-// Process buffer directly
-processBuffer(buffer);
-```
-
-### 2. Clear Large Arrays Periodically
-
-```typescript
-const results: any[] = [];
-
-for (const record of records) {
-  results.push(transformRecord(record));
-
-  // Clear memory every 10,000 records
-  if (results.length >= 10000) {
-    await writeToDestination(results);
-    results.length = 0; // Clear array to free memory
-  }
-}
-```
-
-### 3. Process Files in Batches
-
-```typescript
-// Instead of loading all files at once:
-const files = await s3.listFiles({ prefix: 'data/' });
-
-// Process in batches of 100
-for (let i = 0; i < files.length; i += 100) {
-  const batch = files.slice(i, i + 100);
-  await Promise.all(batch.map(f => processFile(f)));
-
-  // Give GC a chance to run
-  await new Promise(resolve => setTimeout(resolve, 100));
-}
-```
-
-### 4. Use Deno/Versori Buffer Import
-
-```typescript
-// CRITICAL for Deno/Versori runtime
-import { Buffer } from 'node:buffer';
-
-const buffer = Buffer.from(content, 'utf8');
-// ... use buffer
-```
-
-## Performance Comparison
-
-### Download Methods
-
-```typescript
-// Small files (< 10 MB): Direct string download
-const content = await s3.downloadFile('small.csv'); // Fast
-const lines = content.split('\n');
-
-// Large files (> 10 MB): Binary download + chunk processing
-const buffer = await s3.downloadFile('large.csv', { encoding: 'binary' });
-processInChunks(buffer, 1024 * 1024); // 1 MB chunks
-```
-
-### Parallel vs Sequential
-
-```typescript
-// Parallel (fast, high memory)
-await Promise.all(files.map(f => processFile(f)));
-
-// Sequential (slow, low memory)
-for (const file of files) {
-  await processFile(file);
-}
-
-// Batched parallel (balanced)
-for (let i = 0; i < files.length; i += 10) {
-  await Promise.all(files.slice(i, i + 10).map(f => processFile(f)));
-}
-```
-
-## Key Takeaways
-
-- 🎯 SDK does not provide built-in streaming - use download + chunk processing
-- 🎯 Use `encoding: 'binary'` (S3) or `asBuffer: true` (SFTP) for large files to avoid string conversion overhead
-- 🎯 Process files in chunks to maintain constant memory usage
-- 🎯 Use parallel processing with controlled batch sizes (10-20 files at once)
-- 🎯 Reuse SFTP connections for multiple operations
-- 🎯 Monitor metrics to identify bottlenecks
-- 🎯 Clear large arrays periodically to prevent memory buildup
-- 🎯 Use `maxSize` option in SFTP to prevent downloading oversized files
-
-## Next Steps
-
-Continue to [Module 6: Archive Patterns](./file-operations-06-archive-patterns.md) for advanced archiving workflows.
-
----
-
-**Related Resources:**
-- [Quick Reference](../../../02-CORE-GUIDES/advanced-services/advanced-services-quick-reference.md) - Performance tips
-- [Module 7: Testing & Troubleshooting](./file-operations-07-testing-troubleshooting.md) - Memory issues
-- [Common Patterns](../examples/common-patterns.ts) - Copy-paste ready snippets
+# Module 5: Performance Optimization & Large Files
+
+**Level:** Advanced
+**Estimated Time:** 20 minutes
+
+## Overview
+
+Learn how to handle large files efficiently and optimize file operation performance using batch processing, memory management, and parallel operations.
+
+## Learning Objectives
+
+By the end of this module, you will:
+- ✅ Understand memory-efficient file processing strategies
+- ✅ Implement batch processing for multiple files
+- ✅ Handle large files without running out of memory
+- ✅ Optimize parallel file operations
+- ✅ Monitor performance metrics
+
+## Important Note About Streaming
+
+⚠️ **The SDK does not currently provide built-in streaming methods.**
+
+Both `S3DataSource` and `SftpDataSource` use `downloadFile()` which returns the entire file content as a `string` or `Buffer`. For large files, you need to implement your own processing strategies using the techniques below.
+
+## File Size Guidelines
+
+| File Size | Strategy | Memory Impact | Recommendation |
+|-----------|----------|---------------|----------------|
+| < 10 MB | Direct download | ~File size | ✅ Use `downloadFile()` directly |
+| 10-100 MB | Download + process in chunks | ~File size | ⚠️ Process in batches, clear memory |
+| 100-500 MB | Download with memory management | ~File size | ⚠️ Use Buffer, process in chunks, monitor memory |
+| > 500 MB | Avoid direct download | Very high | 🔴 Split files at source if possible |
+
+## Memory-Efficient Processing Strategies
+
+### Strategy 1: Process Downloaded Files in Chunks
+
+```typescript
+import { S3DataSource } from '@fluentcommerce/fc-connect-sdk';
+
+async function processLargeFileInChunks(s3: S3DataSource, key: string) {
+  // Download as Buffer to handle binary/large files (S3 uses encoding: 'binary')
+  const fileBuffer = await s3.downloadFile(key, { encoding: 'binary' });
+
+  const chunkSize = 1024 * 1024; // 1MB chunks
+  let offset = 0;
+  let recordCount = 0;
+
+  while (offset < fileBuffer.length) {
+    const chunk = fileBuffer.slice(offset, offset + chunkSize);
+    const chunkStr = chunk.toString('utf8');
+
+    // Process chunk line-by-line
+    const lines = chunkStr.split('\n');
+
+    for (const line of lines) {
+      if (line.trim()) {
+        await processRecord(line);
+        recordCount++;
+      }
+    }
+
+    offset += chunkSize;
+
+    // Log progress
+    if (recordCount % 10000 === 0) {
+      console.log(`Processed ${recordCount} records...`);
+    }
+  }
+
+  console.log(`Total records: ${recordCount}`);
+}
+
+async function processRecord(line: string): Promise<void> {
+  // Your processing logic
+  const data = JSON.parse(line);
+  // ... handle data
+}
+```
+
+### Strategy 2: Download with Size Limit
+
+```typescript
+// SFTP supports maxSize option to prevent large downloads
+async function safeDownload(sftp: SftpDataSource, path: string) {
+  try {
+    const content = await sftp.downloadFile(path, {
+      maxSize: 100 * 1024 * 1024  // 100 MB limit
+    });
+
+    return content;
+
+  } catch (error: any) {
+    if (error.message.includes('exceeds maximum')) {
+      console.error(`File ${path} is too large (> 100MB)`);
+      // Handle oversized file (skip, split, etc.)
+      return null;
+    }
+    throw error;
+  }
+}
+```
+
+### Strategy 3: Line-by-Line Processing from Buffer
+
+```typescript
+async function processCSVFromBuffer(content: string | Buffer) {
+  const text = typeof content === 'string' ? content : content.toString('utf8');
+  const lines = text.split('\n');
+
+  const results: any[] = [];
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].trim();
+
+    if (!line) continue; // Skip empty lines
+
+    const record = parseCSVLine(line);
+    results.push(record);
+
+    // Clear memory periodically
+    if (results.length >= 10000) {
+      await processBatch(results);
+      results.length = 0; // Clear array
+    }
+  }
+
+  // Process remaining records
+  if (results.length > 0) {
+    await processBatch(results);
+  }
+}
+
+function parseCSVLine(line: string): any {
+  const fields = line.split(',');
+  return {
+    sku: fields[0],
+    qty: parseInt(fields[1], 10),
+    location: fields[2]
+  };
+}
+
+async function processBatch(records: any[]): Promise<void> {
+  console.log(`Processing batch of ${records.length} records...`);
+  // Your batch processing logic
+}
+```
+
+## Batch File Processing
+
+### Parallel Processing with Controlled Concurrency
+
+```typescript
+import { S3DataSource } from '@fluentcommerce/fc-connect-sdk';
+
+async function processManyFilesInParallel(s3: S3DataSource, prefix: string) {
+  const files = await s3.listFiles({ prefix });
+
+  const concurrency = 10; // Process 10 files at once
+
+  for (let i = 0; i < files.length; i += concurrency) {
+    const batch = files.slice(i, i + concurrency);
+
+    await Promise.all(
+      batch.map(async (file) => {
+        try {
+          console.log(`Processing ${file.name}...`);
+
+          const content = await s3.downloadFile(file.name);
+          const result = await transformData(content);
+
+          // Upload result
+          await s3.uploadFile(
+            file.name.replace('incoming/', 'output/'),
+            JSON.stringify(result),
+            { contentType: 'application/json' }
+          );
+
+          // Archive original
+          await s3.moveFile(
+            file.name,
+            file.name.replace('incoming/', 'archive/'),
+            true // deleteSource
+          );
+
+          console.log(`✓ Completed ${file.name}`);
+
+        } catch (error: any) {
+          console.error(`✗ Failed ${file.name}:`, error.message);
+        }
+      })
+    );
+
+    console.log(`Completed batch ${Math.floor(i / concurrency) + 1}/${Math.ceil(files.length / concurrency)}`);
+  }
+}
+
+async function transformData(content: string): Promise<any> {
+  // Your transformation logic
+  return { processed: true, data: content };
+}
+```
+
+### Sequential Processing for Memory-Constrained Environments
+
+```typescript
+async function processFilesSequentially(s3: S3DataSource, prefix: string) {
+  const files = await s3.listFiles({ prefix });
+
+  let successCount = 0;
+  let failureCount = 0;
+
+  for (const file of files) {
+    try {
+      console.log(`Processing ${file.name}...`);
+
+      // Download and process one file at a time
+      const content = await s3.downloadFile(file.name);
+      await processFileContent(content, file.name);
+
+      // Move to archive
+      await s3.moveFile(
+        file.name,
+        file.name.replace('incoming/', 'archive/'),
+        true
+      );
+
+      successCount++;
+
+      // Force garbage collection hint (Node.js)
+      if (global.gc) {
+        global.gc();
+      }
+
+    } catch (error: any) {
+      console.error(`Failed ${file.name}:`, error.message);
+      failureCount++;
+    }
+  }
+
+  console.log(`\nResults: ${successCount} succeeded, ${failureCount} failed`);
+}
+
+async function processFileContent(content: string, fileName: string): Promise<void> {
+  // Process content line by line to manage memory
+  const lines = content.split('\n');
+  console.log(`  Processing ${lines.length} lines...`);
+
+  for (const line of lines) {
+    if (line.trim()) {
+      // Process each line
+      await handleLine(line);
+    }
+  }
+}
+
+async function handleLine(line: string): Promise<void> {
+  // Your line processing logic
+}
+```
+
+## SFTP Connection Pooling
+
+### Reuse Connections for Better Performance
+
+```typescript
+import { SftpDataSource } from '@fluentcommerce/fc-connect-sdk';
+
+// ✅ GOOD: Reuse connection for multiple operations
+async function processMultipleFilesSftpReuse(config: any, logger: any) {
+  const sftp = new SftpDataSource(config, logger);
+
+  try {
+    const files = await sftp.listFiles({ remotePath: '/incoming' });
+
+    for (const file of files) {
+      const content = await sftp.downloadFile(file.path);
+      await processData(content);
+
+      // Move to processed folder
+      await sftp.moveFile(
+        file.path,
+        file.path.replace('/incoming/', '/processed/'),
+        false // Don't overwrite
+      );
+    }
+
+  } finally {
+    await sftp.dispose(); // Close connection once
+  }
+}
+
+// ❌ BAD: Create new connection for each file
+async function processMultipleFilesSftpSlow(config: any, logger: any, files: any[]) {
+  for (const file of files) {
+    const sftp = new SftpDataSource(config, logger);
+
+    try {
+      const content = await sftp.downloadFile(file.path);
+      await processData(content);
+    } finally {
+      await sftp.dispose(); // Expensive connection overhead!
+    }
+  }
+}
+
+async function processData(content: string | Buffer): Promise<void> {
+  // Your processing logic
+}
+```
+
+## Performance Monitoring
+
+### Track Processing Metrics
+
+```typescript
+interface ProcessingMetrics {
+  filesProcessed: number;
+  filesSkipped: number;
+  filesFailed: number;
+  totalBytes: number;
+  totalRecords: number;
+  startTime: number;
+  endTime?: number;
+}
+
+async function processWithMetrics(s3: S3DataSource, prefix: string) {
+  const metrics: ProcessingMetrics = {
+    filesProcessed: 0,
+    filesSkipped: 0,
+    filesFailed: 0,
+    totalBytes: 0,
+    totalRecords: 0,
+    startTime: Date.now()
+  };
+
+  const files = await s3.listFiles({ prefix });
+  console.log(`Found ${files.length} files to process\n`);
+
+  for (const file of files) {
+    try {
+      // Skip non-CSV files
+      if (!file.name.endsWith('.csv')) {
+        metrics.filesSkipped++;
+        console.log(`⊘ Skipped ${file.name} (not CSV)`);
+        continue;
+      }
+
+      const content = await s3.downloadFile(file.name);
+      metrics.totalBytes += content.length;
+
+      const lines = content.split('\n').filter(l => l.trim());
+      metrics.totalRecords += lines.length;
+
+      await processFileData(content);
+
+      metrics.filesProcessed++;
+      console.log(`✓ Processed ${file.name} (${lines.length} records)`);
+
+    } catch (error: any) {
+      metrics.filesFailed++;
+      console.error(`✗ Failed ${file.name}:`, error.message);
+    }
+  }
+
+  metrics.endTime = Date.now();
+  printMetrics(metrics);
+}
+
+function printMetrics(metrics: ProcessingMetrics): void {
+  const duration = ((metrics.endTime! - metrics.startTime) / 1000).toFixed(2);
+  const throughputMBps = (metrics.totalBytes / (1024 * 1024) / parseFloat(duration)).toFixed(2);
+  const throughputRecSec = (metrics.totalRecords / parseFloat(duration)).toFixed(0);
+
+  console.log('\n' + '='.repeat(50));
+  console.log('Processing Metrics:');
+  console.log('='.repeat(50));
+  console.log(`Files Processed:  ${metrics.filesProcessed}`);
+  console.log(`Files Skipped:    ${metrics.filesSkipped}`);
+  console.log(`Files Failed:     ${metrics.filesFailed}`);
+  console.log(`Total Records:    ${metrics.totalRecords}`);
+  console.log(`Total Data:       ${(metrics.totalBytes / (1024 * 1024)).toFixed(2)} MB`);
+  console.log(`Duration:         ${duration}s`);
+  console.log(`Throughput:       ${throughputMBps} MB/s`);
+  console.log(`Records/Second:   ${throughputRecSec}`);
+  console.log('='.repeat(50) + '\n');
+}
+
+async function processFileData(content: string): Promise<void> {
+  // Your processing logic
+}
+```
+
+## Memory Management Tips
+
+### 1. Use Binary Mode for Large Files
+
+```typescript
+// For large files, use Buffer to avoid string conversion overhead
+// S3: use encoding: 'binary'
+const buffer = await s3.downloadFile(key, { encoding: 'binary' });
+
+// SFTP: use asBuffer: true
+const buffer = await sftp.downloadFile(path, { asBuffer: true });
+
+// Process buffer directly
+processBuffer(buffer);
+```
+
+### 2. Clear Large Arrays Periodically
+
+```typescript
+const results: any[] = [];
+
+for (const record of records) {
+  results.push(transformRecord(record));
+
+  // Clear memory every 10,000 records
+  if (results.length >= 10000) {
+    await writeToDestination(results);
+    results.length = 0; // Clear array to free memory
+  }
+}
+```
+
+### 3. Process Files in Batches
+
+```typescript
+// Instead of loading all files at once:
+const files = await s3.listFiles({ prefix: 'data/' });
+
+// Process in batches of 100
+for (let i = 0; i < files.length; i += 100) {
+  const batch = files.slice(i, i + 100);
+  await Promise.all(batch.map(f => processFile(f)));
+
+  // Give GC a chance to run
+  await new Promise(resolve => setTimeout(resolve, 100));
+}
+```
+
+### 4. Use Deno/Versori Buffer Import
+
+```typescript
+// CRITICAL for Deno/Versori runtime
+import { Buffer } from 'node:buffer';
+
+const buffer = Buffer.from(content, 'utf8');
+// ... use buffer
+```
+
+## Performance Comparison
+
+### Download Methods
+
+```typescript
+// Small files (< 10 MB): Direct string download
+const content = await s3.downloadFile('small.csv'); // Fast
+const lines = content.split('\n');
+
+// Large files (> 10 MB): Binary download + chunk processing
+const buffer = await s3.downloadFile('large.csv', { encoding: 'binary' });
+processInChunks(buffer, 1024 * 1024); // 1 MB chunks
+```
+
+### Parallel vs Sequential
+
+```typescript
+// Parallel (fast, high memory)
+await Promise.all(files.map(f => processFile(f)));
+
+// Sequential (slow, low memory)
+for (const file of files) {
+  await processFile(file);
+}
+
+// Batched parallel (balanced)
+for (let i = 0; i < files.length; i += 10) {
+  await Promise.all(files.slice(i, i + 10).map(f => processFile(f)));
+}
+```
+
+## Key Takeaways
+
+- 🎯 SDK does not provide built-in streaming - use download + chunk processing
+- 🎯 Use `encoding: 'binary'` (S3) or `asBuffer: true` (SFTP) for large files to avoid string conversion overhead
+- 🎯 Process files in chunks to maintain constant memory usage
+- 🎯 Use parallel processing with controlled batch sizes (10-20 files at once)
+- 🎯 Reuse SFTP connections for multiple operations
+- 🎯 Monitor metrics to identify bottlenecks
+- 🎯 Clear large arrays periodically to prevent memory buildup
+- 🎯 Use `maxSize` option in SFTP to prevent downloading oversized files
+
+## Next Steps
+
+Continue to [Module 6: Archive Patterns](./file-operations-06-archive-patterns.md) for advanced archiving workflows.
+
+---
+
+**Related Resources:**
+- [Quick Reference](../../../02-CORE-GUIDES/advanced-services/advanced-services-quick-reference.md) - Performance tips
+- [Module 7: Testing & Troubleshooting](./file-operations-07-testing-troubleshooting.md) - Memory issues
+- [Common Patterns](../examples/common-patterns.ts) - Copy-paste ready snippets