@fluentcommerce/fc-connect-sdk 0.1.54 → 0.1.56

package/docs/03-PATTERN-GUIDES/file-operations/modules/file-operations-06-archive-patterns.md

@@ -1,386 +1,386 @@
-# Module 6: Archive Patterns
-
-**Level:** Intermediate
-**Estimated Time:** 20 minutes
-
-## Overview
-
-Learn advanced file archiving patterns including date-based organization, retention policies, and audit trails.
-
-## Learning Objectives
-
-By the end of this module, you will:
-- ✅ Implement date-based archive structures
-- ✅ Create retention policies
-- ✅ Build audit-compliant workflows
-- ✅ Handle archive organization at scale
-
-## Date-Based Archive Patterns
-
-### Simple Date (YYYY-MM-DD)
-
-```typescript
-function getSimpleDatePath(filename: string, baseFolder: string): string {
-  const date = new Date().toISOString().split('T')[0]; // 2025-01-15
-  return `${baseFolder}/${date}/${filename}`;
-}
-
-// S3
-await s3.moveFile(
-  'incoming/orders.csv',
-  getSimpleDatePath('orders.csv', 'archive')
-);
-// Result: archive/2025-01-15/orders.csv
-
-// SFTP
-await sftp.moveFile(
-  '/incoming/orders.csv',
-  '/' + getSimpleDatePath('orders.csv', 'archive')
-);
-// Result: /archive/2025-01-15/orders.csv
-```
-
-### Year/Month/Day Hierarchy
-
-```typescript
-function getHierarchicalDatePath(filename: string, baseFolder: string): string {
-  const now = new Date();
-  const year = now.getFullYear();
-  const month = String(now.getMonth() + 1).padStart(2, '0');
-  const day = String(now.getDate()).padStart(2, '0');
-
-  return `${baseFolder}/${year}/${month}/${day}/${filename}`;
-}
-
-// Result: archive/2025/01/15/orders.csv
-```
-
-### Type + Date Organization
-
-```typescript
-function getTypedDatePath(
-  filename: string,
-  dataType: string,
-  baseFolder: string = 'archive'
-): string {
-  const now = new Date();
-  const year = now.getFullYear();
-  const month = String(now.getMonth() + 1).padStart(2, '0');
-  const day = String(now.getDate()).padStart(2, '0');
-
-  return `${baseFolder}/${dataType}/${year}/${month}/${day}/${filename}`;
-}
-
-// Examples:
-// archive/orders/2025/01/15/orders-123.csv
-// archive/inventory/2025/01/15/inventory-456.csv
-// archive/products/2025/01/15/products-789.csv
-```
-
-### Timestamp-Based Archives
-
-```typescript
-function getTimestampPath(filename: string, baseFolder: string): string {
-  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
-  const basename = filename.replace(/\.[^/.]+$/, ''); // Remove extension
-  const extension = filename.match(/\.[^/.]+$/)?.[0] || '';
-
-  return `${baseFolder}/${basename}-${timestamp}${extension}`;
-}
-
-// Result: archive/orders-2025-01-15T10-30-45-123Z.csv
-```
-
-## Complete Archive Workflows
-
-### Read-Process-Archive Pattern
-
-```typescript
-async function processWithArchive(
-  s3: S3DataSource,
-  sourcePrefix: string,
-  dataType: string
-) {
-  const files = await s3.listFiles({ prefix: sourcePrefix });
-
-  for (const file of files) {
-    try {
-      // 1. Read file
-      const content = await s3.downloadFile(file.name);
-
-      // 2. Process data
-      const result = await processData(content);
-
-      // 3. Write output
-      const outputKey = file.name
-        .replace(sourcePrefix, 'output/')
-        .replace('.csv', '.json');
-      await s3.uploadFile(outputKey, JSON.stringify(result));
-
-      // 4. Archive source with date structure
-      const archiveKey = getTypedDatePath(
-        file.name.split('/').pop()!,
-        dataType
-      );
-      await s3.moveFile(file.name, archiveKey);
-
-      console.log(`Processed and archived: ${file.name}`);
-
-    } catch (error) {
-      console.error(`Failed to process ${file.name}:`, error);
-
-      // Move to error folder instead of archive
-      const errorKey = file.name.replace(sourcePrefix, 'errors/');
-      await s3.moveFile(file.name, errorKey);
-    }
-  }
-}
-
-// Usage
-await processWithArchive(s3, 'incoming/orders/', 'orders');
-```
-
-### Multi-Folder Archive Pattern
-
-```typescript
-async function processMultipleFolders(s3: S3DataSource) {
-  const folders = [
-    { name: 'orders', extension: '.csv' },
-    { name: 'inventory', extension: '.csv' },
-    { name: 'products', extension: '.json' }
-  ];
-
-  for (const folder of folders) {
-    const allFiles = await s3.listFiles({
-      prefix: `incoming/${folder.name}/`
-    });
-
-    // Filter by extension after listing
-    const files = allFiles.filter(f => f.name.endsWith(folder.extension));
-
-    for (const file of files) {
-      const content = await s3.downloadFile(file.name);
-      await processByType(folder.name, content);
-
-      // Archive with type + date structure
-      const filename = file.name.split('/').pop()!;
-      const archiveKey = getTypedDatePath(filename, folder.name);
-
-      await s3.moveFile(file.name, archiveKey);
-    }
-
-    console.log(`Processed ${files.length} ${folder.name} files`);
-  }
-}
-```
-
-## Retention Policies
-
-### Age-Based Cleanup
-
-```typescript
-async function cleanupOldArchives(
-  s3: S3DataSource,
-  archivePrefix: string,
-  retentionDays: number
-) {
-  const files = await s3.listFiles({ prefix: archivePrefix });
-
-  const cutoffDate = new Date();
-  cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
-
-  let deletedCount = 0;
-
-  for (const file of files) {
-    if (file.lastModified && file.lastModified < cutoffDate) {
-      await s3.deleteFile(file.name);
-      deletedCount++;
-      console.log(`Deleted old file: ${file.name}`);
-    }
-  }
-
-  console.log(`Cleaned up ${deletedCount} files older than ${retentionDays} days`);
-}
-
-// Delete archives older than 90 days
-await cleanupOldArchives(s3, 'archive/', 90);
-```
-
-### Size-Based Retention
-
-```typescript
-async function cleanupBySize(
-  s3: S3DataSource,
-  archivePrefix: string,
-  maxTotalBytes: number
-) {
-  const files = await s3.listFiles({ prefix: archivePrefix });
-
-  // Sort by age (oldest first)
-  files.sort((a, b) => {
-    const timeA = a.lastModified?.getTime() || 0;
-    const timeB = b.lastModified?.getTime() || 0;
-    return timeA - timeB;
-  });
-
-  let totalSize = files.reduce((sum, file) => sum + (file.size || 0), 0);
-
-  for (const file of files) {
-    if (totalSize <= maxTotalBytes) break;
-
-    await s3.deleteFile(file.name);
-    totalSize -= file.size || 0;
-
-    console.log(`Deleted to free space: ${file.name}`);
-  }
-}
-
-// Keep total archive size under 100GB
-await cleanupBySize(s3, 'archive/', 100 * 1024 * 1024 * 1024);
-```
-
-## Audit-Compliant Archiving
-
-### Immutable Archive with Metadata
-
-```typescript
-interface ArchiveMetadata {
-  originalPath: string;
-  archiveTime: string;
-  fileSize: number;
-  checksum: string;
-  processedBy: string;
-  recordCount: number;
-}
-
-async function archiveWithAudit(
-  s3: S3DataSource,
-  sourceKey: string,
-  dataType: string
-) {
-  // 1. Read and process file
-  const content = await s3.downloadFile(sourceKey);
-  const result = await processData(content);
-
-  // 2. Calculate metadata
-  const metadata: ArchiveMetadata = {
-    originalPath: sourceKey,
-    archiveTime: new Date().toISOString(),
-    fileSize: content.length,
-    checksum: calculateChecksum(content),
-    processedBy: 'file-processor-v1',
-    recordCount: content.split('\n').length - 1
-  };
-
-  // 3. Archive file with date structure
-  const filename = sourceKey.split('/').pop()!;
-  const archiveKey = getTypedDatePath(filename, dataType);
-  await s3.moveFile(sourceKey, archiveKey);
-
-  // 4. Write audit metadata
-  const metadataKey = archiveKey.replace(/\.[^/.]+$/, '.metadata.json');
-  await s3.uploadFile(metadataKey, JSON.stringify(metadata, null, 2));
-
-  console.log(`Archived with audit trail: ${archiveKey}`);
-}
-
-function calculateChecksum(content: string): string {
-  const crypto = require('crypto');
-  return crypto.createHash('sha256').update(content).digest('hex');
-}
-```
-
-### Archive Verification
-
-```typescript
-async function verifyArchive(s3: S3DataSource, archiveKey: string): Promise<boolean> {
-  try {
-    // Read archive file
-    const content = await s3.downloadFile(archiveKey);
-
-    // Read metadata
-    const metadataKey = archiveKey.replace(/\.[^/.]+$/, '.metadata.json');
-    const metadataJson = await s3.downloadFile(metadataKey);
-    const metadata: ArchiveMetadata = JSON.parse(metadataJson);
-
-    // Verify checksum
-    const actualChecksum = calculateChecksum(content);
-
-    if (actualChecksum !== metadata.checksum) {
-      console.error('Checksum mismatch!');
-      return false;
-    }
-
-    // Verify file size
-    if (content.length !== metadata.fileSize) {
-      console.error('File size mismatch!');
-      return false;
-    }
-
-    console.log('Archive verification passed');
-    return true;
-
-  } catch (error) {
-    console.error('Archive verification failed:', error);
-    return false;
-  }
-}
-```
-
-## Error Handling in Archives
-
-### Archive with Error Segregation
-
-```typescript
-async function processWithErrorHandling(s3: S3DataSource, sourcePrefix: string) {
-  const files = await s3.listFiles({ prefix: sourcePrefix });
-
-  for (const file of files) {
-    try {
-      const content = await s3.downloadFile(file.name);
-      await processData(content);
-
-      // Success: Archive to processed folder
-      const archiveKey = file.name.replace(sourcePrefix, 'archive/processed/');
-      await s3.moveFile(file.name, archiveKey);
-
-    } catch (error) {
-      console.error(`Processing failed for ${file.name}:`, error);
-
-      // Error: Archive to error folder with timestamp
-      const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
-      const filename = file.name.split('/').pop()!;
-      const errorKey = `archive/errors/${timestamp}/${filename}`;
-
-      await s3.moveFile(file.name, errorKey);
-
-      // Write error details
-      const errorLogKey = `archive/errors/${timestamp}/${filename}.error.log`;
-      await s3.uploadFile(errorLogKey, JSON.stringify({
-        file: file.name,
-        error: error.message,
-        timestamp: new Date().toISOString()
-      }, null, 2));
-    }
-  }
-}
-```
-
-## Key Takeaways
-
-- 🎯 Use year/month/day hierarchy for better organization and performance
-- 🎯 Combine data type + date for multi-source workflows
-- 🎯 Implement retention policies to manage storage costs
-- 🎯 Add audit metadata for compliance requirements
-- 🎯 Segregate successful and failed processing into different archive paths
-- 🎯 Never delete source files immediately; archive for recovery
-
-## Next Steps
-
-Continue to [Module 7: Testing & Troubleshooting](./file-operations-07-testing-troubleshooting.md) to learn about test coverage and debugging.
-
----
-
-**Related Resources:**
-- [Quick Reference](../../../02-CORE-GUIDES/advanced-services/advanced-services-quick-reference.md) - Archive pattern examples
-- [Common Patterns](../examples/common-patterns.ts) - Copy-paste ready snippets
+# Module 6: Archive Patterns
+
+**Level:** Intermediate
+**Estimated Time:** 20 minutes
+
+## Overview
+
+Learn advanced file archiving patterns including date-based organization, retention policies, and audit trails.
+
+## Learning Objectives
+
+By the end of this module, you will:
+- ✅ Implement date-based archive structures
+- ✅ Create retention policies
+- ✅ Build audit-compliant workflows
+- ✅ Handle archive organization at scale
+
+## Date-Based Archive Patterns
+
+### Simple Date (YYYY-MM-DD)
+
+```typescript
+function getSimpleDatePath(filename: string, baseFolder: string): string {
+  const date = new Date().toISOString().split('T')[0]; // 2025-01-15
+  return `${baseFolder}/${date}/${filename}`;
+}
+
+// S3
+await s3.moveFile(
+  'incoming/orders.csv',
+  getSimpleDatePath('orders.csv', 'archive')
+);
+// Result: archive/2025-01-15/orders.csv
+
+// SFTP
+await sftp.moveFile(
+  '/incoming/orders.csv',
+  '/' + getSimpleDatePath('orders.csv', 'archive')
+);
+// Result: /archive/2025-01-15/orders.csv
+```
+
+### Year/Month/Day Hierarchy
+
+```typescript
+function getHierarchicalDatePath(filename: string, baseFolder: string): string {
+  const now = new Date();
+  const year = now.getFullYear();
+  const month = String(now.getMonth() + 1).padStart(2, '0');
+  const day = String(now.getDate()).padStart(2, '0');
+
+  return `${baseFolder}/${year}/${month}/${day}/${filename}`;
+}
+
+// Result: archive/2025/01/15/orders.csv
+```
+
+### Type + Date Organization
+
+```typescript
+function getTypedDatePath(
+  filename: string,
+  dataType: string,
+  baseFolder: string = 'archive'
+): string {
+  const now = new Date();
+  const year = now.getFullYear();
+  const month = String(now.getMonth() + 1).padStart(2, '0');
+  const day = String(now.getDate()).padStart(2, '0');
+
+  return `${baseFolder}/${dataType}/${year}/${month}/${day}/${filename}`;
+}
+
+// Examples:
+// archive/orders/2025/01/15/orders-123.csv
+// archive/inventory/2025/01/15/inventory-456.csv
+// archive/products/2025/01/15/products-789.csv
+```
+
+### Timestamp-Based Archives
+
+```typescript
+function getTimestampPath(filename: string, baseFolder: string): string {
+  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const basename = filename.replace(/\.[^/.]+$/, ''); // Remove extension
+  const extension = filename.match(/\.[^/.]+$/)?.[0] || '';
+
+  return `${baseFolder}/${basename}-${timestamp}${extension}`;
+}
+
+// Result: archive/orders-2025-01-15T10-30-45-123Z.csv
+```
+
+## Complete Archive Workflows
+
+### Read-Process-Archive Pattern
+
+```typescript
+async function processWithArchive(
+  s3: S3DataSource,
+  sourcePrefix: string,
+  dataType: string
+) {
+  const files = await s3.listFiles({ prefix: sourcePrefix });
+
+  for (const file of files) {
+    try {
+      // 1. Read file
+      const content = await s3.downloadFile(file.name);
+
+      // 2. Process data
+      const result = await processData(content);
+
+      // 3. Write output
+      const outputKey = file.name
+        .replace(sourcePrefix, 'output/')
+        .replace('.csv', '.json');
+      await s3.uploadFile(outputKey, JSON.stringify(result));
+
+      // 4. Archive source with date structure
+      const archiveKey = getTypedDatePath(
+        file.name.split('/').pop()!,
+        dataType
+      );
+      await s3.moveFile(file.name, archiveKey);
+
+      console.log(`Processed and archived: ${file.name}`);
+
+    } catch (error) {
+      console.error(`Failed to process ${file.name}:`, error);
+
+      // Move to error folder instead of archive
+      const errorKey = file.name.replace(sourcePrefix, 'errors/');
+      await s3.moveFile(file.name, errorKey);
+    }
+  }
+}
+
+// Usage
+await processWithArchive(s3, 'incoming/orders/', 'orders');
+```
+
+### Multi-Folder Archive Pattern
+
+```typescript
+async function processMultipleFolders(s3: S3DataSource) {
+  const folders = [
+    { name: 'orders', extension: '.csv' },
+    { name: 'inventory', extension: '.csv' },
+    { name: 'products', extension: '.json' }
+  ];
+
+  for (const folder of folders) {
+    const allFiles = await s3.listFiles({
+      prefix: `incoming/${folder.name}/`
+    });
+
+    // Filter by extension after listing
+    const files = allFiles.filter(f => f.name.endsWith(folder.extension));
+
+    for (const file of files) {
+      const content = await s3.downloadFile(file.name);
+      await processByType(folder.name, content);
+
+      // Archive with type + date structure
+      const filename = file.name.split('/').pop()!;
+      const archiveKey = getTypedDatePath(filename, folder.name);
+
+      await s3.moveFile(file.name, archiveKey);
+    }
+
+    console.log(`Processed ${files.length} ${folder.name} files`);
+  }
+}
+```
+
+## Retention Policies
+
+### Age-Based Cleanup
+
+```typescript
+async function cleanupOldArchives(
+  s3: S3DataSource,
+  archivePrefix: string,
+  retentionDays: number
+) {
+  const files = await s3.listFiles({ prefix: archivePrefix });
+
+  const cutoffDate = new Date();
+  cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
+
+  let deletedCount = 0;
+
+  for (const file of files) {
+    if (file.lastModified && file.lastModified < cutoffDate) {
+      await s3.deleteFile(file.name);
+      deletedCount++;
+      console.log(`Deleted old file: ${file.name}`);
+    }
+  }
+
+  console.log(`Cleaned up ${deletedCount} files older than ${retentionDays} days`);
+}
+
+// Delete archives older than 90 days
+await cleanupOldArchives(s3, 'archive/', 90);
+```
+
+### Size-Based Retention
+
+```typescript
+async function cleanupBySize(
+  s3: S3DataSource,
+  archivePrefix: string,
+  maxTotalBytes: number
+) {
+  const files = await s3.listFiles({ prefix: archivePrefix });
+
+  // Sort by age (oldest first)
+  files.sort((a, b) => {
+    const timeA = a.lastModified?.getTime() || 0;
+    const timeB = b.lastModified?.getTime() || 0;
+    return timeA - timeB;
+  });
+
+  let totalSize = files.reduce((sum, file) => sum + (file.size || 0), 0);
+
+  for (const file of files) {
+    if (totalSize <= maxTotalBytes) break;
+
+    await s3.deleteFile(file.name);
+    totalSize -= file.size || 0;
+
+    console.log(`Deleted to free space: ${file.name}`);
+  }
+}
+
+// Keep total archive size under 100GB
+await cleanupBySize(s3, 'archive/', 100 * 1024 * 1024 * 1024);
+```
+
+## Audit-Compliant Archiving
+
+### Immutable Archive with Metadata
+
+```typescript
+interface ArchiveMetadata {
+  originalPath: string;
+  archiveTime: string;
+  fileSize: number;
+  checksum: string;
+  processedBy: string;
+  recordCount: number;
+}
+
+async function archiveWithAudit(
+  s3: S3DataSource,
+  sourceKey: string,
+  dataType: string
+) {
+  // 1. Read and process file
+  const content = await s3.downloadFile(sourceKey);
+  const result = await processData(content);
+
+  // 2. Calculate metadata
+  const metadata: ArchiveMetadata = {
+    originalPath: sourceKey,
+    archiveTime: new Date().toISOString(),
+    fileSize: content.length,
+    checksum: calculateChecksum(content),
+    processedBy: 'file-processor-v1',
+    recordCount: content.split('\n').length - 1
+  };
+
+  // 3. Archive file with date structure
+  const filename = sourceKey.split('/').pop()!;
+  const archiveKey = getTypedDatePath(filename, dataType);
+  await s3.moveFile(sourceKey, archiveKey);
+
+  // 4. Write audit metadata
+  const metadataKey = archiveKey.replace(/\.[^/.]+$/, '.metadata.json');
+  await s3.uploadFile(metadataKey, JSON.stringify(metadata, null, 2));
+
+  console.log(`Archived with audit trail: ${archiveKey}`);
+}
+
+function calculateChecksum(content: string): string {
+  const crypto = require('crypto');
+  return crypto.createHash('sha256').update(content).digest('hex');
+}
+```
+
+### Archive Verification
+
+```typescript
+async function verifyArchive(s3: S3DataSource, archiveKey: string): Promise<boolean> {
+  try {
+    // Read archive file
+    const content = await s3.downloadFile(archiveKey);
+
+    // Read metadata
+    const metadataKey = archiveKey.replace(/\.[^/.]+$/, '.metadata.json');
+    const metadataJson = await s3.downloadFile(metadataKey);
+    const metadata: ArchiveMetadata = JSON.parse(metadataJson);
+
+    // Verify checksum
+    const actualChecksum = calculateChecksum(content);
+
+    if (actualChecksum !== metadata.checksum) {
+      console.error('Checksum mismatch!');
+      return false;
+    }
+
+    // Verify file size
+    if (content.length !== metadata.fileSize) {
+      console.error('File size mismatch!');
+      return false;
+    }
+
+    console.log('Archive verification passed');
+    return true;
+
+  } catch (error) {
+    console.error('Archive verification failed:', error);
+    return false;
+  }
+}
+```
+
+## Error Handling in Archives
+
+### Archive with Error Segregation
+
+```typescript
+async function processWithErrorHandling(s3: S3DataSource, sourcePrefix: string) {
+  const files = await s3.listFiles({ prefix: sourcePrefix });
+
+  for (const file of files) {
+    try {
+      const content = await s3.downloadFile(file.name);
+      await processData(content);
+
+      // Success: Archive to processed folder
+      const archiveKey = file.name.replace(sourcePrefix, 'archive/processed/');
+      await s3.moveFile(file.name, archiveKey);
+
+    } catch (error) {
+      console.error(`Processing failed for ${file.name}:`, error);
+
+      // Error: Archive to error folder with timestamp
+      const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+      const filename = file.name.split('/').pop()!;
+      const errorKey = `archive/errors/${timestamp}/${filename}`;
+
+      await s3.moveFile(file.name, errorKey);
+
+      // Write error details
+      const errorLogKey = `archive/errors/${timestamp}/${filename}.error.log`;
+      await s3.uploadFile(errorLogKey, JSON.stringify({
+        file: file.name,
+        error: error.message,
+        timestamp: new Date().toISOString()
+      }, null, 2));
+    }
+  }
+}
+```
+
+## Key Takeaways
+
+- 🎯 Use year/month/day hierarchy for better organization and performance
+- 🎯 Combine data type + date for multi-source workflows
+- 🎯 Implement retention policies to manage storage costs
+- 🎯 Add audit metadata for compliance requirements
+- 🎯 Segregate successful and failed processing into different archive paths
+- 🎯 Never delete source files immediately; archive for recovery
+
+## Next Steps
+
+Continue to [Module 7: Testing & Troubleshooting](./file-operations-07-testing-troubleshooting.md) to learn about test coverage and debugging.
+
+---
+
+**Related Resources:**
+- [Quick Reference](../../../02-CORE-GUIDES/advanced-services/advanced-services-quick-reference.md) - Archive pattern examples
+- [Common Patterns](../examples/common-patterns.ts) - Copy-paste ready snippets