@fluentcommerce/fc-connect-sdk 0.1.54 → 0.1.56

package/docs/02-CORE-GUIDES/data-sources/examples/common-patterns.ts

@@ -1,409 +1,409 @@
-/**
- * Data Sources Common Patterns
- *
- * Reusable patterns combining S3 and SFTP data sources for typical integration scenarios.
- * Shows error handling, retry logic, and best practices.
- *
- * @package @fluentcommerce/fc-connect-sdk
- */
-
-import { S3DataSource, SftpDataSource } from '@fluentcommerce/fc-connect-sdk';
-
-/**
- * Pattern 1: Download with Retry Logic
- *
- * Automatically retry failed downloads with exponential backoff
- */
-export async function downloadWithRetry<T>(
-  downloadFn: () => Promise<T>,
-  maxRetries: number = 3,
-  initialDelayMs: number = 1000
-): Promise<T> {
-  let lastError: Error | undefined;
-
-  for (let attempt = 0; attempt < maxRetries; attempt++) {
-    try {
-      return await downloadFn();
-    } catch (error) {
-      lastError = error as Error;
-      const delay = initialDelayMs * Math.pow(2, attempt);
-
-      console.warn(`Attempt ${attempt + 1} failed, retrying in ${delay}ms...`);
-
-      if (attempt < maxRetries - 1) {
-        await new Promise(resolve => setTimeout(resolve, delay));
-      }
-    }
-  }
-
-  throw new Error(`Failed after ${maxRetries} attempts: ${lastError?.message}`);
-}
-
-/**
- * Pattern 2: Safe File Download with Validation
- *
- * Download file and validate it exists and has content
- */
-export async function safeDownload(
-  source: S3DataSource | SftpDataSource,
-  filePath: string,
-  options: any = {}
-): Promise<string | Buffer> {
-  try {
-    // For S3: list and filter if you need existence checks; otherwise attempt download and handle 404
-    const content = await source.downloadFile(filePath, options);
-
-    // Validate content
-    if (!content || content.length === 0) {
-      throw new Error(`Downloaded file is empty: ${filePath}`);
-    }
-
-    console.log(`✓ Downloaded ${filePath} (${content.length} bytes)`);
-    return content;
-  } catch (error) {
-    console.error(`Failed to download ${filePath}:`, error);
-    throw error;
-  }
-}
-
-/**
- * Pattern 3: Batch Download with Progress Tracking
- *
- * Download multiple files with progress updates
- */
-export async function batchDownload(
-  source: S3DataSource | SftpDataSource,
-  fileKeys: string[],
-  options: any = {}
-): Promise<Array<{ key: string; content: string | Buffer; success: boolean }>> {
-  const results: Array<{ key: string; content: string | Buffer; success: boolean }> = [];
-  let completed = 0;
-
-  console.log(`Downloading ${fileKeys.length} files...`);
-
-  for (const key of fileKeys) {
-    try {
-      const content = await source.downloadFile(key, options);
-      results.push({ key, content, success: true });
-      completed++;
-      console.log(`Progress: ${completed}/${fileKeys.length} (${key})`);
-    } catch (error) {
-      console.error(`Failed to download ${key}:`, error);
-      results.push({ key, content: '', success: false });
-    }
-  }
-
-  const successful = results.filter(r => r.success).length;
-  console.log(`✓ Downloaded ${successful}/${fileKeys.length} files successfully`);
-
-  return results;
-}
-
-/**
- * Pattern 4: Conditional Upload (only if changed)
- *
- * Upload file only if content has changed
- */
-export async function uploadIfChanged(
-  source: S3DataSource | SftpDataSource,
-  filePath: string,
-  newContent: string | Buffer,
-  options: any = {}
-): Promise<boolean> {
-  try {
-    // Try to download existing file (best-effort)
-    let existingContent: string | Buffer | undefined;
-    try {
-      existingContent = await source.downloadFile(filePath, options);
-    } catch {
-      // Not found; proceed
-    }
-
-    if (existingContent) {
-      const existingStr =
-        typeof existingContent === 'string' ? existingContent : existingContent.toString();
-      const newStr = typeof newContent === 'string' ? newContent : newContent.toString();
-
-      if (existingStr === newStr) {
-        console.log(`Skipping upload, content unchanged: ${filePath}`);
-        return false;
-      }
-    }
-
-    // Upload
-    await source.uploadFile(filePath, newContent, options);
-    console.log(`✓ Uploaded ${filePath}`);
-    return true;
-  } catch (error) {
-    console.error(`Failed to upload ${filePath}:`, error);
-    throw error;
-  }
-}
-
-/**
- * Pattern 5: Archive After Processing
- *
- * Process file and move to archive location
- */
-export async function processAndArchive(
-  source: S3DataSource,
-  sourceKey: string,
-  processFn: (content: string | Buffer) => Promise<void>,
-  options: { bucket: string }
-): Promise<void> {
-  const archiveKey = sourceKey.replace('incoming/', 'archive/');
-  const timestamp = new Date().toISOString();
-
-  try {
-    // Download and process
-    const content = await source.downloadFile(sourceKey, options);
-    await processFn(content);
-
-    // Upload to archive with metadata
-    await source.uploadFile(archiveKey, content, {
-      ...options,
-      metadata: {
-        processedAt: timestamp,
-        originalKey: sourceKey,
-      },
-    });
-
-    // Delete original
-    await source.deleteFile(sourceKey);
-
-    console.log(`✓ Processed and archived: ${sourceKey} → ${archiveKey}`);
-  } catch (error) {
-    console.error(`Failed to process and archive ${sourceKey}:`, error);
-    throw error;
-  }
-}
-
-/**
- * Pattern 6: Error File Generation
- *
- * Create error file when processing fails
- */
-export async function handleProcessingError(
-  source: S3DataSource,
-  originalKey: string,
-  error: Error,
-  options: { bucket: string }
-): Promise<void> {
-  const errorKey = originalKey.replace('incoming/', 'errors/') + '.error.json';
-
-  const errorReport = {
-    timestamp: new Date().toISOString(),
-    originalFile: originalKey,
-    error: {
-      message: error.message,
-      stack: error.stack,
-    },
-  };
-
-  try {
-    await source.uploadFile(errorKey, JSON.stringify(errorReport, null, 2), {
-      ...options,
-      contentType: 'application/json',
-    });
-
-    console.log(`✓ Created error report: ${errorKey}`);
-  } catch (uploadError) {
-    console.error('Failed to create error report:', uploadError);
-  }
-}
-
-/**
- * Pattern 7: Multi-Source Download (try SFTP, fallback to S3)
- *
- * Try to download from SFTP first, fall back to S3 if unavailable
- */
-export async function multiSourceDownload(
-  sftpSource: SftpDataSource,
-  s3Source: S3DataSource,
-  fileName: string,
-  sftpPath: string,
-  s3Options: { bucket: string; key: string }
-): Promise<{ content: string | Buffer; source: 'sftp' | 's3' }> {
-  // Try SFTP first
-  try {
-    const content = await sftpSource.downloadFile(sftpPath);
-
-    console.log(`✓ Downloaded from SFTP: ${fileName}`);
-    return { content, source: 'sftp' };
-  } catch (sftpError) {
-    console.warn('SFTP download failed, trying S3...', sftpError);
-  }
-
-  // Fallback to S3
-  try {
-    const content = await s3Source.downloadFile(s3Options.key, {});
-
-    console.log(`✓ Downloaded from S3: ${fileName}`);
-    return { content, source: 's3' };
-  } catch (s3Error) {
-    throw new Error(`Failed to download from both SFTP and S3: ${fileName}`);
-  }
-}
-
-/**
- * Pattern 8: Directory Synchronization
- *
- * Sync entire directory from source to destination
- */
-export async function syncDirectory(
-  sourceS3: S3DataSource,
-  destS3: S3DataSource,
-  sourceBucket: string,
-  destBucket: string,
-  prefix: string
-): Promise<number> {
-  try {
-    // List all files in source
-    const files = await sourceS3.listFiles({ prefix });
-
-    console.log(`Syncing ${files.length} files from ${prefix}`);
-
-    let syncedCount = 0;
-
-    for (const file of files) {
-      const path = file.path;
-
-      // Download from source
-      const content = await sourceS3.downloadFile(path);
-
-      // Upload to destination
-      await destS3.uploadFile(path, content);
-
-      syncedCount++;
-      console.log(`  Synced: ${path} (${syncedCount}/${files.length})`);
-    }
-
-    console.log(`✓ Directory sync complete: ${syncedCount} files`);
-    return syncedCount;
-  } catch (error) {
-    console.error('Directory sync failed:', error);
-    throw error;
-  }
-}
-
-/**
- * Pattern 9: Incremental Backup
- *
- * Back up only files modified since last backup
- */
-export async function incrementalBackup(
-  source: S3DataSource,
-  lastBackupTime: Date,
-  sourceBucket: string,
-  backupBucket: string,
-  prefix: string
-): Promise<number> {
-  try {
-    const files = await source.listFiles({ prefix });
-
-    // Filter files modified after last backup
-    const filesToBackup = files.filter(
-      f => f.lastModified && new Date(f.lastModified) > lastBackupTime
-    );
-
-    console.log(
-      `Found ${filesToBackup.length} files modified since ${lastBackupTime.toISOString()}`
-    );
-
-    for (const file of filesToBackup) {
-      const path = file.path;
-      const content = await source.downloadFile(path);
-
-      // Add backup timestamp to path
-      const backupPath = `backups/${new Date().toISOString().split('T')[0]}/${path}`;
-
-      await source.uploadFile(backupPath, content);
-      console.log(`  Backed up: ${path}`);
-    }
-
-    console.log(`✓ Incremental backup complete: ${filesToBackup.length} files`);
-    return filesToBackup.length;
-  } catch (error) {
-    console.error('Incremental backup failed:', error);
-    throw error;
-  }
-}
-
-/**
- * Pattern 10: Health Check
- *
- * Verify data source connectivity and permissions
- */
-export async function healthCheck(
-  source: S3DataSource | SftpDataSource
-): Promise<{ healthy: boolean; message: string }> {
-  try {
-    if (source instanceof SftpDataSource) {
-      try {
-        const ok = await source.validateConnection();
-        return {
-          healthy: ok,
-          message: ok ? 'SFTP connection successful' : 'SFTP validation failed',
-        };
-      } catch (e: any) {
-        return { healthy: false, message: `SFTP validation error: ${e.message}` };
-      }
-    } else {
-      const ok = await (source as S3DataSource).validateConnection();
-      return { healthy: ok, message: ok ? 'S3 connection successful' : 'S3 validation failed' };
-    }
-  } catch (error) {
-    return {
-      healthy: false,
-      message: `Health check failed: ${(error as Error).message}`,
-    };
-  }
-}
-
-/**
- * Usage Example
- */
-async function main() {
-  const s3 = new S3DataSource({
-    type: 'S3_CSV',
-    s3Config: {
-      accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
-      region: process.env.AWS_REGION!,
-      bucket: process.env.S3_BUCKET!,
-    },
-  } as any);
-
-  const sftp = new SftpDataSource({
-    type: 'SFTP_CSV',
-    connectionId: 'example',
-    settings: {
-      host: process.env.SFTP_HOST!,
-      port: parseInt(process.env.SFTP_PORT || '22'),
-      username: process.env.SFTP_USERNAME!,
-      password: process.env.SFTP_PASSWORD!,
-      remotePath: '/',
-      filePattern: '*',
-    },
-  } as any);
-
-  // Example 1: Download with retry
-  await downloadWithRetry(() => s3.downloadFile('test.csv'), 3);
-
-  // Example 2: Safe download
-  await safeDownload(s3, 'test.csv');
-
-  // Example 3: Batch download
-  await batchDownload(s3, ['file1.csv', 'file2.csv']);
-
-  // Example 4: Health check
-  const s3Health = await healthCheck(s3);
-  console.log('S3 Health:', s3Health);
-
-  const sftpHealth = await healthCheck(sftp);
-  console.log('SFTP Health:', sftpHealth);
-}
-
-if (require.main === module) {
-  main().catch(console.error);
-}
+/**
+ * Data Sources Common Patterns
+ *
+ * Reusable patterns combining S3 and SFTP data sources for typical integration scenarios.
+ * Shows error handling, retry logic, and best practices.
+ *
+ * @package @fluentcommerce/fc-connect-sdk
+ */
+
+import { S3DataSource, SftpDataSource } from '@fluentcommerce/fc-connect-sdk';
+
+/**
+ * Pattern 1: Download with Retry Logic
+ *
+ * Automatically retry failed downloads with exponential backoff
+ */
+export async function downloadWithRetry<T>(
+  downloadFn: () => Promise<T>,
+  maxRetries: number = 3,
+  initialDelayMs: number = 1000
+): Promise<T> {
+  let lastError: Error | undefined;
+
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
+    try {
+      return await downloadFn();
+    } catch (error) {
+      lastError = error as Error;
+      const delay = initialDelayMs * Math.pow(2, attempt);
+
+      console.warn(`Attempt ${attempt + 1} failed, retrying in ${delay}ms...`);
+
+      if (attempt < maxRetries - 1) {
+        await new Promise(resolve => setTimeout(resolve, delay));
+      }
+    }
+  }
+
+  throw new Error(`Failed after ${maxRetries} attempts: ${lastError?.message}`);
+}
+
+/**
+ * Pattern 2: Safe File Download with Validation
+ *
+ * Download file and validate it exists and has content
+ */
+export async function safeDownload(
+  source: S3DataSource | SftpDataSource,
+  filePath: string,
+  options: any = {}
+): Promise<string | Buffer> {
+  try {
+    // For S3: list and filter if you need existence checks; otherwise attempt download and handle 404
+    const content = await source.downloadFile(filePath, options);
+
+    // Validate content
+    if (!content || content.length === 0) {
+      throw new Error(`Downloaded file is empty: ${filePath}`);
+    }
+
+    console.log(`✓ Downloaded ${filePath} (${content.length} bytes)`);
+    return content;
+  } catch (error) {
+    console.error(`Failed to download ${filePath}:`, error);
+    throw error;
+  }
+}
+
+/**
+ * Pattern 3: Batch Download with Progress Tracking
+ *
+ * Download multiple files with progress updates
+ */
+export async function batchDownload(
+  source: S3DataSource | SftpDataSource,
+  fileKeys: string[],
+  options: any = {}
+): Promise<Array<{ key: string; content: string | Buffer; success: boolean }>> {
+  const results: Array<{ key: string; content: string | Buffer; success: boolean }> = [];
+  let completed = 0;
+
+  console.log(`Downloading ${fileKeys.length} files...`);
+
+  for (const key of fileKeys) {
+    try {
+      const content = await source.downloadFile(key, options);
+      results.push({ key, content, success: true });
+      completed++;
+      console.log(`Progress: ${completed}/${fileKeys.length} (${key})`);
+    } catch (error) {
+      console.error(`Failed to download ${key}:`, error);
+      results.push({ key, content: '', success: false });
+    }
+  }
+
+  const successful = results.filter(r => r.success).length;
+  console.log(`✓ Downloaded ${successful}/${fileKeys.length} files successfully`);
+
+  return results;
+}
+
+/**
+ * Pattern 4: Conditional Upload (only if changed)
+ *
+ * Upload file only if content has changed
+ */
+export async function uploadIfChanged(
+  source: S3DataSource | SftpDataSource,
+  filePath: string,
+  newContent: string | Buffer,
+  options: any = {}
+): Promise<boolean> {
+  try {
+    // Try to download existing file (best-effort)
+    let existingContent: string | Buffer | undefined;
+    try {
+      existingContent = await source.downloadFile(filePath, options);
+    } catch {
+      // Not found; proceed
+    }
+
+    if (existingContent) {
+      const existingStr =
+        typeof existingContent === 'string' ? existingContent : existingContent.toString();
+      const newStr = typeof newContent === 'string' ? newContent : newContent.toString();
+
+      if (existingStr === newStr) {
+        console.log(`Skipping upload, content unchanged: ${filePath}`);
+        return false;
+      }
+    }
+
+    // Upload
+    await source.uploadFile(filePath, newContent, options);
+    console.log(`✓ Uploaded ${filePath}`);
+    return true;
+  } catch (error) {
+    console.error(`Failed to upload ${filePath}:`, error);
+    throw error;
+  }
+}
+
+/**
+ * Pattern 5: Archive After Processing
+ *
+ * Process file and move to archive location
+ */
+export async function processAndArchive(
+  source: S3DataSource,
+  sourceKey: string,
+  processFn: (content: string | Buffer) => Promise<void>,
+  options: { bucket: string }
+): Promise<void> {
+  const archiveKey = sourceKey.replace('incoming/', 'archive/');
+  const timestamp = new Date().toISOString();
+
+  try {
+    // Download and process
+    const content = await source.downloadFile(sourceKey, options);
+    await processFn(content);
+
+    // Upload to archive with metadata
+    await source.uploadFile(archiveKey, content, {
+      ...options,
+      metadata: {
+        processedAt: timestamp,
+        originalKey: sourceKey,
+      },
+    });
+
+    // Delete original
+    await source.deleteFile(sourceKey);
+
+    console.log(`✓ Processed and archived: ${sourceKey} → ${archiveKey}`);
+  } catch (error) {
+    console.error(`Failed to process and archive ${sourceKey}:`, error);
+    throw error;
+  }
+}
+
+/**
+ * Pattern 6: Error File Generation
+ *
+ * Create error file when processing fails
+ */
+export async function handleProcessingError(
+  source: S3DataSource,
+  originalKey: string,
+  error: Error,
+  options: { bucket: string }
+): Promise<void> {
+  const errorKey = originalKey.replace('incoming/', 'errors/') + '.error.json';
+
+  const errorReport = {
+    timestamp: new Date().toISOString(),
+    originalFile: originalKey,
+    error: {
+      message: error.message,
+      stack: error.stack,
+    },
+  };
+
+  try {
+    await source.uploadFile(errorKey, JSON.stringify(errorReport, null, 2), {
+      ...options,
+      contentType: 'application/json',
+    });
+
+    console.log(`✓ Created error report: ${errorKey}`);
+  } catch (uploadError) {
+    console.error('Failed to create error report:', uploadError);
+  }
+}
+
+/**
+ * Pattern 7: Multi-Source Download (try SFTP, fallback to S3)
+ *
+ * Try to download from SFTP first, fall back to S3 if unavailable
+ */
+export async function multiSourceDownload(
+  sftpSource: SftpDataSource,
+  s3Source: S3DataSource,
+  fileName: string,
+  sftpPath: string,
+  s3Options: { bucket: string; key: string }
+): Promise<{ content: string | Buffer; source: 'sftp' | 's3' }> {
+  // Try SFTP first
+  try {
+    const content = await sftpSource.downloadFile(sftpPath);
+
+    console.log(`✓ Downloaded from SFTP: ${fileName}`);
+    return { content, source: 'sftp' };
+  } catch (sftpError) {
+    console.warn('SFTP download failed, trying S3...', sftpError);
+  }
+
+  // Fallback to S3
+  try {
+    const content = await s3Source.downloadFile(s3Options.key, {});
+
+    console.log(`✓ Downloaded from S3: ${fileName}`);
+    return { content, source: 's3' };
+  } catch (s3Error) {
+    throw new Error(`Failed to download from both SFTP and S3: ${fileName}`);
+  }
+}
+
+/**
+ * Pattern 8: Directory Synchronization
+ *
+ * Sync entire directory from source to destination
+ */
+export async function syncDirectory(
+  sourceS3: S3DataSource,
+  destS3: S3DataSource,
+  sourceBucket: string,
+  destBucket: string,
+  prefix: string
+): Promise<number> {
+  try {
+    // List all files in source
+    const files = await sourceS3.listFiles({ prefix });
+
+    console.log(`Syncing ${files.length} files from ${prefix}`);
+
+    let syncedCount = 0;
+
+    for (const file of files) {
+      const path = file.path;
+
+      // Download from source
+      const content = await sourceS3.downloadFile(path);
+
+      // Upload to destination
+      await destS3.uploadFile(path, content);
+
+      syncedCount++;
+      console.log(`  Synced: ${path} (${syncedCount}/${files.length})`);
+    }
+
+    console.log(`✓ Directory sync complete: ${syncedCount} files`);
+    return syncedCount;
+  } catch (error) {
+    console.error('Directory sync failed:', error);
+    throw error;
+  }
+}
+
+/**
+ * Pattern 9: Incremental Backup
+ *
+ * Back up only files modified since last backup
+ */
+export async function incrementalBackup(
+  source: S3DataSource,
+  lastBackupTime: Date,
+  sourceBucket: string,
+  backupBucket: string,
+  prefix: string
+): Promise<number> {
+  try {
+    const files = await source.listFiles({ prefix });
+
+    // Filter files modified after last backup
+    const filesToBackup = files.filter(
+      f => f.lastModified && new Date(f.lastModified) > lastBackupTime
+    );
+
+    console.log(
+      `Found ${filesToBackup.length} files modified since ${lastBackupTime.toISOString()}`
+    );
+
+    for (const file of filesToBackup) {
+      const path = file.path;
+      const content = await source.downloadFile(path);
+
+      // Add backup timestamp to path
+      const backupPath = `backups/${new Date().toISOString().split('T')[0]}/${path}`;
+
+      await source.uploadFile(backupPath, content);
+      console.log(`  Backed up: ${path}`);
+    }
+
+    console.log(`✓ Incremental backup complete: ${filesToBackup.length} files`);
+    return filesToBackup.length;
+  } catch (error) {
+    console.error('Incremental backup failed:', error);
+    throw error;
+  }
+}
+
+/**
+ * Pattern 10: Health Check
+ *
+ * Verify data source connectivity and permissions
+ */
+export async function healthCheck(
+  source: S3DataSource | SftpDataSource
+): Promise<{ healthy: boolean; message: string }> {
+  try {
+    if (source instanceof SftpDataSource) {
+      try {
+        const ok = await source.validateConnection();
+        return {
+          healthy: ok,
+          message: ok ? 'SFTP connection successful' : 'SFTP validation failed',
+        };
+      } catch (e: any) {
+        return { healthy: false, message: `SFTP validation error: ${e.message}` };
+      }
+    } else {
+      const ok = await (source as S3DataSource).validateConnection();
+      return { healthy: ok, message: ok ? 'S3 connection successful' : 'S3 validation failed' };
+    }
+  } catch (error) {
+    return {
+      healthy: false,
+      message: `Health check failed: ${(error as Error).message}`,
+    };
+  }
+}
+
+/**
+ * Usage Example
+ */
+async function main() {
+  const s3 = new S3DataSource({
+    type: 'S3_CSV',
+    s3Config: {
+      accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
+      region: process.env.AWS_REGION!,
+      bucket: process.env.S3_BUCKET!,
+    },
+  } as any);
+
+  const sftp = new SftpDataSource({
+    type: 'SFTP_CSV',
+    connectionId: 'example',
+    settings: {
+      host: process.env.SFTP_HOST!,
+      port: parseInt(process.env.SFTP_PORT || '22'),
+      username: process.env.SFTP_USERNAME!,
+      password: process.env.SFTP_PASSWORD!,
+      remotePath: '/',
+      filePattern: '*',
+    },
+  } as any);
+
+  // Example 1: Download with retry
+  await downloadWithRetry(() => s3.downloadFile('test.csv'), 3);
+
+  // Example 2: Safe download
+  await safeDownload(s3, 'test.csv');
+
+  // Example 3: Batch download
+  await batchDownload(s3, ['file1.csv', 'file2.csv']);
+
+  // Example 4: Health check
+  const s3Health = await healthCheck(s3);
+  console.log('S3 Health:', s3Health);
+
+  const sftpHealth = await healthCheck(sftp);
+  console.log('SFTP Health:', sftpHealth);
+}
+
+if (require.main === module) {
+  main().catch(console.error);
+}