@fluentcommerce/fc-connect-sdk 0.1.53 → 0.1.55

package/docs/02-CORE-GUIDES/data-sources/examples/sftp-operations.ts

@@ -1,371 +1,371 @@
-/**
- * SFTP Data Source Operations
- *
- * Complete examples for SftpDataSource operations including connect, download,
- * upload, list, and directory management.
- *
- * @package @fluentcommerce/fc-connect-sdk
- */
-
-import { SftpDataSource } from '@fluentcommerce/fc-connect-sdk';
-import { writeFileSync, readFileSync } from 'fs';
-
-/**
- * Initialize SFTP Data Source with password auth
- */
-function createSftpDataSourceWithPassword() {
-  return new SftpDataSource({
-    host: process.env.SFTP_HOST!,
-    port: parseInt(process.env.SFTP_PORT || '22'),
-    username: process.env.SFTP_USERNAME!,
-    password: process.env.SFTP_PASSWORD!,
-  });
-}
-
-/**
- * Initialize SFTP Data Source with private key auth
- */
-function createSftpDataSourceWithKey() {
-  const privateKey = readFileSync(process.env.SFTP_KEY_PATH!, 'utf-8');
-
-  return new SftpDataSource({
-    host: process.env.SFTP_HOST!,
-    port: parseInt(process.env.SFTP_PORT || '22'),
-    username: process.env.SFTP_USERNAME!,
-    privateKey,
-  });
-}
-
-/**
- * Example 1: Connect and list files in directory
- */
-export async function listRemoteFiles() {
-  const sftp = createSftpDataSourceWithPassword();
-
-  try {
-    const files = await sftp.listFiles({ remotePath: '/inventory/incoming', filePattern: '*' });
-
-    console.log(`Found ${files.length} files:`);
-    files.forEach(file => {
-      const type = file.type === 'd' ? 'DIR' : 'FILE';
-      console.log(`  ${type} ${file.name} (${file.size} bytes, ${file.modifyTime})`);
-    });
-
-    return files;
-  } catch (error) {
-    console.error('Failed to list files:', error);
-    throw error;
-  }
-}
-
-/**
- * Example 2: Download CSV file from SFTP
- */
-export async function downloadCSVFile(remotePath: string) {
-  const sftp = createSftpDataSourceWithPassword();
-
-  try {
-    const content = await sftp.downloadFile(remotePath, { encoding: 'utf-8' });
-
-    console.log(`✓ Downloaded ${remotePath} (${content.length} bytes)`);
-
-    return content;
-  } catch (error) {
-    console.error('Failed to download file:', error);
-    throw error;
-  }
-}
-
-/**
- * Example 3: Upload file to SFTP
- */
-export async function uploadFile(localContent: string, remotePath: string) {
-  const sftp = createSftpDataSourceWithPassword();
-
-  try {
-    await sftp.uploadContent(localContent, remotePath);
-
-    console.log(`✓ Uploaded file to ${remotePath}`);
-  } catch (error) {
-    console.error('Failed to upload file:', error);
-    throw error;
-  }
-}
-
-/**
- * Example 4: Download file to local filesystem
- */
-export async function downloadToLocal(remotePath: string, localPath: string) {
-  const sftp = createSftpDataSourceWithPassword();
-
-  try {
-    const content = await sftp.downloadFile(remotePath);
-
-    writeFileSync(localPath, content);
-
-    console.log(`✓ Downloaded ${remotePath} → ${localPath}`);
-  } catch (error) {
-    console.error('Failed to download to local:', error);
-    throw error;
-  }
-}
-
-/**
- * Example 5: Upload file from local filesystem
- */
-export async function uploadFromLocal(localPath: string, remotePath: string) {
-  const sftp = createSftpDataSourceWithPassword();
-
-  try {
-    const content = readFileSync(localPath, 'utf-8');
-    await sftp.uploadContent(content, remotePath);
-
-    console.log(`✓ Uploaded ${localPath} → ${remotePath}`);
-  } catch (error) {
-    console.error('Failed to upload from local:', error);
-    throw error;
-  }
-}
-
-/**
- * Example 6: Create directory on SFTP server
- */
-export async function createDirectory(dirPath: string) {
-  const sftp = createSftpDataSourceWithPassword();
-
-  try {
-    await sftp.createDirectory(dirPath, true);
-
-    console.log(`✓ Created directory: ${dirPath}`);
-  } catch (error) {
-    console.error('Failed to create directory:', error);
-    throw error;
-  }
-}
-
-/**
- * Example 7: Delete file from SFTP server
- */
-export async function deleteFile(remotePath: string) {
-  const sftp = createSftpDataSourceWithPassword();
-
-  try {
-    await sftp.deleteFile(remotePath);
-
-    console.log(`✓ Deleted: ${remotePath}`);
-  } catch (error) {
-    console.error('Failed to delete file:', error);
-    throw error;
-  }
-}
-
-/**
- * Example 8: Check if file exists on SFTP
- */
-export async function fileExists(remotePath: string): Promise<boolean> {
-  const sftp = createSftpDataSourceWithPassword();
-
-  try {
-    const exists = await sftp.fileExists(remotePath);
-
-    console.log(`File ${remotePath} ${exists ? 'exists' : 'does not exist'}`);
-
-    return exists;
-  } catch (error) {
-    console.error('Failed to check file existence:', error);
-    throw error;
-  }
-}
-
-/**
- * Example 9: Process all CSV files in directory
- *
- * NOTE: This example shows manual filtering for demonstration.
- * In production, use filePattern in listFiles() to avoid redundant filtering.
- */
-export async function processIncomingFiles() {
-  const sftp = createSftpDataSourceWithPassword();
-
-  try {
-    // OPTION 1: Let SDK filter (RECOMMENDED)
-    // const files = await sftp.listFiles({
-    //   remotePath: '/inventory/incoming',
-    //   filePattern: '*.csv'  // SDK handles glob pattern matching
-    // });
-    // const csvFiles = files; // Already filtered by SDK!
-
-    // OPTION 2: Manual filtering (shown here for learning)
-    const files = await sftp.listFiles({ remotePath: '/inventory/incoming', filePattern: '*' });
-
-    // Filter for CSV files only
-    // IMPORTANT: Use .endsWith('.csv') NOT .endsWith('*.csv')
-    // The asterisk (*) is a glob pattern, not part of the extension
-    const csvFiles = files.filter(f => f.type === '-' && f.name.endsWith('.csv'));
-
-    console.log(`Found ${csvFiles.length} CSV files to process`);
-
-    for (const file of csvFiles) {
-      // file.path = full path (e.g., "/inventory/incoming/file.csv")
-      // file.name = filename only (e.g., "file.csv")
-      const remotePath = file.path;
-      const content = await sftp.downloadFile(remotePath, { encoding: 'utf-8' });
-
-      console.log(`Processing ${file.name} (${content.length} bytes)`);
-
-      // Process file content here
-      // ... your processing logic ...
-
-      // Move processed file to archive
-      // Use file.name (just filename) to construct destination path
-      const archivePath = `/inventory/archive/${file.name}`;
-      await sftp.uploadContent(content, archivePath);
-      await sftp.deleteFile(remotePath);
-
-      console.log(`  ✓ Processed and archived ${file.name}`);
-    }
-
-    return csvFiles.length;
-  } catch (error) {
-    console.error('Failed to process incoming files:', error);
-    throw error;
-  }
-}
-
-/**
- * Example 10: Sync files from SFTP to S3
- */
-export async function syncSftpToS3() {
-  const sftp = createSftpDataSourceWithPassword();
-  // Import S3DataSource for sync operation
-  const { S3DataSource } = require('@fluentcommerce/fc-connect-sdk');
-
-  const s3 = new S3DataSource({
-    accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
-    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
-    region: process.env.AWS_REGION!,
-  });
-
-  try {
-    const files = await sftp.listFiles({ remotePath: '/inventory/export', filePattern: '*' });
-
-    console.log(`Syncing ${files.length} files from SFTP to S3`);
-
-    for (const file of files) {
-      if (file.type === 'd') continue; // Skip directories
-
-      const remotePath = file.path;
-      const content = await sftp.downloadFile(remotePath);
-
-      // Upload to S3
-      await s3.uploadFile(`sftp-sync/${file.name}`, content, {
-        bucket: process.env.S3_BUCKET!,
-      });
-
-      console.log(`  ✓ Synced ${file.name} to S3`);
-    }
-
-    console.log('✓ Sync complete');
-  } catch (error) {
-    console.error('Failed to sync SFTP to S3:', error);
-    throw error;
-  }
-}
-
-/**
- * Example 11: Rename/Move file on SFTP
- */
-export async function moveFile(oldPath: string, newPath: string) {
-  const sftp = createSftpDataSourceWithPassword();
-
-  try {
-    await sftp.moveFile(oldPath, newPath, false);
-
-    console.log(`✓ Moved ${oldPath} → ${newPath}`);
-  } catch (error) {
-    console.error('Failed to move file:', error);
-    throw error;
-  }
-}
-
-/**
- * Example 12: Download only new files (based on timestamp)
- */
-export async function downloadNewFiles(lastSync: Date) {
-  const sftp = createSftpDataSourceWithPassword();
-
-  try {
-    const files = await sftp.listFiles({ remotePath: '/inventory/incoming', filePattern: '*' });
-
-    const newFiles = files.filter(f => {
-      const modTime = new Date(f.modifyTime);
-      return modTime > lastSync;
-    });
-
-    console.log(`Found ${newFiles.length} new files since ${lastSync.toISOString()}`);
-
-    const downloaded = [];
-    for (const file of newFiles) {
-      const remotePath = file.path;
-      const content = await sftp.downloadFile(remotePath);
-
-      downloaded.push({
-        name: file.name,
-        content,
-        modifyTime: file.modifyTime,
-      });
-
-      console.log(`  ✓ Downloaded ${file.name}`);
-    }
-
-    return downloaded;
-  } catch (error) {
-    console.error('Failed to download new files:', error);
-    throw error;
-  }
-}
-
-/**
- * Usage Example
- */
-async function main() {
-  console.log('Example 1: List remote files');
-  await listRemoteFiles();
-
-  console.log('\nExample 2: Download CSV file');
-  await downloadCSVFile('/inventory/incoming/inventory.csv');
-
-  console.log('\nExample 3: Upload file');
-  await uploadFile('test content', '/inventory/outgoing/test.txt');
-
-  console.log('\nExample 4: Download to local filesystem');
-  await downloadToLocal('/inventory/data.csv', '/tmp/data.csv');
-
-  console.log('\nExample 5: Upload from local filesystem');
-  await uploadFromLocal('/tmp/upload.csv', '/inventory/incoming/upload.csv');
-
-  console.log('\nExample 6: Create directory');
-  await createDirectory('/inventory/test-dir');
-
-  console.log('\nExample 7: Delete file');
-  await deleteFile('/inventory/test-dir/temp.txt');
-
-  console.log('\nExample 8: Check file existence');
-  await fileExists('/inventory/incoming/inventory.csv');
-
-  console.log('\nExample 9: Process all incoming CSV files');
-  await processIncomingFiles();
-
-  console.log('\nExample 10: Sync SFTP to S3');
-  await syncSftpToS3();
-
-  console.log('\nExample 11: Move/rename file');
-  await moveFile('/inventory/old.csv', '/inventory/archive/old.csv');
-
-  console.log('\nExample 12: Download only new files');
-  const lastSync = new Date('2024-01-01T00:00:00Z');
-  await downloadNewFiles(lastSync);
-}
-
-if (require.main === module) {
-  main().catch(console.error);
-}
+/**
+ * SFTP Data Source Operations
+ *
+ * Complete examples for SftpDataSource operations including connect, download,
+ * upload, list, and directory management.
+ *
+ * @package @fluentcommerce/fc-connect-sdk
+ */
+
+import { SftpDataSource } from '@fluentcommerce/fc-connect-sdk';
+import { writeFileSync, readFileSync } from 'fs';
+
+/**
+ * Initialize SFTP Data Source with password auth
+ */
+function createSftpDataSourceWithPassword() {
+  return new SftpDataSource({
+    host: process.env.SFTP_HOST!,
+    port: parseInt(process.env.SFTP_PORT || '22'),
+    username: process.env.SFTP_USERNAME!,
+    password: process.env.SFTP_PASSWORD!,
+  });
+}
+
+/**
+ * Initialize SFTP Data Source with private key auth
+ */
+function createSftpDataSourceWithKey() {
+  const privateKey = readFileSync(process.env.SFTP_KEY_PATH!, 'utf-8');
+
+  return new SftpDataSource({
+    host: process.env.SFTP_HOST!,
+    port: parseInt(process.env.SFTP_PORT || '22'),
+    username: process.env.SFTP_USERNAME!,
+    privateKey,
+  });
+}
+
+/**
+ * Example 1: Connect and list files in directory
+ */
+export async function listRemoteFiles() {
+  const sftp = createSftpDataSourceWithPassword();
+
+  try {
+    const files = await sftp.listFiles({ remotePath: '/inventory/incoming', filePattern: '*' });
+
+    console.log(`Found ${files.length} files:`);
+    files.forEach(file => {
+      const type = file.type === 'd' ? 'DIR' : 'FILE';
+      console.log(`  ${type} ${file.name} (${file.size} bytes, ${file.modifyTime})`);
+    });
+
+    return files;
+  } catch (error) {
+    console.error('Failed to list files:', error);
+    throw error;
+  }
+}
+
+/**
+ * Example 2: Download CSV file from SFTP
+ */
+export async function downloadCSVFile(remotePath: string) {
+  const sftp = createSftpDataSourceWithPassword();
+
+  try {
+    const content = await sftp.downloadFile(remotePath, { encoding: 'utf-8' });
+
+    console.log(`✓ Downloaded ${remotePath} (${content.length} bytes)`);
+
+    return content;
+  } catch (error) {
+    console.error('Failed to download file:', error);
+    throw error;
+  }
+}
+
+/**
+ * Example 3: Upload file to SFTP
+ */
+export async function uploadFile(localContent: string, remotePath: string) {
+  const sftp = createSftpDataSourceWithPassword();
+
+  try {
+    await sftp.uploadContent(localContent, remotePath);
+
+    console.log(`✓ Uploaded file to ${remotePath}`);
+  } catch (error) {
+    console.error('Failed to upload file:', error);
+    throw error;
+  }
+}
+
+/**
+ * Example 4: Download file to local filesystem
+ */
+export async function downloadToLocal(remotePath: string, localPath: string) {
+  const sftp = createSftpDataSourceWithPassword();
+
+  try {
+    const content = await sftp.downloadFile(remotePath);
+
+    writeFileSync(localPath, content);
+
+    console.log(`✓ Downloaded ${remotePath} → ${localPath}`);
+  } catch (error) {
+    console.error('Failed to download to local:', error);
+    throw error;
+  }
+}
+
+/**
+ * Example 5: Upload file from local filesystem
+ */
+export async function uploadFromLocal(localPath: string, remotePath: string) {
+  const sftp = createSftpDataSourceWithPassword();
+
+  try {
+    const content = readFileSync(localPath, 'utf-8');
+    await sftp.uploadContent(content, remotePath);
+
+    console.log(`✓ Uploaded ${localPath} → ${remotePath}`);
+  } catch (error) {
+    console.error('Failed to upload from local:', error);
+    throw error;
+  }
+}
+
+/**
+ * Example 6: Create directory on SFTP server
+ */
+export async function createDirectory(dirPath: string) {
+  const sftp = createSftpDataSourceWithPassword();
+
+  try {
+    await sftp.createDirectory(dirPath, true);
+
+    console.log(`✓ Created directory: ${dirPath}`);
+  } catch (error) {
+    console.error('Failed to create directory:', error);
+    throw error;
+  }
+}
+
+/**
+ * Example 7: Delete file from SFTP server
+ */
+export async function deleteFile(remotePath: string) {
+  const sftp = createSftpDataSourceWithPassword();
+
+  try {
+    await sftp.deleteFile(remotePath);
+
+    console.log(`✓ Deleted: ${remotePath}`);
+  } catch (error) {
+    console.error('Failed to delete file:', error);
+    throw error;
+  }
+}
+
+/**
+ * Example 8: Check if file exists on SFTP
+ */
+export async function fileExists(remotePath: string): Promise<boolean> {
+  const sftp = createSftpDataSourceWithPassword();
+
+  try {
+    const exists = await sftp.fileExists(remotePath);
+
+    console.log(`File ${remotePath} ${exists ? 'exists' : 'does not exist'}`);
+
+    return exists;
+  } catch (error) {
+    console.error('Failed to check file existence:', error);
+    throw error;
+  }
+}
+
+/**
+ * Example 9: Process all CSV files in directory
+ *
+ * NOTE: This example shows manual filtering for demonstration.
+ * In production, use filePattern in listFiles() to avoid redundant filtering.
+ */
+export async function processIncomingFiles() {
+  const sftp = createSftpDataSourceWithPassword();
+
+  try {
+    // OPTION 1: Let SDK filter (RECOMMENDED)
+    // const files = await sftp.listFiles({
+    //   remotePath: '/inventory/incoming',
+    //   filePattern: '*.csv'  // SDK handles glob pattern matching
+    // });
+    // const csvFiles = files; // Already filtered by SDK!
+
+    // OPTION 2: Manual filtering (shown here for learning)
+    const files = await sftp.listFiles({ remotePath: '/inventory/incoming', filePattern: '*' });
+
+    // Filter for CSV files only
+    // IMPORTANT: Use .endsWith('.csv') NOT .endsWith('*.csv')
+    // The asterisk (*) is a glob pattern, not part of the extension
+    const csvFiles = files.filter(f => f.type === '-' && f.name.endsWith('.csv'));
+
+    console.log(`Found ${csvFiles.length} CSV files to process`);
+
+    for (const file of csvFiles) {
+      // file.path = full path (e.g., "/inventory/incoming/file.csv")
+      // file.name = filename only (e.g., "file.csv")
+      const remotePath = file.path;
+      const content = await sftp.downloadFile(remotePath, { encoding: 'utf-8' });
+
+      console.log(`Processing ${file.name} (${content.length} bytes)`);
+
+      // Process file content here
+      // ... your processing logic ...
+
+      // Move processed file to archive
+      // Use file.name (just filename) to construct destination path
+      const archivePath = `/inventory/archive/${file.name}`;
+      await sftp.uploadContent(content, archivePath);
+      await sftp.deleteFile(remotePath);
+
+      console.log(`  ✓ Processed and archived ${file.name}`);
+    }
+
+    return csvFiles.length;
+  } catch (error) {
+    console.error('Failed to process incoming files:', error);
+    throw error;
+  }
+}
+
+/**
+ * Example 10: Sync files from SFTP to S3
+ */
+export async function syncSftpToS3() {
+  const sftp = createSftpDataSourceWithPassword();
+  // Import S3DataSource for sync operation
+  const { S3DataSource } = require('@fluentcommerce/fc-connect-sdk');
+
+  const s3 = new S3DataSource({
+    accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
+    region: process.env.AWS_REGION!,
+  });
+
+  try {
+    const files = await sftp.listFiles({ remotePath: '/inventory/export', filePattern: '*' });
+
+    console.log(`Syncing ${files.length} files from SFTP to S3`);
+
+    for (const file of files) {
+      if (file.type === 'd') continue; // Skip directories
+
+      const remotePath = file.path;
+      const content = await sftp.downloadFile(remotePath);
+
+      // Upload to S3
+      await s3.uploadFile(`sftp-sync/${file.name}`, content, {
+        bucket: process.env.S3_BUCKET!,
+      });
+
+      console.log(`  ✓ Synced ${file.name} to S3`);
+    }
+
+    console.log('✓ Sync complete');
+  } catch (error) {
+    console.error('Failed to sync SFTP to S3:', error);
+    throw error;
+  }
+}
+
+/**
+ * Example 11: Rename/Move file on SFTP
+ */
+export async function moveFile(oldPath: string, newPath: string) {
+  const sftp = createSftpDataSourceWithPassword();
+
+  try {
+    await sftp.moveFile(oldPath, newPath, false);
+
+    console.log(`✓ Moved ${oldPath} → ${newPath}`);
+  } catch (error) {
+    console.error('Failed to move file:', error);
+    throw error;
+  }
+}
+
+/**
+ * Example 12: Download only new files (based on timestamp)
+ */
+export async function downloadNewFiles(lastSync: Date) {
+  const sftp = createSftpDataSourceWithPassword();
+
+  try {
+    const files = await sftp.listFiles({ remotePath: '/inventory/incoming', filePattern: '*' });
+
+    const newFiles = files.filter(f => {
+      const modTime = new Date(f.modifyTime);
+      return modTime > lastSync;
+    });
+
+    console.log(`Found ${newFiles.length} new files since ${lastSync.toISOString()}`);
+
+    const downloaded = [];
+    for (const file of newFiles) {
+      const remotePath = file.path;
+      const content = await sftp.downloadFile(remotePath);
+
+      downloaded.push({
+        name: file.name,
+        content,
+        modifyTime: file.modifyTime,
+      });
+
+      console.log(`  ✓ Downloaded ${file.name}`);
+    }
+
+    return downloaded;
+  } catch (error) {
+    console.error('Failed to download new files:', error);
+    throw error;
+  }
+}
+
+/**
+ * Usage Example
+ */
+async function main() {
+  console.log('Example 1: List remote files');
+  await listRemoteFiles();
+
+  console.log('\nExample 2: Download CSV file');
+  await downloadCSVFile('/inventory/incoming/inventory.csv');
+
+  console.log('\nExample 3: Upload file');
+  await uploadFile('test content', '/inventory/outgoing/test.txt');
+
+  console.log('\nExample 4: Download to local filesystem');
+  await downloadToLocal('/inventory/data.csv', '/tmp/data.csv');
+
+  console.log('\nExample 5: Upload from local filesystem');
+  await uploadFromLocal('/tmp/upload.csv', '/inventory/incoming/upload.csv');
+
+  console.log('\nExample 6: Create directory');
+  await createDirectory('/inventory/test-dir');
+
+  console.log('\nExample 7: Delete file');
+  await deleteFile('/inventory/test-dir/temp.txt');
+
+  console.log('\nExample 8: Check file existence');
+  await fileExists('/inventory/incoming/inventory.csv');
+
+  console.log('\nExample 9: Process all incoming CSV files');
+  await processIncomingFiles();
+
+  console.log('\nExample 10: Sync SFTP to S3');
+  await syncSftpToS3();
+
+  console.log('\nExample 11: Move/rename file');
+  await moveFile('/inventory/old.csv', '/inventory/archive/old.csv');
+
+  console.log('\nExample 12: Download only new files');
+  const lastSync = new Date('2024-01-01T00:00:00Z');
+  await downloadNewFiles(lastSync);
+}
+
+if (require.main === module) {
+  main().catch(console.error);
+}