@tmlmobilidade/env-sync 20260304.1625.33

package/README.md

@@ -0,0 +1,150 @@
+# Environment Sync CLI
+
+CLI tool to sync production and staging environments for MongoDB and Storage (using RClone).
+
+## Installation
+
+```bash
+npm install
+npm run build
+```
+
+## Usage
+
+### Basic Usage
+
+```bash
+# Sync both MongoDB and Storage (interactive mode)
+npm run dev
+
+# Run with arguments (use -- to pass arguments to the script)
+npm run dev -- --help
+npm run dev -- --db-only
+npm run dev -- --storage-only
+
+# Or after building:
+./dist/index.js
+./dist/index.js --help
+```
+
+### Command Line Options
+
+```bash
+# Sync only MongoDB database
+env-sync --db-only
+
+# Sync only storage
+env-sync --storage-only
+
+# Use replica set mode
+env-sync --replica-set
+
+# Skip cleanup of old backups
+env-sync --no-cleanup
+
+# Upload backup artifacts to OCI bucket (for CI/CD, replaces GitHub artifacts)
+env-sync --db-only --upload-artifacts
+
+# Show help
+env-sync --help
+```
+
+## Configuration
+
+Create a `.env` file in the `cli/env-sync-ts/` directory with the following variables:
+
+### MongoDB Configuration
+
+```env
+# Production MongoDB
+PROD_HOST=production-mongo-host:27017
+PROD_USERNAME=admin
+PROD_PASSWORD=password
+PROD_AUTH_DATABASE=admin
+PROD_DB=production_database
+
+# Staging MongoDB
+STAGING_HOST=staging-mongo-host:27017
+STAGING_USERNAME=admin
+STAGING_PASSWORD=password
+STAGING_AUTH_DATABASE=admin
+STAGING_DB=staging_database
+
+# Optional: Collections to exclude from sync (space-separated)
+EXCLUDE_COLLECTIONS=logs sessions temp_data
+
+# Optional: Backup retention days (default: 7)
+BACKUP_RETENTION_DAYS=7
+```
+
+### Storage Configuration (OCI/RClone)
+
+```env
+# RClone Configuration
+STORAGE_REMOTE_NAME=oci_storage
+STORAGE_TYPE=oracleobjectstorage
+STORAGE_SOURCE=production-bucket/path/to/source
+STORAGE_DEST=staging-bucket/path/to/dest
+
+# OCI Authentication
+OCI_USER=ocid1.user.oc1..
+OCI_FINGERPRINT=aa:bb:cc:dd:ee:ff:00:11:22:33:44:55:66:77:88:99
+OCI_KEY_FILE=/path/to/private_key.pem
+OCI_TENANCY=ocid1.tenancy.oc1..
+OCI_REGION=us-ashburn-1
+OCI_COMPARTMENT=ocid1.compartment.oc1..
+OCI_NAMESPACE=your_namespace
+```
+
+### Artifacts Configuration (for CI/CD)
+
+```env
+# OCI bucket for storing backup artifacts (required for --upload-artifacts)
+ARTIFACTS_BUCKET=your-artifacts-bucket
+
+# Optional: prefix/folder within the bucket (default: "env-sync")
+ARTIFACTS_PREFIX=env-sync
+```
+
+## MongoDB Backup Strategy
+
+- Full database dump with all collections (excluding configured collections)
+- Dumps all collections from production database
+- Restores to staging database with `--drop` flag
+
+### Backup Metadata
+- Backup metadata is stored in `backups/.backup_metadata`
+- Tracks last backup timestamp
+
+## Storage Sync
+
+- Uses RClone to sync files from production OCI storage to staging
+- Configured via environment variables (no rclone config file needed)
+- Progress reporting during sync
+
+## Development
+
+```bash
+# Run in development mode
+npm run dev
+
+# Build
+npm run build
+
+# Lint
+npm run lint
+```
+
+## Requirements
+
+- Node.js (v18+)
+- MongoDB tools (`mongodump`, `mongorestore`)
+- RClone (`rclone`)
+
+## Notes
+
+- The script maintains compatibility with the existing bash scripts' `.env` file format
+- Backups are stored in `backups/` directory
+- Old backups are automatically cleaned up (configurable retention period)
+- Interactive prompts use Clack for a modern CLI experience
+

package/dist/artifacts/upload.service.d.ts

@@ -0,0 +1,17 @@
+import type { StorageConfig } from '../config/config-loader.js';
+export interface UploadArtifactsOptions {
+    /** OCI bucket name to upload to */
+    bucket: string;
+    /** Local directory containing artifacts to upload */
+    localPath: string;
+    /** Optional prefix/folder within the bucket */
+    prefix?: string;
+    /** Storage config with OCI credentials */
+    storageConfig: StorageConfig;
+}
+/**
+ * Upload artifacts (backups) to OCI Object Storage bucket.
+ * This replaces the GitHub artifacts upload for security in public repos.
+ * The backup folder is zipped before uploading to reduce size and upload time.
+ */
+export declare function uploadArtifacts(options: UploadArtifactsOptions): Promise<void>;

package/dist/artifacts/upload.service.js

@@ -0,0 +1,181 @@
+import archiver from 'archiver';
+import { createWriteStream, existsSync, readdirSync, rmSync, statSync, writeFileSync } from 'fs';
+import os from 'os';
+import path from 'path';
+import { checkCommandAvailable, execCommandStream } from '../utils/exec.js';
+import { logger } from '../utils/logger.js';
+/**
+ * Build a temporary OCI CLI config file using the credentials from StorageConfig.
+ * This lets rclone use the "user_principal_auth" provider without needing ~/.oci/config.
+ */
+function buildOciConfig(config, profileName) {
+    return [
+        `[${profileName}]`,
+        `user=${config.user}`,
+        `fingerprint=${config.fingerprint}`,
+        `key_file=${config.keyFile}`,
+        `tenancy=${config.tenancy}`,
+        `region=${config.region}`,
+        '',
+    ].join('\n');
+}
+/**
+ * Build the rclone backend config pointing at the temporary OCI config file.
+ */
+function buildRcloneConfig(config, ociConfigPath, profileName) {
+    return [
+        `[${config.remoteName}]`,
+        `type = ${config.type}`,
+        `namespace = ${config.namespace}`,
+        `compartment = ${config.compartment}`,
+        `region = ${config.region}`,
+        'provider = user_principal_auth',
+        `config_file = ${ociConfigPath}`,
+        `config_profile = ${profileName}`,
+        '',
+    ].join('\n');
+}
+/**
+ * Check if the directory is empty or has no files to upload
+ */
+function isDirectoryEmpty(dirPath) {
+    if (!existsSync(dirPath)) {
+        return true;
+    }
+    const stats = statSync(dirPath);
+    if (!stats.isDirectory()) {
+        return false;
+    }
+    const entries = readdirSync(dirPath);
+    // Filter out hidden files like .backup_metadata
+    const files = entries.filter((entry) => {
+        const entryPath = path.join(dirPath, entry);
+        const entryStats = statSync(entryPath);
+        return entryStats.isFile();
+    });
+    return files.length === 0;
+}
+/**
+ * Create a zip archive of the backup directory using archiver
+ */
+async function createZipArchive(sourceDir, outputPath) {
+    logger.info(`Creating zip archive: ${outputPath}`);
+    return new Promise((resolve, reject) => {
+        const output = createWriteStream(outputPath);
+        const archive = archiver('zip', {
+            zlib: { level: 9 }, // Maximum compression
+        });
+        // Listen for all archive data to be written
+        output.on('close', () => {
+            logger.verbose(`Zip archive created successfully: ${outputPath} (${archive.pointer()} bytes)`);
+            resolve();
+        });
+        // Catch warnings (e.g., stat failures and other non-blocking errors)
+        archive.on('warning', (err) => {
+            if (err.code === 'ENOENT') {
+                logger.verbose(`Archive warning: ${err.message}`);
+            }
+            else {
+                reject(err);
+            }
+        });
+        // Catch errors
+        archive.on('error', (err) => {
+            reject(new Error(`Failed to create zip archive: ${err.message}`));
+        });
+        // Pipe archive data to the file
+        archive.pipe(output);
+        // Append the entire directory to the archive
+        const folderName = path.basename(sourceDir);
+        archive.directory(sourceDir, folderName);
+        // Finalize the archive (i.e., we are done appending files)
+        archive.finalize();
+    });
+}
+/**
+ * Upload artifacts (backups) to OCI Object Storage bucket.
+ * This replaces the GitHub artifacts upload for security in public repos.
+ * The backup folder is zipped before uploading to reduce size and upload time.
+ */
+export async function uploadArtifacts(options) {
+    const { bucket, localPath, prefix, storageConfig } = options;
+    logger.info('Starting artifact upload to OCI Object Storage...');
+    // Check if local path exists
+    if (!existsSync(localPath)) {
+        logger.warn(`Artifact path does not exist: ${localPath}`);
+        logger.info('No artifacts to upload.');
+        return;
+    }
+    // Check if directory is empty
+    if (isDirectoryEmpty(localPath)) {
+        logger.warn(`Artifact directory is empty: ${localPath}`);
+        logger.info('No artifacts to upload.');
+        return;
+    }
+    // Check if rclone is available
+    if (!(await checkCommandAvailable('rclone'))) {
+        throw new Error('rclone not found. Please install rclone.');
+    }
+    // Create zip archive
+    const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
+    const zipFileName = `backups-${timestamp}.zip`;
+    const zipFilePath = path.join(os.tmpdir(), zipFileName);
+    let zipCreated = false;
+    try {
+        await createZipArchive(localPath, zipFilePath);
+        zipCreated = true;
+        if (existsSync(zipFilePath)) {
+            const zipSizeMB = Math.round(statSync(zipFilePath).size / 1024 / 1024 * 100) / 100;
+            logger.info(`Zip archive created: ${zipFileName} (${zipSizeMB} MB)`);
+        }
+        else {
+            logger.info(`Zip archive created: ${zipFileName}`);
+        }
+    }
+    catch (error) {
+        logger.error(`Failed to create zip archive: ${error instanceof Error ? error.message : String(error)}`);
+        throw error;
+    }
+    // Prepare temporary OCI config
+    logger.info('Building temporary OCI config...');
+    const profileName = 'Default';
+    const ociConfigFile = path.join(os.tmpdir(), `oci-artifacts-${Date.now()}.conf`);
+    const ociConfigContent = buildOciConfig(storageConfig, profileName);
+    writeFileSync(ociConfigFile, ociConfigContent);
+    // Build rclone config pointing at the temporary OCI config
+    logger.info('Building rclone config...');
+    const rcloneConfig = buildRcloneConfig(storageConfig, ociConfigFile, profileName);
+    const rcloneConfigFile = path.join(os.tmpdir(), `rclone-artifacts-${Date.now()}.conf`);
+    writeFileSync(rcloneConfigFile, rcloneConfig);
+    // Build the destination path
+    const destFolder = prefix ?? '';
+    const destPath = `${storageConfig.remoteName}:${bucket}/${destFolder}`;
+    logger.info(`Uploading zip archive to: ${destPath}`);
+    // Use rclone copy to upload the zip file
+    const baseCmd = `rclone copy "${zipFilePath}" "${destPath}/" --config ${rcloneConfigFile}`;
+    const verboseFlags = logger.isVerbose() ? ' --progress --verbose' : '';
+    const copyCmd = `${baseCmd}${verboseFlags}`;
+    try {
+        await execCommandStream(copyCmd);
+        logger.success(`Artifacts uploaded successfully to: ${destPath}/${zipFileName}`);
+        // Clean up temporary files
+        rmSync(ociConfigFile);
+        rmSync(rcloneConfigFile);
+        if (zipCreated && existsSync(zipFilePath)) {
+            rmSync(zipFilePath);
+            logger.verbose('Temporary zip file cleaned up');
+        }
+    }
+    catch (error) {
+        // Clean up on error too
+        if (existsSync(ociConfigFile))
+            rmSync(ociConfigFile);
+        if (existsSync(rcloneConfigFile))
+            rmSync(rcloneConfigFile);
+        if (zipCreated && existsSync(zipFilePath)) {
+            rmSync(zipFilePath);
+        }
+        logger.error(`Artifact upload failed: ${error instanceof Error ? error.message : String(error)}`);
+        throw error;
+    }
+}

package/dist/cli/commands.d.ts

@@ -0,0 +1,12 @@
+export interface CliOptions {
+    backupOnly?: boolean;
+    dbOnly?: boolean;
+    help?: boolean;
+    noCleanup?: boolean;
+    replicaSet?: boolean;
+    storageOnly?: boolean;
+    uploadArtifacts?: boolean;
+    verbose?: boolean;
+}
+export declare function parseArgs(args: string[]): CliOptions;
+export declare function showHelp(): void;

package/dist/cli/commands.js

@@ -0,0 +1,111 @@
+export function parseArgs(args) {
+    const options = {};
+    for (const arg of args) {
+        switch (arg) {
+            case '--backup-only':
+                options.backupOnly = true;
+                break;
+            case '--db-only':
+                options.dbOnly = true;
+                break;
+            case '--help':
+            case '-h':
+                options.help = true;
+                break;
+            case '--no-cleanup':
+                options.noCleanup = true;
+                break;
+            case '--no-replica-set':
+                options.replicaSet = false;
+                break;
+            case '--replica-set':
+                options.replicaSet = true;
+                break;
+            case '--storage-only':
+                options.storageOnly = true;
+                break;
+            case '--upload-artifacts':
+                options.uploadArtifacts = true;
+                break;
+            case '--verbose':
+            case '-v':
+                options.verbose = true;
+                break;
+            default:
+                if (arg.startsWith('-')) {
+                    throw new Error(`Unknown option: ${arg}\nRun 'env-sync --help' for more information.`);
+                }
+                break;
+        }
+    }
+    return options;
+}
+export function showHelp() {
+    console.log(`
+Environment Sync CLI - Production to Staging
+
+SYNOPSIS
+    env-sync [OPTIONS]
+
+DESCRIPTION
+    Syncs production environment data to staging, handling both OCI file storage
+    and MongoDB database. Performs full MongoDB database dumps and restores.
+
+MongoDB Backup Strategy:
+    - Full database dump with all collections (excluding configured collections)
+    - Restores to staging database with --drop flag
+
+OPTIONS
+    --backup-only         Backup MongoDB only (dump without restoring to staging)
+    --db-only             Sync only MongoDB database, skip file sync
+    --storage-only        Sync only OCI files, skip database sync
+    --upload-artifacts    Upload backup artifacts to OCI bucket (instead of GitHub artifacts)
+    --replica-set         Use replica set sync mode (overrides .env setting)
+    --no-replica-set      Disable replica set sync mode (overrides .env setting)
+    --no-cleanup          Skip cleanup of old backups (older than 7 days)
+    -v, --verbose         Enable verbose output (show detailed command execution)
+    -h, --help            Show this help message and exit
+
+EXAMPLES
+    # Sync both files and database (default behavior)
+    env-sync
+
+    # Sync only database
+    env-sync --db-only
+
+    # Sync files only
+    env-sync --storage-only
+
+    # Sync with replica set mode
+    env-sync --replica-set
+
+    # Upload backup artifacts to OCI bucket (for CI/CD)
+    env-sync --upload-artifacts
+
+    # Backup database only (no restore to staging, no storage sync)
+    env-sync --backup-only
+
+    # Backup and upload artifacts to OCI bucket
+    env-sync --backup-only --upload-artifacts
+
+    # Or combine with sync operations
+    env-sync --db-only --upload-artifacts
+
+CONFIGURATION
+    The script reads configuration from .env file in the script directory.
+    Required variables:
+    - MongoDB: PROD_HOST, PROD_USERNAME, PROD_PASSWORD, PROD_AUTH_DATABASE, PROD_DB
+               STAGING_HOST, STAGING_USERNAME, STAGING_PASSWORD, STAGING_AUTH_DATABASE, STAGING_DB
+    - OCI/Rclone: STORAGE_REMOTE_NAME, STORAGE_TYPE, OCI_COMPARTMENT, OCI_NAMESPACE, OCI_REGION
+                  OCI_USER, OCI_FINGERPRINT, OCI_KEY_FILE, OCI_TENANCY
+    - Paths: STORAGE_SOURCE, STORAGE_DEST
+    - Artifacts: ARTIFACTS_BUCKET (required for --upload-artifacts)
+                 ARTIFACTS_PREFIX (optional, default: "env-sync")
+    - Optional: EXCLUDE_COLLECTIONS (space-separated list of collections to exclude)
+                BACKUP_RETENTION_DAYS (default: 7)
+
+BACKUP METADATA
+    Backup metadata is stored in backups/.backup_metadata and tracks:
+    - Last backup timestamp
+`);
+}

package/dist/config/config-loader.d.ts

@@ -0,0 +1,37 @@
+export interface MongoConfig {
+    authDatabase: string;
+    database: string;
+    host: string;
+    password: string;
+    username: string;
+}
+export interface StorageConfig {
+    compartment: string;
+    dest: string;
+    fingerprint: string;
+    keyFile: string;
+    namespace: string;
+    region: string;
+    remoteName: string;
+    source: string;
+    tenancy: string;
+    type: string;
+    user: string;
+}
+export interface ArtifactsConfig {
+    /** OCI bucket name for storing artifacts */
+    bucket: string;
+    /** Optional prefix/folder within the bucket */
+    prefix: string;
+}
+export interface SyncConfig {
+    artifacts: ArtifactsConfig;
+    backupDir: string;
+    backupRetentionDays: number;
+    databaseProduction: MongoConfig;
+    databaseStaging: MongoConfig;
+    excludeCollections: string[];
+    scriptDir: string;
+    storage: StorageConfig;
+}
+export declare function loadConfig(): SyncConfig;

package/dist/config/config-loader.js

@@ -0,0 +1,91 @@
+import { existsSync, readFileSync } from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+function parseEnvValue(value, defaultValue = '') {
+    return value?.trim() || defaultValue;
+}
+function parseEnvNumber(value, defaultValue) {
+    if (!value)
+        return defaultValue;
+    const parsed = parseInt(value.trim(), 10);
+    return isNaN(parsed) ? defaultValue : parsed;
+}
+export function loadConfig() {
+    const __filename = fileURLToPath(import.meta.url);
+    const __dirname = path.dirname(__filename);
+    const scriptDir = path.resolve(__dirname, '../..');
+    const envFile = path.join(scriptDir, '.env');
+    if (!existsSync(envFile)) {
+        throw new Error(`Environment file not found at ${envFile}\nPlease copy env.example to .env and configure it`);
+    }
+    // Load .env file manually (dotenv doesn't work well with ESM in this context)
+    const envContent = readFileSync(envFile, 'utf-8');
+    const envVars = {};
+    for (const line of envContent.split('\n')) {
+        const trimmed = line.trim();
+        if (!trimmed || trimmed.startsWith('#'))
+            continue;
+        const match = trimmed.match(/^([^=]+)=(.*)$/);
+        if (match) {
+            const key = match[1].trim();
+            const value = match[2].trim().replace(/^["']|["']$/g, '');
+            envVars[key] = value;
+        }
+    }
+    // Validate required MongoDB variables
+    const requiredMongoVars = ['PROD_HOST', 'STAGING_HOST'];
+    for (const varName of requiredMongoVars) {
+        if (!envVars[varName]) {
+            throw new Error(`MongoDB configuration missing: ${varName} is required in ${envFile}`);
+        }
+    }
+    // Validate required Storage variables
+    const requiredStorageVars = ['STORAGE_SOURCE', 'STORAGE_DEST', 'STORAGE_REMOTE_NAME', 'STORAGE_TYPE', 'OCI_COMPARTMENT', 'OCI_FINGERPRINT', 'OCI_KEY_FILE', 'OCI_NAMESPACE', 'OCI_REGION', 'OCI_TENANCY', 'OCI_USER'];
+    for (const varName of requiredStorageVars) {
+        if (!envVars[varName]) {
+            throw new Error(`Storage configuration missing: ${varName} is required in ${envFile}`);
+        }
+    }
+    const backupDir = path.join(scriptDir, 'backups');
+    return {
+        artifacts: {
+            bucket: parseEnvValue(envVars.ARTIFACTS_BUCKET, ''),
+            prefix: parseEnvValue(envVars.ARTIFACTS_PREFIX, ''),
+        },
+        backupDir,
+        backupRetentionDays: parseEnvNumber(envVars.BACKUP_RETENTION_DAYS, 7),
+        databaseProduction: {
+            authDatabase: parseEnvValue(envVars.PROD_AUTH_DATABASE, 'admin'),
+            database: parseEnvValue(envVars.PROD_DB),
+            host: parseEnvValue(envVars.PROD_HOST),
+            password: parseEnvValue(envVars.PROD_PASSWORD),
+            username: parseEnvValue(envVars.PROD_USERNAME),
+        },
+        databaseStaging: {
+            authDatabase: parseEnvValue(envVars.STAGING_AUTH_DATABASE, 'admin'),
+            database: parseEnvValue(envVars.STAGING_DB),
+            host: parseEnvValue(envVars.STAGING_HOST),
+            password: parseEnvValue(envVars.STAGING_PASSWORD),
+            username: parseEnvValue(envVars.STAGING_USERNAME),
+        },
+        excludeCollections: parseEnvValue(envVars.EXCLUDE_COLLECTIONS, '')
+            .split(/\s+/)
+            .filter(c => c.length > 0),
+        scriptDir,
+        storage: {
+            //
+            dest: parseEnvValue(envVars.STORAGE_DEST),
+            remoteName: parseEnvValue(envVars.STORAGE_REMOTE_NAME),
+            source: parseEnvValue(envVars.STORAGE_SOURCE),
+            type: parseEnvValue(envVars.STORAGE_TYPE),
+            //
+            compartment: parseEnvValue(envVars.OCI_COMPARTMENT),
+            fingerprint: parseEnvValue(envVars.OCI_FINGERPRINT),
+            keyFile: parseEnvValue(envVars.OCI_KEY_FILE),
+            namespace: parseEnvValue(envVars.OCI_NAMESPACE),
+            region: parseEnvValue(envVars.OCI_REGION),
+            tenancy: parseEnvValue(envVars.OCI_TENANCY),
+            user: parseEnvValue(envVars.OCI_USER),
+        },
+    };
+}

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export declare const renderTitle: () => void;