cloudmason 1.9.33 → 2.0.36

package/commands/helpers/stacks/asg.yaml

@@ -187,7 +187,7 @@ Resources:
           Name:
             Ref: AppEc2Profile
         UserData:
-          Fn::Base64: 
+          Fn::Base64:
             !Sub |
               #!/bin/bash
               echo "Running user data"

package/commands/ssh_build.js

@@ -20,17 +20,6 @@ const {
     waitUntilInstanceTerminated
 } = require('@aws-sdk/client-ec2');
 
-const { 
-    IAMClient, 
-    CreateRoleCommand,
-    PutRolePolicyCommand,
-    CreateInstanceProfileCommand,
-    AddRoleToInstanceProfileCommand,
-    RemoveRoleFromInstanceProfileCommand,
-    DeleteInstanceProfileCommand,
-    DeleteRolePolicyCommand,
-    DeleteRoleCommand
-} = require('@aws-sdk/client-iam');
 
 const { Client } = require('ssh2');
 const fs = require('fs');
@@ -38,49 +27,45 @@ const path = require('path');
 
 // All SSH setup commands - array of [description, command]
 const SETUP_COMMANDS = [
-    ['Updating system packages', 'sudo dnf update -y'],
+    ['Upgrading to latest AL2023 release', 'sudo dnf upgrade --releasever=latest -y'],
+    ['Setting up NodeSource for Node.js 24 LTS', 'curl -fsSL https://rpm.nodesource.com/setup_24.x | sudo bash -'],
     ['Installing nodejs', 'sudo dnf install -y nodejs'],
     ['Node version', 'node --version'],
     ['Installing cloudwatch agent', 'sudo dnf install -y amazon-cloudwatch-agent'],
     ['Installing python', 'sudo dnf -y install python3'],
     ['Installing unzip', 'sudo dnf -y install unzip'],
     ['Installing pm2', 'sudo npm install -g pm2'],
-    ['Creating app directory', 'sudo mkdir -p /app'],
+    ['Creating app directory', 'sudo mkdir -p /home/ec2-user/app'],
 ];
 
 
 class EC2AMIBuilder {
-    constructor(amiName, instanceType = 'm6a.large', s3PackageUrl) {
-        if (!amiName || !s3PackageUrl) {
-            throw new Error('amiName and s3PackageUrl are required parameters');
+    constructor(amiName, instanceType = 'm6a.large', localZipPath) {
+        if (!amiName || !localZipPath) {
+            throw new Error('amiName and localZipPath are required parameters');
         }
-        
+
         this.amiName = amiName;
         this.instanceType = instanceType;
-        this.s3PackageUrl = s3PackageUrl;
-        
+        this.localZipPath = localZipPath;
+
         // AWS clients
-        const region = process.env.AWS_REGION || 'us-east-1';
+        const region = process.env.orgRegion || process.env.AWS_REGION || 'us-east-1';
         this.ec2Client = new EC2Client({ region });
-        this.iamClient = new IAMClient({ region });
-        
+
         // Generate unique names for temporary resources
         this.timestamp = Date.now();
         this.keyPairName = `ec2-builder-keypair-${this.timestamp}`;
         this.securityGroupName = `ec2-builder-sg-${this.timestamp}`;
-        this.iamRoleName = `ec2-builder-role-${this.timestamp}`;
-        this.instanceProfileName = `ec2-builder-profile-${this.timestamp}`;
         this.privateKeyPath = path.join(__dirname, `${this.keyPairName}.pem`);
-        
+
         // Resource tracking for cleanup
         this.createdResources = {
             instanceId: null,
             keyPairName: null,
-            securityGroupId: null,
-            iamRoleName: null,
-            instanceProfileName: null
+            securityGroupId: null
         };
-        
+
         this.sshConnection = null;
         this.publicIp = null;
     }
@@ -225,95 +210,18 @@ class EC2AMIBuilder {
         return securityGroupId;
     }
 
-    async createIAMRole() {
-        console.log('👤 Creating IAM role for S3 access...');
-        
-        // Trust policy for EC2
-        const trustPolicy = {
-            Version: '2012-10-17',
-            Statement: [
-                {
-                    Effect: 'Allow',
-                    Principal: { Service: 'ec2.amazonaws.com' },
-                    Action: 'sts:AssumeRole'
-                }
-            ]
-        };
-        
-        // Create IAM role
-        const createRoleCommand = new CreateRoleCommand({
-            RoleName: this.iamRoleName,
-            AssumeRolePolicyDocument: JSON.stringify(trustPolicy),
-            Description: 'Temporary role for EC2 AMI builder to access S3'
-        });
-        
-        await this.iamClient.send(createRoleCommand);
-        this.createdResources.iamRoleName = this.iamRoleName;
-        
-        // S3 access policy
-        const s3Policy = {
-            Version: '2012-10-17',
-            Statement: [
-                {
-                    Effect: 'Allow',
-                    Action: [
-                        's3:GetObject',
-                        's3:GetObjectVersion'
-                    ],
-                    Resource: '*'
-                }
-            ]
-        };
-        
-        // Attach inline policy
-        const putPolicyCommand = new PutRolePolicyCommand({
-            RoleName: this.iamRoleName,
-            PolicyName: 'S3AccessPolicy',
-            PolicyDocument: JSON.stringify(s3Policy)
-        });
-        
-        await this.iamClient.send(putPolicyCommand);
-        
-        // Create instance profile
-        const createProfileCommand = new CreateInstanceProfileCommand({
-            InstanceProfileName: this.instanceProfileName
-        });
-        
-        await this.iamClient.send(createProfileCommand);
-        this.createdResources.instanceProfileName = this.instanceProfileName;
-        
-        // Add role to instance profile
-        const addRoleCommand = new AddRoleToInstanceProfileCommand({
-            InstanceProfileName: this.instanceProfileName,
-            RoleName: this.iamRoleName
-        });
-        
-        await this.iamClient.send(addRoleCommand);
-        
-        // Wait for IAM propagation
-        console.log('⏳ Waiting for IAM role propagation...');
-        await new Promise(resolve => setTimeout(resolve, 10000));
-        
-        console.log(`✅ IAM role created: ${this.iamRoleName}`);
-        return this.instanceProfileName;
-    }
-
     async launchInstance() {
         console.log('🚀 Launching EC2 instance...');
-        
+
         const amiId = await this.getLatestAmazonLinuxAMI();
         const keyPairName = await this.createKeyPair();
         const securityGroupId = await this.createSecurityGroup();
-        const instanceProfileName = await this.createIAMRole();
-        
+
         const command = new RunInstancesCommand({
             ImageId: amiId,
             InstanceType: this.instanceType,
             KeyName: keyPairName,
             SecurityGroupIds: [securityGroupId],
-            IamInstanceProfile: {
-                Name: instanceProfileName
-            },
             MinCount: 1,
             MaxCount: 1,
             BlockDeviceMappings: [
@@ -339,12 +247,12 @@ class EC2AMIBuilder {
 
         const result = await this.ec2Client.send(command);
         this.createdResources.instanceId = result.Instances[0].InstanceId;
-        
+
         console.log(`✅ Instance launched: ${this.createdResources.instanceId}`);
-        
+
         await this.waitForInstanceRunning();
         await this.getInstancePublicIP();
-        
+
         console.log(`🌐 Instance public IP: ${this.publicIp}`);
     }
 
@@ -456,33 +364,93 @@ class EC2AMIBuilder {
         console.log('✅ System setup completed');
     }
 
-    async downloadAndSetupApp() {
-        console.log('📦 Setting up Node.js application...');
-        
-        // Application setup commands (dynamic based on S3 URL)
+    async uploadAppViaSFTP() {
+        return new Promise((resolve, reject) => {
+            console.log('📤 Uploading application via SFTP...');
+            console.log(`   Local file: ${this.localZipPath}`);
+
+            this.sshConnection.sftp((err, sftp) => {
+                if (err) {
+                    console.error('❌ SFTP session error:', err.message);
+                    return reject(err);
+                }
+
+                const readStream = fs.createReadStream(this.localZipPath);
+                const writeStream = sftp.createWriteStream('/tmp/app.zip');
+
+                const fileSize = fs.statSync(this.localZipPath).size;
+                let uploaded = 0;
+
+                readStream.on('data', (chunk) => {
+                    uploaded += chunk.length;
+                    const percent = Math.round((uploaded / fileSize) * 100);
+                    process.stdout.write(`\r   Progress: ${percent}% (${Math.round(uploaded / 1024)}KB / ${Math.round(fileSize / 1024)}KB)`);
+                });
+
+                writeStream.on('close', () => {
+                    console.log('\n✅ SFTP upload completed');
+                    resolve();
+                });
+
+                writeStream.on('error', (err) => {
+                    console.error('\n❌ SFTP write error:', err.message);
+                    reject(err);
+                });
+
+                readStream.on('error', (err) => {
+                    console.error('\n❌ File read error:', err.message);
+                    reject(err);
+                });
+
+                readStream.pipe(writeStream);
+            });
+        });
+    }
+
+    async uploadAndSetupApp() {
+        console.log('📦 Setting up application...');
+
+        // Upload via SFTP
+        await this.uploadAppViaSFTP();
+
+        // Application setup commands
         const appCommands = [
-            ['Downloading Node.js app package from S3', `aws s3 cp "${this.s3PackageUrl}" ./app-package.zip`],
-            ['Extracting application package', 'sudo unzip -o app-package.zip -d /app >/dev/null'],
-            ['Cleaning up package archive', 'sudo rm -f app-package.zip'],
-            ['Directory files', 'ls -A /app'],
-            ['Showing application structure', 'find /app -maxdepth 2 -name "node_modules" -prune -o -print']
+            ['Extracting application package', 'sudo unzip -o /tmp/app.zip -d /home/ec2-user/app'],
+            ['Setting ownership to ec2-user', 'sudo chown -R ec2-user:ec2-user /home/ec2-user/app'],
+            ['Cleaning up package archive', 'rm -f /tmp/app.zip'],
+            ['Directory files', 'ls -la /home/ec2-user/app'],
+            ['Showing application structure', 'find /home/ec2-user/app -maxdepth 2 -name "node_modules" -prune -o -print']
         ];
-        
+
         // Execute all app setup commands
         for (const [description, command] of appCommands) {
             await this.executeCommand(command, description);
         }
-        
+
         console.log('✅ Application setup completed');
     }
 
     async createAMI() {
         console.log('📸 Creating AMI from instance...');
-        
-        // Cleanup commands before AMI creation
+
+        // Cleanup commands before AMI creation - remove all sensitive data
         const cleanupCommands = [
-            ['Cleaning up instance before AMI creation', 'sudo dnf clean all && sudo rm -rf /tmp/* /var/tmp/* /var/log/messages* /var/log/secure* ~/.bash_history'],
-            ['Checking disk usage', 'df -h && du -sh .']
+            // Remove SSH authorized keys (contains the temporary build key)
+            ['Removing SSH authorized keys', 'rm -f ~/.ssh/authorized_keys && sudo rm -f /root/.ssh/authorized_keys'],
+            // Remove SSH host keys (new instances will regenerate their own)
+            ['Removing SSH host keys', 'sudo rm -f /etc/ssh/ssh_host_*'],
+            // Clean cloud-init so it runs fresh on new instances
+            ['Cleaning cloud-init data', 'sudo rm -rf /var/lib/cloud/*'],
+            // Reset machine-id for unique instance identification
+            ['Resetting machine-id', 'sudo truncate -s 0 /etc/machine-id'],
+            // Clean bash history for all users
+            ['Cleaning bash history', 'rm -f ~/.bash_history && sudo rm -f /root/.bash_history'],
+            // Clean logs and temp files
+            ['Cleaning logs and temp files', 'sudo rm -rf /tmp/* /var/tmp/* /var/log/messages* /var/log/secure* /var/log/cloud-init*.log'],
+            // Clean DNF cache
+            ['Cleaning DNF cache', 'sudo dnf clean all'],
+            // Verify cleanup and check disk usage
+            ['Checking disk usage', 'df -h && du -sh /home/ec2-user/app']
         ];
         
         // Execute cleanup commands
@@ -526,7 +494,7 @@ class EC2AMIBuilder {
         console.log('⏳ Waiting for AMI to be available (this may take several minutes)...');
         
         await waitUntilImageAvailable(
-            { client: this.ec2Client, maxWaitTime: 1800 },
+            { client: this.ec2Client, maxWaitTime: 3800 },
             { ImageIds: [amiId] }
         );
         
@@ -579,45 +547,10 @@ class EC2AMIBuilder {
                 const deleteKeyPairCommand = new DeleteKeyPairCommand({
                     KeyName: this.createdResources.keyPairName
                 });
-                
+
                 await this.ec2Client.send(deleteKeyPairCommand);
             }
-            
-            // Clean up IAM resources
-            if (this.createdResources.instanceProfileName && this.createdResources.iamRoleName) {
-                console.log('🗑️  Cleaning up IAM resources...');
-                
-                // Remove role from instance profile
-                const removeRoleCommand = new RemoveRoleFromInstanceProfileCommand({
-                    InstanceProfileName: this.createdResources.instanceProfileName,
-                    RoleName: this.createdResources.iamRoleName
-                });
-                
-                await this.iamClient.send(removeRoleCommand);
-                
-                // Delete instance profile
-                const deleteProfileCommand = new DeleteInstanceProfileCommand({
-                    InstanceProfileName: this.createdResources.instanceProfileName
-                });
-                
-                await this.iamClient.send(deleteProfileCommand);
-                
-                // Delete role policy
-                const deletePolicyCommand = new DeleteRolePolicyCommand({
-                    RoleName: this.createdResources.iamRoleName,
-                    PolicyName: 'S3AccessPolicy'
-                });
-                
-                await this.iamClient.send(deletePolicyCommand);
-                
-                // Delete role
-                const deleteRoleCommand = new DeleteRoleCommand({
-                    RoleName: this.createdResources.iamRoleName
-                });
-                
-                await this.iamClient.send(deleteRoleCommand);
-            }
-            
+
             console.log('✅ Cleanup completed');
             
         } catch (error) {
@@ -632,18 +565,18 @@ class EC2AMIBuilder {
             await this.launchInstance();
             await this.connectSSH();
             await this.setupSystem();
-            await this.downloadAndSetupApp();
-            console.log('Build complete after', Math.ceil((Date.now() - start)/1000/60));
+            await this.uploadAndSetupApp();
+            console.log('Build complete after', Math.ceil((Date.now() - start)/1000/60), 'minutes');
             const amiId = await this.createAMI();
-            console.log('AMI Created after', Math.ceil((Date.now() - start)/1000/60));
+            console.log('AMI Created after', Math.ceil((Date.now() - start)/1000/60), 'minutes');
             console.log(`📋 Summary:`);
             console.log(`   - AMI ID: ${amiId}`);
             console.log(`   - AMI Name: ${this.amiName}`);
             console.log(`   - Instance Type Used: ${this.instanceType}`);
-            console.log(`   - S3 Package: ${this.s3PackageUrl}`);
+            console.log(`   - Local Package: ${this.localZipPath}`);
 
             return amiId;
-            
+
         } catch (error) {
             console.error('❌ AMI Build failed:', error.message);
             throw error;
@@ -653,8 +586,8 @@ class EC2AMIBuilder {
     }
 }
 
-async function sshAMI(amiName,s3PackageUrl,instanceType){
-    const builder = new EC2AMIBuilder(amiName, instanceType, s3PackageUrl);
+async function sshAMI(amiName, localZipPath, instanceType){
+    const builder = new EC2AMIBuilder(amiName, instanceType, localZipPath);
     const result = await builder.build();
     console.log('AMI ID:', result);
     return result;
@@ -664,8 +597,6 @@ async function sshAMI(amiName,s3PackageUrl,instanceType){
 // Convenience function for direct usage
 module.exports.buildAMI = sshAMI;
 
-sshAMI('test-ami-2', 's3://coreinfra-infrabucket-qtfrahre6vbl/apps/theorim/3.6/app.zip')
-
 // // CLI usage if called directly
 // if (require.main === module) {
 //     const [,, amiName, instanceType, s3PackageUrl] = process.argv;

package/commands/update_app.js

@@ -1,15 +1,10 @@
 const path = require('path');
 const fs = require('fs');
-const EC2 = require('./helpers/ec2');
-const { EC2Client, RunInstancesCommand,CreateImageCommand,TerminateInstancesCommand,DescribeInstanceStatusCommand,DeregisterImageCommand,DescribeImagesCommand,CopyImageCommand } = require("@aws-sdk/client-ec2");
-
 const AdmZip = require("adm-zip");
 
 const Params = require('./helpers/params')
 const S3 = require('./helpers/s3');
-
-
-const INSTANCE_TYPE="t2.micro"
+const { buildAMI } = require('./ssh_build');
 
 exports.main = async function(args){
     console.log(`Updating ${args.app} v${args.v}`);
@@ -52,42 +47,19 @@ exports.main = async function(args){
     }
 
     // --- III BUILD IMAGE ---
-    // Launch ec2
-    const orgParams = await Params.getOrgConfig();
-
-    // const awsLinuxAMI = await findLinuxAMI(process.env.orgRegion);
-    const awsLinuxAMI = EC2.awsLinuxAMI(process.env.orgRegion);
-    const instance_id = await launchInstance({
-        app: app.name,
-        linuxAMI: awsLinuxAMI,
-        version: args.v,
-        sec_group: orgParams.buildSecGroup,
-        iam: orgParams.buildInstanceProfile,
-        node: app.nodeV,
-        py: app.pyV
-    });
-    console.log('Instance Launched:',instance_id);
-    console.log('Waiting 60s to initiate checks');
-    await sleep(60*1000);
-    console.log('Checking Instance Status');
-    await waitUntilInstanceReady(instance_id,process.env.orgRegion);
-    console.log('Waiting 5m for app to be ready');
-    await sleep(300*1000);
-    
-    // Create AMI
     const buildNumber = (app.versions[args.v]?.currentBuild || 0) + 1;
     const appVID = `${app.name.toLowerCase()}-v${args.v}.${buildNumber}`;
 
-    var success = false;
+    console.log(`Building AMI: ${appVID}`);
+    console.log(`Using local zip: ${zipFilePath}`);
+
     let ami_id;
     try {
-        ami_id = await createAMI(instance_id, appVID,process.env.orgRegion)
-        success = true;
-    } catch(e){
-        console.log("Error Creating AMI:" + e)
+        ami_id = await buildAMI(appVID, zipFilePath);
+    } catch(e) {
+        console.log("Error Creating AMI:" + e);
+        throw new Error("Error - Build Not Complete");
     }
-    await terminateInstance(instance_id,process.env.orgRegion)
-    if (success === false){ throw new Error("Error - Build Not Complete") }
 
     // --- IV UPDATE PARAMS ---
     const versionInfo = {
@@ -132,121 +104,3 @@ async function prepZip(appPath){
     process.on('exit', function(){ fs.unlinkSync(zipPath) });
     return zipPath;
 }
-
-async function launchInstance(launchParams){
-    console.log('Launching Instance in ' + process.env.orgRegion);
-    const nodeRepo = launchParams.node === '' ? 'echo default_version' : `https://rpm.nodesource.com/setup_${launchParams.node}.x | sudo bash -`;
-    const user_data = [
-        `#!/bin/bash -xe`,
-        nodeRepo,
-        `yum -y install nodejs`,
-        `yum install -y amazon-cloudwatch-agent`,
-        `yum -y install python3`,
-        `yum -y install unzip`,
-        `npm install -g pm2`,
-        `cd /home/ec2-user`,
-        `aws s3 cp s3://${process.env.orgBucket}/apps/${launchParams.app.toLowerCase()}/${launchParams.version}/app.zip .`,
-        `sleep 10`,
-        `unzip app.zip -d app`,
-        `touch app/ami_ok.txt`,
-        `rm -r app.zip`
-    ].join('\n')
-
-    const ud_b64 = Buffer.from(user_data).toString('base64');
-
-    const client = new EC2Client({region: process.env.orgRegion });
-
-    const createInstanceParams = {
-        ImageId: launchParams.linuxAMI,
-        InstanceType: INSTANCE_TYPE,
-        SecurityGroupIds: [
-            launchParams.sec_group
-        ],
-        MinCount: 1,
-        MaxCount: 1,
-        UserData: ud_b64,
-        IamInstanceProfile: {
-            Arn: launchParams.iam
-        }
-    };
-    const command = new RunInstancesCommand(createInstanceParams);
-    const response = await client.send(command);
-    const instance_id = response.Instances[0].InstanceId
-   
-    console.log('Instance Launched:',instance_id);
-    return instance_id;
-}
-
-async function waitUntilInstanceReady(instance_id,region){
-    console.log(`Awaiting ${instance_id} status of ok`)
-    const client = new EC2Client({region});
-    const input = { // DescribeInstanceStatusRequest
-        InstanceIds: [ // InstanceIdStringList
-            instance_id
-        ],
-        DryRun: false,
-        IncludeAllInstances: true
-    };
-    
-    let totalSleepTime = 0;
-    let ok = false;
-    const command = new DescribeInstanceStatusCommand(input);
-    for (let i=0; i<100; i++){
-        const response = await client.send(command);
-        const status = response.InstanceStatuses[0].InstanceStatus.Status;
-        console.log(`\tCheck ${i+1} @ ${totalSleepTime}s: EC2 Status is ${status}`)
-        if (status !== 'ok'){
-            await sleep(10000);
-            totalSleepTime += 10;
-        } else {
-            console.log('Ec2 Instance Ready:' + status);
-            ok = true;
-            break;
-        }
-    }
-
-    if (ok === false){
-        console.log('ERR:::', `Ec2 Instance Not Ready After ${totalSleepTime}s`)
-        throw `Ec2 Instance Not Ready After ${totalSleepTime}s`
-    } else {
-        console.log(`Instance Ready After ${totalSleepTime}s. Waiting 5m to Proceed`);
-        await sleep(300000);
-    }
-    return true;
-}
-
-async function createAMI(instance_id,image_name,region){
-    console.log(`Building ${image_name} in ${region}`)
-    const client = new EC2Client({region});
-    const input = { // CreateImageRequest
-        Description: `Base Application Image`,
-        DryRun: false,
-        InstanceId: instance_id, // required
-        Name: image_name, // required
-        NoReboot: true
-      };
-      const command = new CreateImageCommand(input);
-      const response = await client.send(command);
-      console.log(`Created Image ${image_name} ID:${response.ImageId}`)
-      return response.ImageId;
-}
-
-async function terminateInstance(instance_id,region){
-    console.log('Terminating Instance ' + instance_id)
-    const client = new EC2Client({region});
-    const input = { // TerminateInstancesRequest
-        InstanceIds: [ instance_id ],
-        DryRun: false,
-    };
-    const command = new TerminateInstancesCommand(input);
-    const response = await client.send(command);
-    return true;
-}
-
-
-async function sleep(time){
-    return new Promise(function (resolve, reject) {
-        setTimeout(function () { resolve(true);
-        }, time);
-    });
-}

package/package.json

@@ -1 +1 @@
-{"name":"cloudmason","version":"1.9.33","description":"","main":"main.js","scripts":{"build":"node build.js"},"bin":{"mason":"./main.js"},"repository":{"type":"git","url":"https://github.com/kai-harvey/cloudmason.git"},"author":"Kai Harvey","license":"ISC","dependencies":{"@aws-sdk/client-acm":"^3.418.0","@aws-sdk/client-auto-scaling":"^3.470.0","@aws-sdk/client-cloudformation":"^3.418.0","@aws-sdk/client-ec2":"^3.864.0","@aws-sdk/client-iam":"^3.864.0","@aws-sdk/client-marketplace-catalog":"^3.716.0","@aws-sdk/client-route-53":"^3.425.0","@aws-sdk/client-s3":"^3.418.0","@aws-sdk/client-ssm":"^3.421.0","adm-zip":"^0.5.10","ssh2":"^1.16.0","yaml":"^2.6.1"}}
+{"name":"cloudmason","version":"2.0.36","description":"","main":"main.js","scripts":{"build":"node build.js"},"bin":{"mason":"./main.js"},"repository":{"type":"git","url":"https://github.com/kai-harvey/cloudmason.git"},"author":"Kai Harvey","license":"ISC","dependencies":{"@aws-sdk/client-acm":"^3.418.0","@aws-sdk/client-auto-scaling":"^3.470.0","@aws-sdk/client-cloudformation":"^3.418.0","@aws-sdk/client-ec2":"^3.864.0","@aws-sdk/client-iam":"^3.864.0","@aws-sdk/client-marketplace-catalog":"^3.716.0","@aws-sdk/client-route-53":"^3.425.0","@aws-sdk/client-s3":"^3.418.0","@aws-sdk/client-ssm":"^3.421.0","adm-zip":"^0.5.10","ssh2":"^1.16.0","yaml":"^2.6.1"}}