vm_shepherd 1.0.3 → 1.1.0

data/spec/fixtures/shepherd/vcloud.yml

@@ -1,40 +1,41 @@
 iaas_type: vcloud
-vm_shepherd_configs:
-- creds:
-    url: VAPP_URL
-    organization: VAPP_ORGANIZATION
-    user: VAPP_USERNAME
-    password: VAPP_PASSWORD
-  vdc:
-    name: VDC_NAME
-    catalog: VDC_CATALOG
-    network: VDC_NETWORK
-  vapp:
-    ops_manager_name: VAPP_NAME
-    product_catalog: PRODUCT_CATALOG
-    product_names:
-      - PRODUCT_1
-      - PRODUCT_2
-    ip: VAPP_IP
-    public_ip:
-    gateway: VAPP_GATEWAY
-    netmask: VAPP_NETMASK
-    dns: VAPP_DNS
-    ntp: VAPP_NTP
-- creds:
-    url: VAPP_URL-2
-    organization: VAPP_ORGANIZATION-2
-    user: VAPP_USERNAME-2
-    password: VAPP_PASSWORD-2
-  vdc:
-    name: VDC_NAME-2
-    catalog: VDC_CATALOG-2
-    network: VDC_NETWORK-2
-  vapp:
-    ops_manager_name: VAPP_NAME-2
-    ip: VAPP_IP-2
-    public_ip:
-    gateway: VAPP_GATEWAY-2
-    netmask: VAPP_NETMASK-2
-    dns: VAPP_DNS-2
-    ntp: VAPP_NTP-2
+vm_shepherd:
+  vm_configs:
+  - creds:
+      url: VAPP_URL
+      organization: VAPP_ORGANIZATION
+      user: VAPP_USERNAME
+      password: VAPP_PASSWORD
+    vdc:
+      name: VDC_NAME
+      catalog: VDC_CATALOG
+      network: VDC_NETWORK
+    vapp:
+      ops_manager_name: VAPP_NAME
+      product_catalog: PRODUCT_CATALOG
+      product_names:
+        - PRODUCT_1
+        - PRODUCT_2
+      ip: VAPP_IP
+      public_ip:
+      gateway: VAPP_GATEWAY
+      netmask: VAPP_NETMASK
+      dns: VAPP_DNS
+      ntp: VAPP_NTP
+  - creds:
+      url: VAPP_URL-2
+      organization: VAPP_ORGANIZATION-2
+      user: VAPP_USERNAME-2
+      password: VAPP_PASSWORD-2
+    vdc:
+      name: VDC_NAME-2
+      catalog: VDC_CATALOG-2
+      network: VDC_NETWORK-2
+    vapp:
+      ops_manager_name: VAPP_NAME-2
+      ip: VAPP_IP-2
+      public_ip:
+      gateway: VAPP_GATEWAY-2
+      netmask: VAPP_NETMASK-2
+      dns: VAPP_DNS-2
+      ntp: VAPP_NTP-2

data/spec/fixtures/shepherd/vsphere.yml

@@ -1,56 +1,57 @@
 iaas_type: vsphere
-vm_shepherd_configs:
-- vcenter_creds:
-    ip: OVA_URL
-    username: OVA_ORGANIZATION
-    password: OVA_PASSWORD
-  vsphere:
-    datacenter: VSPHERE_DATACENTER
-    cluster: VSPHERE_CLUSTER
-    network: VSPHERE_NETWORK
-    resource_pool: VSPHERE_RESOURCE_POOL
-    datastore: VSPHERE_DATASTORE
-    folder: VSPHERE_FOLDER
-  vm:
-    ip: OVA_IP
-    gateway: OVA_GATEWAY
-    netmask: OVA_NETMASK
-    dns: OVA_DNS
-    ntp_servers: OVA_NTP
-  cleanup:
-    datacenter: VSPHERE_OTHER_DATACENTER
-    datastores:
-      - VSPHERE_DATASTORE_ONE
-      - VSPHERE_DATASTORE_TWO
-    datacenter_folders_to_clean:
-      - DC_FOLDER_ONE
-      - DC_FOLDER_TWO
-    datastore_folders_to_clean:
-      - DS_DISK_FOLDER
-- vcenter_creds:
-    ip: OVA_URL-2
-    username: OVA_ORGANIZATION-2
-    password: OVA_PASSWORD-2
-  vsphere:
-    datacenter: VSPHERE_DATACENTER-2
-    cluster: VSPHERE_CLUSTER-2
-    network: VSPHERE_NETWORK-2
-    resource_pool: VSPHERE_RESOURCE_POOL-2
-    datastore: VSPHERE_DATASTORE-2
-    folder: VSPHERE_FOLDER-2
-  vm:
-    ip: OVA_IP-2
-    gateway: OVA_GATEWAY-2
-    netmask: OVA_NETMASK-2
-    dns: OVA_DNS-2
-    ntp_servers: OVA_NTP-2
-  cleanup:
-    datacenter: VSPHERE_OTHER_DATACENTER-2
-    datastores:
-      - VSPHERE_DATASTORE_ONE-2
-      - VSPHERE_DATASTORE_TWO-2
-    datacenter_folders_to_clean:
-      - DC_FOLDER_ONE-2
-      - DC_FOLDER_TWO-2
-    datastore_folders_to_clean:
-      - DS_DISK_FOLDER-2
+vm_shepherd:
+  vm_configs:
+  - vcenter_creds:
+      ip: OVA_URL
+      username: OVA_ORGANIZATION
+      password: OVA_PASSWORD
+    vsphere:
+      datacenter: VSPHERE_DATACENTER
+      cluster: VSPHERE_CLUSTER
+      network: VSPHERE_NETWORK
+      resource_pool: VSPHERE_RESOURCE_POOL
+      datastore: VSPHERE_DATASTORE
+      folder: VSPHERE_FOLDER
+    vm:
+      ip: OVA_IP
+      gateway: OVA_GATEWAY
+      netmask: OVA_NETMASK
+      dns: OVA_DNS
+      ntp_servers: OVA_NTP
+    cleanup:
+      datacenter: VSPHERE_OTHER_DATACENTER
+      datastores:
+        - VSPHERE_DATASTORE_ONE
+        - VSPHERE_DATASTORE_TWO
+      datacenter_folders_to_clean:
+        - DC_FOLDER_ONE
+        - DC_FOLDER_TWO
+      datastore_folders_to_clean:
+        - DS_DISK_FOLDER
+  - vcenter_creds:
+      ip: OVA_URL-2
+      username: OVA_ORGANIZATION-2
+      password: OVA_PASSWORD-2
+    vsphere:
+      datacenter: VSPHERE_DATACENTER-2
+      cluster: VSPHERE_CLUSTER-2
+      network: VSPHERE_NETWORK-2
+      resource_pool: VSPHERE_RESOURCE_POOL-2
+      datastore: VSPHERE_DATASTORE-2
+      folder: VSPHERE_FOLDER-2
+    vm:
+      ip: OVA_IP-2
+      gateway: OVA_GATEWAY-2
+      netmask: OVA_NETMASK-2
+      dns: OVA_DNS-2
+      ntp_servers: OVA_NTP-2
+    cleanup:
+      datacenter: VSPHERE_OTHER_DATACENTER-2
+      datastores:
+        - VSPHERE_DATASTORE_ONE-2
+        - VSPHERE_DATASTORE_TWO-2
+      datacenter_folders_to_clean:
+        - DC_FOLDER_ONE-2
+        - DC_FOLDER_TWO-2
+      datastore_folders_to_clean:
+        - DS_DISK_FOLDER-2

data/spec/vm_shepherd/aws_manager_spec.rb

@@ -9,38 +9,97 @@ module VmShepherd
     let(:elastic_ip_id) { 'elastic-ip-id' }
     let(:ec2) { double('AWS.ec2') }
 
-    let(:aws_options) do
+    let(:env_config) do
       {
+        stack_name: 'aws-stack-name',
         aws_access_key: 'aws-access-key',
         aws_secret_key: 'aws-secret-key',
-        ssh_key_name: 'ssh-key-name',
-        security_group_id: 'security-group-id',
-        public_subnet_id: 'public-subnet-id',
-        private_subnet_id: 'private-subnet-id',
-        elastic_ip_id: elastic_ip_id,
-        vm_name: 'Ops Manager: clean_install_spec'
+        json_file: 'cloudformation.json',
+        parameters: {
+          'some_parameter' => 'some-answer',
+        },
+        outputs: {
+          ssh_key_name: 'ssh-key-name',
+          security_group: 'security-group-id',
+          public_subnet_id: 'public-subnet-id',
+          private_subnet_id: 'private-subnet-id',
+        },
       }
     end
 
-    subject(:ami_manager) { AwsManager.new(aws_options) }
+    let(:vm_config) do
+      {
+        vm_name: 'some-vm-name',
+      }
+    end
+
+    subject(:ami_manager) { AwsManager.new(env_config) }
 
     before do
       expect(AWS).to receive(:config).with(
-          access_key_id: aws_options.fetch(:aws_access_key),
-          secret_access_key: aws_options.fetch(:aws_secret_key),
+          access_key_id: env_config.fetch(:aws_access_key),
+          secret_access_key: env_config.fetch(:aws_secret_key),
           region: 'us-east-1',
         )
 
       allow(AWS).to receive(:ec2).and_return(ec2)
+      allow(ami_manager).to receive(:sleep) # speed up retry logic
+    end
+
+    describe '#prepare_environment' do
+      let(:cloudformation_template_file) { Tempfile.new('cloudformation_template_file').tap { |f| f.write('{}'); f.close } }
+      let(:cfm) { instance_double(AWS::CloudFormation, stacks: stack_collection) }
+      let(:stack) { instance_double(AWS::CloudFormation::Stack, status: 'CREATE_COMPLETE') }
+      let(:stack_collection) { instance_double(AWS::CloudFormation::StackCollection) }
+
+      before do
+        allow(AWS::CloudFormation).to receive(:new).and_return(cfm)
+        allow(stack_collection).to receive(:create).and_return(stack)
+      end
+
+      it 'creates the stack with the correct parameters' do
+        expect(stack_collection).to receive(:create).with(
+            'aws-stack-name',
+            '{}',
+            parameters: {
+              'some_parameter' => 'some-answer',
+            },
+            capabilities: ['CAPABILITY_IAM']
+          )
+        ami_manager.prepare_environment(cloudformation_template_file.path)
+      end
+
+      it 'waits for the stack to finish creating' do
+        expect(stack).to receive(:status).and_return('CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_IN_PROGRESS', 'CREATE_COMPLETE')
+
+        ami_manager.prepare_environment(cloudformation_template_file.path)
+      end
+
+      it 'stops retrying after 360 times' do
+        expect(stack).to receive(:status).and_return('CREATE_IN_PROGRESS').
+            exactly(360).times
+
+        expect { ami_manager.prepare_environment(cloudformation_template_file.path) }.to raise_error(AwsManager::RetryLimitExceeded)
+      end
+
+      it 'aborts if stack fails to create' do
+        expect(stack).to receive(:status).and_return('CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_COMPLETE').ordered
+        expect(stack).to receive(:delete)
+        expect {
+          ami_manager.prepare_environment(cloudformation_template_file.path)
+        }.to raise_error('Unexpected status for stack aws-stack-name : ROLLBACK_COMPLETE')
+      end
     end
 
     describe '#deploy' do
       let(:instance) { instance_double(AWS::EC2::Instance, status: :running, associate_elastic_ip: nil, add_tag: nil) }
+      let(:elastic_ip) { instance_double(AWS::EC2::ElasticIp, allocation_id: 'allocation-id') }
       let(:instances) { instance_double(AWS::EC2::InstanceCollection, create: instance) }
+      let(:elastic_ips) { instance_double(AWS::EC2::ElasticIpCollection, create: elastic_ip) }
 
       before do
         allow(ec2).to receive(:instances).and_return(instances)
-        allow(ami_manager).to receive(:sleep) # speed up retry logic
+        allow(ec2).to receive(:elastic_ips).and_return(elastic_ips)
       end
 
       it 'creates an instance using AWS SDK v1' do
@@ -48,10 +107,10 @@ module VmShepherd
             image_id: ami_id,
             key_name: 'ssh-key-name',
             security_group_ids: ['security-group-id'],
-            subnet: aws_options.fetch(:public_subnet_id),
+            subnet: 'public-subnet-id',
             instance_type: 'm3.medium').and_return(instance)
 
-        ami_manager.deploy(ami_file_path)
+        ami_manager.deploy(ami_file_path: ami_file_path, vm_config: vm_config)
       end
 
       context 'when the ip address is in use' do
@@ -59,21 +118,21 @@ module VmShepherd
           expect(instances).to receive(:create).and_raise(AWS::EC2::Errors::InvalidIPAddress::InUse).once
           expect(instances).to receive(:create).and_return(instance).once
 
-          ami_manager.deploy(ami_file_path)
+          ami_manager.deploy(ami_file_path: ami_file_path, vm_config: vm_config)
         end
 
         it 'stops retrying after 60 times' do
           expect(instances).to receive(:create).and_raise(AWS::EC2::Errors::InvalidIPAddress::InUse).
               exactly(AwsManager::RETRY_LIMIT).times
 
-          expect { ami_manager.deploy(ami_file_path) }.to raise_error(AwsManager::RetryLimitExceeded)
+          expect { ami_manager.deploy(ami_file_path: ami_file_path, vm_config: vm_config) }.to raise_error(AwsManager::RetryLimitExceeded)
         end
       end
 
       it 'does not return until the instance is running' do
         expect(instance).to receive(:status).and_return(:pending, :pending, :pending, :running)
 
-        ami_manager.deploy(ami_file_path)
+        ami_manager.deploy(ami_file_path: ami_file_path, vm_config: vm_config)
       end
 
       it 'handles API endpoints not knowing (right away) about the instance created' do
@@ -81,26 +140,29 @@ module VmShepherd
             exactly(AwsManager::RETRY_LIMIT - 1).times
         expect(instance).to receive(:status).and_return(:running).once
 
-        ami_manager.deploy(ami_file_path)
+        ami_manager.deploy(ami_file_path: ami_file_path, vm_config: vm_config)
       end
 
       it 'stops retrying after 60 times' do
         expect(instance).to receive(:status).and_return(:pending).
             exactly(AwsManager::RETRY_LIMIT).times
 
-        expect { ami_manager.deploy(ami_file_path) }.to raise_error(AwsManager::RetryLimitExceeded)
+        expect { ami_manager.deploy(ami_file_path: ami_file_path, vm_config: vm_config) }.to raise_error(AwsManager::RetryLimitExceeded)
       end
 
-      it 'attaches the elastic IP' do
-        expect(instance).to receive(:associate_elastic_ip).with(aws_options.fetch(:elastic_ip_id))
+      it 'creates and attaches an elastic IP' do
+        expect(ec2).to receive_message_chain(:elastic_ips, :create).with(
+            vpc: true).and_return(elastic_ip)
 
-        ami_manager.deploy(ami_file_path)
+        expect(instance).to receive(:associate_elastic_ip).with(elastic_ip.allocation_id)
+
+        ami_manager.deploy(ami_file_path: ami_file_path, vm_config: vm_config)
       end
 
       it 'tags the instance with a name' do
-        expect(instance).to receive(:add_tag).with('Name', value: aws_options.fetch(:vm_name))
+        expect(instance).to receive(:add_tag).with('Name', value: 'some-vm-name')
 
-        ami_manager.deploy(ami_file_path)
+        ami_manager.deploy(ami_file_path: ami_file_path, vm_config: vm_config)
       end
     end
 
@@ -112,19 +174,28 @@ module VmShepherd
       let(:instance2) { instance_double(AWS::EC2::Instance, tags: {}) }
       let(:subnet1_instances) { [instance1] }
       let(:subnet2_instances) { [instance2] }
+      let(:cfm) { instance_double(AWS::CloudFormation, stacks: stack_collection) }
+      let(:stack) { instance_double(AWS::CloudFormation::Stack, status: 'DELETE_COMPLETE', delete: nil) }
+      let(:stack_collection) { instance_double(AWS::CloudFormation::StackCollection) }
 
-      let(:instance1_volume) { instance_double(AWS::EC2::Volume)}
+      let(:instance1_volume) { instance_double(AWS::EC2::Volume) }
       let(:instance1_attachment) do
         instance_double(AWS::EC2::Attachment, volume: instance1_volume, delete_on_termination: true)
       end
 
       before do
+        allow(AWS::CloudFormation).to receive(:new).and_return(cfm)
+        allow(stack_collection).to receive(:[]).and_return(stack)
+
         allow(ec2).to receive(:subnets).and_return(subnets)
         allow(subnets).to receive(:[]).with('public-subnet-id').and_return(subnet1)
         allow(subnets).to receive(:[]).with('private-subnet-id').and_return(subnet2)
 
         allow(instance1).to receive(:attachments).and_return({'/dev/test' => instance1_attachment})
         allow(instance2).to receive(:attachments).and_return({})
+
+        allow(instance1).to receive(:terminate)
+        allow(instance2).to receive(:terminate)
       end
 
       it 'terminates all VMs in the subnet' do
@@ -134,20 +205,46 @@ module VmShepherd
         ami_manager.clean_environment
       end
 
-      context 'when an instance has the magical tag' do
-        let(:persistent_instance) { instance_double(AWS::EC2::Instance, tags: persist_tag) }
-        let(:instances) { [instance1, instance2, persistent_instance] }
+      it 'deletes the stack' do
+        expect(stack_collection).to receive(:[]).with('aws-stack-name').and_return(stack)
+        expect(stack).to receive(:delete)
+        ami_manager.clean_environment
+      end
 
-        context 'when the do not terminate tag is present' do
-          let(:persist_tag) { { AwsManager::DO_NOT_TERMINATE_TAG_KEY => 'any value' } }
-          it 'does not attempt to terminate this instance' do
-            expect(instance1).to receive(:terminate)
-            expect(instance2).to receive(:terminate)
-            expect(persistent_instance).not_to receive(:terminate)
+      it 'waits for stack deletion to complete' do
+        expect(stack).to receive(:status).and_return('DELETE_IN_PROGRESS', 'DELETE_IN_PROGRESS', 'DELETE_IN_PROGRESS', 'DELETE_COMPLETE')
 
-            ami_manager.clean_environment
-          end
-        end
+        ami_manager.clean_environment
+      end
+
+      it 'stops retrying after 360 times' do
+        expect(stack).to receive(:status).and_return('DELETE_IN_PROGRESS').
+            exactly(360).times
+
+        expect { ami_manager.clean_environment }.to raise_error(AwsManager::RetryLimitExceeded)
+      end
+
+      it 'aborts if stack reports unexpected status' do
+        expect(stack).to receive(:status).and_return('DELETE_IN_PROGRESS', 'UNEXPECTED_STATUS').ordered
+        expect {
+          ami_manager.clean_environment
+        }.to raise_error('Unexpected status for stack aws-stack-name : UNEXPECTED_STATUS')
+      end
+
+      it 'aborts if stack throws error' do
+        expect(stack).to receive(:status).and_raise(AWS::CloudFormation::Errors::ValidationError)
+        allow(stack).to receive(:exists?).and_return(true)
+        expect {
+          ami_manager.clean_environment
+        }.to raise_error(AWS::CloudFormation::Errors::ValidationError)
+      end
+
+      it 'succeeds if stack throws error and stack deletion has completed' do
+        expect(stack).to receive(:status).and_raise(AWS::CloudFormation::Errors::ValidationError)
+        allow(stack).to receive(:exists?).and_return(false)
+        expect {
+          ami_manager.clean_environment
+        }.not_to raise_error
       end
 
       context 'when the instance has volumes that are NOT delete_on_termination' do
@@ -181,33 +278,43 @@ module VmShepherd
     end
 
     describe '#destroy' do
-      let(:elastic_ips) { instance_double(AWS::EC2::ElasticIpCollection) }
-      let(:elastic_ip) { instance_double(AWS::EC2::ElasticIp, instance: instance, allocation_id: elastic_ip_id) }
-      let(:instance) { instance_double(AWS::EC2::Instance, tags: {}) }
+      let(:elastic_ip) { nil }
+      let(:instance) { instance_double(AWS::EC2::Instance, tags: {'Name' => 'some-vm-name'}, elastic_ip: elastic_ip) }
+      let(:non_terminated_instance) { instance_double(AWS::EC2::Instance, tags: {}) }
+      let(:instances) { [non_terminated_instance, instance] }
 
       before do
-        allow(ec2).to receive(:elastic_ips).and_return(elastic_ips)
-        allow(elastic_ips).to receive(:each).and_yield(elastic_ip)
+        allow(ec2).to receive(:instances).and_return(instances)
+        allow(instance).to receive(:terminate)
       end
 
-      it 'terminates the VM that matches the IP' do
+      it 'terminates the VM with the specified name' do
+        expect(non_terminated_instance).not_to receive(:terminate)
         expect(instance).to receive(:terminate)
 
-        ami_manager.destroy
+        ami_manager.destroy(vm_config)
       end
 
-      context 'when an instance has the magical tag' do
-        let(:persistent_instance) { instance_double(AWS::EC2::Instance, tags: persist_tag) }
-        let(:instances) { [instance1, instance2, persistent_instance] }
+      context 'when there is an elastic ip' do
+        let(:elastic_ip) { instance_double(AWS::EC2::ElasticIp) }
 
-        context 'when there is no instance attached' do
-          before do
-            allow(elastic_ip).to receive(:instance).and_return(nil)
-          end
+        before do
+          allow(elastic_ip).to receive(:delete)
+          allow(elastic_ip).to receive(:disassociate)
+        end
 
-          it 'does not explode' do
-            ami_manager.destroy
-          end
+        it 'terminates the VM with the specified name' do
+          expect(non_terminated_instance).not_to receive(:terminate)
+          expect(instance).to receive(:terminate)
+
+          ami_manager.destroy(vm_config)
+        end
+
+        it 'disassociates and deletes the ip associated with the terminated vm' do
+          expect(elastic_ip).to receive(:disassociate).ordered
+          expect(elastic_ip).to receive(:delete).ordered
+
+          ami_manager.destroy(vm_config)
         end
       end
     end