vm_shepherd 3.3.1 → 3.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a01907599698c0cda9c15c4ecb3f487b247e46ce
-  data.tar.gz: 3f1b44a2551dd9a833023f233ff9cfbdb9c19a32
+  metadata.gz: 25a49a93d04e9d20815c0e49f6d1a59b0dcfec95
+  data.tar.gz: 0884c51e7c2c8fee2a9211fb401eac3c07786bb7
 SHA512:
-  metadata.gz: a2fb77c590f2fe3b2b94c4a29e2cae4c196f60d64fd1581cea69d267628d98c2e53c613206e62369d548e77ed722dd4a1a24fa1e2278a3d1a370b352c400a9b1
-  data.tar.gz: acf114dc587b67f95ad680403717c288362eb30fdc44a6f35d0112cadc932fa89b0d672f4d4900d9ccde44ddcb423a867b438fa33a6347c9e0f347484686c3fe
+  metadata.gz: 3771e8eae943ca778bde5c58ad09a751a0f5d1d53e893f0bea4d4f397851b1067aa77b36eb598eef374799e5f1b720f2d9d66f89af562e52cd9deb0253978e14
+  data.tar.gz: 5f43ab61a483e64fda40436bd49945803522a1d24e39aca8b33465f993fd27f18a77aef8d0384c83b249b0c9e29258085b2b0075c7be78b743c1b85490970a45

data/lib/vm_shepherd/aws_manager.rb

@@ -189,7 +189,9 @@ module VmShepherd
       logger.info("Clearing contents of subnet: #{subnet_id}")
       subnet = AWS.ec2.subnets[subnet_id]
       volumes = []
+      instance_ids = []
       subnet.instances.each do |instance|
+        instance_ids.push instance.id
         instance.attachments.each do |_, attachment|
           volumes.push(attachment.volume) unless attachment.delete_on_termination
         end
@@ -197,6 +199,7 @@ module VmShepherd
         instance.terminate
       end
       destroy_volumes(volumes)
+      ensure_instances_terminated(instance_ids)
     end
 
     def destroy_volumes(volumes)
@@ -317,5 +320,18 @@ module VmShepherd
     def read_ami_id(ami_file_path)
       YAML.load_file(ami_file_path)[env_config.fetch('region')]
     end
+
+    def ensure_instances_terminated(instance_ids)
+      instances = instance_ids.collect {|instance_id| AWS.ec2.instances[instance_id] }
+      instances.each do |instance|
+        while instance.status == :shutting_down
+          sleep VmShepherd::RetryHelper::RETRY_INTERVAL
+        end
+
+        if instance.status != :terminated
+          raise "Expected instance: #{instance.id} to be terminated, but was #{instance.status}"
+        end
+      end
+    end
   end
 end

data/lib/vm_shepherd/version.rb

@@ -1,3 +1,3 @@
 module VmShepherd
-  VERSION = '3.3.1'.freeze
+  VERSION = '3.3.2'.freeze
 end

data/spec/vm_shepherd/aws_manager_spec.rb

@@ -412,8 +412,8 @@ module VmShepherd
       let(:subnets) { instance_double(AWS::EC2::SubnetCollection) }
       let(:subnet1) { instance_double(AWS::EC2::Subnet, instances: subnet1_instances) }
       let(:subnet2) { instance_double(AWS::EC2::Subnet, instances: subnet2_instances) }
-      let(:instance1) { instance_double(AWS::EC2::Instance, tags: {}, id: 'instance1') }
-      let(:instance2) { instance_double(AWS::EC2::Instance, tags: {}, id: 'instance2') }
+      let(:instance1) { instance_double(AWS::EC2::Instance, tags: {}, id: 'instance1', status: :terminated) }
+      let(:instance2) { instance_double(AWS::EC2::Instance, tags: {}, id: 'instance2', status: :terminated) }
       let(:subnet1_instances) { [instance1] }
       let(:subnet2_instances) { [instance2] }
       let(:cfn) { instance_double(AWS::CloudFormation, stacks: stack_collection) }
@@ -444,6 +444,8 @@ module VmShepherd
 
         allow(AWS::S3).to receive(:new).and_return(s3_client)
         allow(buckets).to receive(:[]).and_return(instance_double(AWS::S3::Bucket, exists?: false))
+
+        allow(ec2).to receive_message_chain(:instances, :[]).and_return(instance1, instance2)
       end
 
       it 'terminates all VMs in the subnet' do
@@ -453,8 +455,24 @@ module VmShepherd
         ami_manager.clean_environment
       end
 
+      it 'ensures the instances are terminated' do
+        expect(instance1).to receive(:status).and_return(:shutting_down, :shutting_down, :terminated)
+        expect(instance2).to receive(:status).and_return(:terminated)
+
+        ami_manager.clean_environment
+      end
+
+      it 'raises if any vms do not terminate' do
+        allow(ec2).to receive_message_chain(:instances, :[]).and_return(instance1, instance2)
+
+        expect(instance1).to receive(:status).and_return(:terminated).twice
+        expect(instance2).to receive(:status).and_return(:running).exactly(3).times
+
+        expect { ami_manager.clean_environment }.to raise_error("Expected instance: #{instance2.id} to be terminated, but was running")
+      end
+
       it 'polls the status every 30' do
-        expect(ami_manager).to receive(:retry_until).with(retry_limit: 60, retry_interval: 30)
+        expect(ami_manager).to receive(:retry_until).with(retry_limit: 90, retry_interval: 30)
 
         ami_manager.clean_environment
       end
@@ -471,9 +489,9 @@ module VmShepherd
         ami_manager.clean_environment
       end
 
-      it 'stops retrying after 60 times' do
+      it 'stops retrying after 90 times' do
         expect(stack).to receive(:status).and_return('DELETE_IN_PROGRESS').
-          exactly(60).times
+          exactly(90).times
 
         expect { ami_manager.clean_environment }.to raise_error(AwsManager::RetryLimitExceeded)
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: vm_shepherd
 version: !ruby/object:Gem::Version
-  version: 3.3.1
+  version: 3.3.2
 platform: ruby
 authors:
 - Ops Manager Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-03 00:00:00.000000000 Z
+date: 2016-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-v1