vm_shepherd 3.1.4 → 3.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 32cc9f496785d39b7c220ce6b64d8664c7de8768
-  data.tar.gz: bcb21b3a0ed017a5141de5d9947ca4f5ceef6577
+  metadata.gz: e062bc9d9f56f7ebaed36eaf5843e00ef1b61343
+  data.tar.gz: a3a2ed3f83e784d2ad0686c627505c121397a77b
 SHA512:
-  metadata.gz: 7951b966a53745015b015d83e22400e2da8938ec7382bccbec5b00ede38a9d50c1620087e9b20e544e89994038d27111ed86fba795d405fd24ac84ae1074556b
-  data.tar.gz: ba1a4a420d41d6d091879a6e2afd91156a8561783bafe868cea2e9f181b9548ac20831a90268433752bdb96e76059aff0a5f4e11e805acc769160d85ff3f0beb
+  metadata.gz: 2f3693d16a10aa1420ee10834e453a344b7ab46f721d4f84ed1709a8eefc5f3914f77ae01253423b48981a9a9fc61a23b709a89a0a0e3602ac995c43a98f8210
+  data.tar.gz: 53636d6efa44563fba84a341c49596a9c7e614d417cf713ce0c802ffd264ed32f3049cbfc9cb5b82fc8d47e54681138b0dae4242e7d6017fe6d0ea11eb22c7fa

data/lib/vm_shepherd/aws_manager.rb

@@ -17,6 +17,7 @@ module VmShepherd
     ROLLBACK_COMPLETE = 'ROLLBACK_COMPLETE'
     DELETE_IN_PROGRESS = 'DELETE_IN_PROGRESS'
     DELETE_COMPLETE = 'DELETE_COMPLETE'
+    DELETE_FAILED = 'DELETE_FAILED'
 
     def initialize(env_config:, logger:)
       AWS.config(
@@ -36,7 +37,7 @@ module VmShepherd
       stack = cfm.stacks.create(env_config.fetch('stack_name'), template, parameters: env_config.fetch('parameters'), capabilities: ['CAPABILITY_IAM'])
 
       logger.info("Waiting for status: #{CREATE_COMPLETE}")
-      retry_until(retry_limit: 60, retry_interval: 30) do
+      retry_until(retry_limit: 80, retry_interval: 30) do
         status = stack.status
         logger.info("current stack status: #{status}")
         case status
@@ -253,6 +254,7 @@ module VmShepherd
       cfm = AWS::CloudFormation.new
       stack = cfm.stacks[stack_name]
       logger.info('deleting CloudFormation stack')
+      delete_retried = false
       stack.delete
       logger.info("waiting until status: #{DELETE_COMPLETE}")
       retry_until(retry_limit: 60, retry_interval: 30) do
@@ -264,6 +266,15 @@ module VmShepherd
               true
             when DELETE_IN_PROGRESS
               false
+            when DELETE_FAILED
+              if delete_retried
+                raise "Two delete retries have failed #{stack_name} : #{status}"
+              else
+                delete_retried = true
+                stack.delete
+                sleep 2 * 60
+                false
+              end
             else
               raise "Unexpected status for stack #{stack_name} : #{status}"
           end

data/lib/vm_shepherd/version.rb

@@ -1,3 +1,3 @@
 module VmShepherd
-  VERSION = '3.1.4'.freeze
+  VERSION = '3.1.5'.freeze
 end

data/spec/vm_shepherd/aws_manager_spec.rb

@@ -72,7 +72,7 @@ module VmShepherd
 
       describe 'cloudformation' do
         it 'polls the status every 30 seconds' do
-          expect(ami_manager).to receive(:retry_until).with(retry_limit: 60, retry_interval: 30)
+          expect(ami_manager).to receive(:retry_until).with(retry_limit: 80, retry_interval: 30)
 
           ami_manager.prepare_environment(cloudformation_template_file.path)
         end
@@ -95,9 +95,9 @@ module VmShepherd
           ami_manager.prepare_environment(cloudformation_template_file.path)
         end
 
-        it 'stops retrying after 30 times' do
+        it 'stops retrying after 80 times' do
           expect(stack).to receive(:status).and_return('CREATE_IN_PROGRESS').
-            exactly(60).times
+            exactly(80).times
 
           expect { ami_manager.prepare_environment(cloudformation_template_file.path) }.to raise_error(AwsManager::RetryLimitExceeded)
         end
@@ -418,7 +418,6 @@ module VmShepherd
         expect { ami_manager.clean_environment }.to raise_error(AwsManager::RetryLimitExceeded)
       end
 
-
       it 'aborts if stack reports unexpected status' do
         expect(stack).to receive(:status).and_return('DELETE_IN_PROGRESS', 'UNEXPECTED_STATUS').ordered
         expect {
@@ -457,6 +456,28 @@ module VmShepherd
         ami_manager.clean_environment
       end
 
+      context 'when the stack fails to delete' do
+        it 'retries deletes twice' do
+          expect(stack).to receive(:delete).ordered
+          expect(stack).to receive(:status).and_return('DELETE_IN_PROGRESS', 'DELETE_FAILED').ordered
+          expect(stack).to receive(:delete).ordered
+          expect(stack).to receive(:status).and_return('DELETE_IN_PROGRESS', 'DELETE_COMPLETE').ordered
+
+          ami_manager.clean_environment
+        end
+
+        it 'fails after two delete retries' do
+          expect(stack).to receive(:delete).ordered
+          expect(stack).to receive(:status).and_return('DELETE_IN_PROGRESS', 'DELETE_FAILED').ordered
+          expect(stack).to receive(:delete).ordered
+          expect(stack).to receive(:status).and_return('DELETE_IN_PROGRESS', 'DELETE_FAILED').ordered
+
+          expect {
+            ami_manager.clean_environment
+          }.to raise_error(/Two delete retries have failed/)
+        end
+      end
+
       context 'when a subnet is not provided' do
         before do
           env_config['outputs']['subnets'] = env_config['outputs']['subnets'][0..-2]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: vm_shepherd
 version: !ruby/object:Gem::Version
-  version: 3.1.4
+  version: 3.1.5
 platform: ruby
 authors:
 - Ops Manager Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-16 00:00:00.000000000 Z
+date: 2016-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-v1