cf_deployer 1.5.0 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6f550eef2ccbd8df2bff3ad4441d228800bab756
-  data.tar.gz: f3981f2ef12ab4338ddb5bec47fc31c6326e2582
+  metadata.gz: b15fbdcf8f040017be2e4f7fd2fca49e99327164
+  data.tar.gz: 6138254475f90d7983da1823754ace6134be9d85
 SHA512:
-  metadata.gz: 0d09d5100924380dd8643c041bdd188b6d53fd3a2b408a4a7609ce785fc935149dc37a39d2b192c9003651cc695f1757c8014134bcbe930f679c4648691e3a78
-  data.tar.gz: 93e9e12ece59c4a0a01508047be7da243977770b82ed8d1d844f1eea81c8b98b3875e942db5e88231ed4716db1004f8c87baca255fd9a61d3a02f24b9b79a0d6
+  metadata.gz: 209ff6fd2211ae9b37ef0592aaec30066139ee37742cc36bdb45593ad0bc8f4476393ccc76be3d04204b9021bef07621b703bdd09b65b85580f18827ff5ceb9a
+  data.tar.gz: 9d3a7676274ac56d77ef0d545f1dc2441bf9a84ce07bf50cae1aec809df349ff8a5f23202bfdabedf8064d08383ac191d879606c9a80b3586e91792f2819f91b

data/ChangeLog.md

@@ -57,3 +57,9 @@ version 1.4.0
 
 version 1.5.0
   - Treat deployments that end in a rollback as a failure
+
+version 1.6.0
+  - Improve warm up for AutoScalingGroup based deployments (See: https://github.com/manheim/cf_deployer/issues/32)
+  - Automatically rollback on failures for AutoScalingGroup based deployments (See: https://github.com/manheim/cf_deployer/issues/39)
+  - Improve error message when no stack is exists on ASG-based switch (See: https://github.com/manheim/cf_deployer/issues/50)
+  - Reduce AWS calls during healthy_instance_count (See: https://github.com/manheim/cf_deployer/issues/48)

data/Rakefile

@@ -22,7 +22,12 @@ namespace :spec do
     t.pattern = 'spec/functional/**/*_spec.rb'
   end
 
-  task :all => [:unit, :functional]
+  RSpec::Core::RakeTask.new(:aws_call_count) do |t|
+    t.rspec_opts = RSPEC_OPTS
+    t.pattern = 'spec/aws_call_count/**/*_spec.rb'
+  end
+
+  task :all => [:unit, :functional, :aws_call_count]
 end
 
 task :default  => 'spec:all'

data/cf_deployer.gemspec

@@ -18,6 +18,8 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency 'pry', '~> 0.10.1'
   gem.add_development_dependency 'rspec', '2.14.1'
   gem.add_development_dependency 'rake', '~> 10.3.0'
+  gem.add_development_dependency 'webmock', '~> 2.1.0'
+  gem.add_development_dependency 'vcr', '~> 2.9.3'
 
   gem.files         = `git ls-files`.split($\).reject {|f| f =~ /^samples\// }
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }

data/fixtures/vcr_cassettes/aws_call_count/driver/auto_scaling_group/healthy_instance_count.yml

@@ -0,0 +1,243 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: https://autoscaling.us-east-1.amazonaws.com/
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ! "<DescribeAutoScalingGroupsResponse xmlns=\"http://autoscaling.amazonaws.com/doc/2011-01-01/\">\n
+        \ <DescribeAutoScalingGroupsResult>\n    <AutoScalingGroups>\n      <member>\n
+        \       <HealthCheckType>ELB</HealthCheckType>\n        <LoadBalancerNames>\n
+        \         <member>myELB</member>\n        </LoadBalancerNames>\n
+        \       <Instances>\n          <member>\n            <LaunchConfigurationName>myLaunchConfig</LaunchConfigurationName>\n
+        \           <LifecycleState>InService</LifecycleState>\n            <InstanceId>instance1</InstanceId>\n
+        \           <HealthStatus>Healthy</HealthStatus>\n            <ProtectedFromScaleIn>false</ProtectedFromScaleIn>\n
+        \           <AvailabilityZone>us-east-1d</AvailabilityZone>\n          </member>\n
+        \         <member>\n            <LaunchConfigurationName>myLaunchConfig</LaunchConfigurationName>\n
+        \           <LifecycleState>InService</LifecycleState>\n            <InstanceId>instance2</InstanceId>\n
+        \           <HealthStatus>Healthy</HealthStatus>\n            <ProtectedFromScaleIn>false</ProtectedFromScaleIn>\n
+        \           <AvailabilityZone>us-east-1c</AvailabilityZone>\n          </member>\n
+        \         <member>\n            <LaunchConfigurationName>myLaunchConfig</LaunchConfigurationName>\n
+        \           <LifecycleState>InService</LifecycleState>\n            <InstanceId>instance3</InstanceId>\n
+        \           <HealthStatus>Healthy</HealthStatus>\n            <ProtectedFromScaleIn>false</ProtectedFromScaleIn>\n
+        \           <AvailabilityZone>us-east-1e</AvailabilityZone>\n          </member>\n
+        \         <member>\n            <LaunchConfigurationName>myLaunchConfig</LaunchConfigurationName>\n
+        \           <LifecycleState>InService</LifecycleState>\n            <InstanceId>instance4</InstanceId>\n
+        \           <HealthStatus>Healthy</HealthStatus>\n            <ProtectedFromScaleIn>false</ProtectedFromScaleIn>\n
+        \           <AvailabilityZone>us-east-1e</AvailabilityZone>\n          </member>\n
+        \       </Instances>\n        <TerminationPolicies>\n          <member>Default</member>\n
+        \       </TerminationPolicies>\n        <DefaultCooldown>300</DefaultCooldown>\n
+        \       <AutoScalingGroupARN>arn:aws:autoscaling:us-east-1:12345:autoScalingGroup:12345:autoScalingGroupName/myASG</AutoScalingGroupARN>\n
+        \       <EnabledMetrics/>\n        <MaxSize>5</MaxSize>\n        <AvailabilityZones>\n
+        \         <member>us-east-1c</member>\n          <member>us-east-1d</member>\n
+        \         <member>us-east-1e</member>\n        </AvailabilityZones>\n        <TargetGroupARNs/>\n
+        \       <Tags/>\n
+        \       <LaunchConfigurationName>myLaunchConfig</LaunchConfigurationName>\n
+        \       <AutoScalingGroupName>myASG</AutoScalingGroupName>\n
+        \       <HealthCheckGracePeriod>600</HealthCheckGracePeriod>\n        <NewInstancesProtectedFromScaleIn>false</NewInstancesProtectedFromScaleIn>\n
+        \       <CreatedTime>2016-09-23T03:52:21.733Z</CreatedTime>\n        <MinSize>1</MinSize>\n
+        \       <SuspendedProcesses/>\n        <DesiredCapacity>4</DesiredCapacity>\n
+        \       <VPCZoneIdentifier>subnet</VPCZoneIdentifier>\n
+        \     </member>\n    </AutoScalingGroups>\n  </DescribeAutoScalingGroupsResult>\n
+        \ <ResponseMetadata>\n    <RequestId>12345</RequestId>\n
+        \ </ResponseMetadata>\n</DescribeAutoScalingGroupsResponse>\n"
+    http_version: 
+  recorded_at: Tue, 27 Sep 2016 05:47:40 GMT
+- request:
+    method: post
+    uri: https://autoscaling.us-east-1.amazonaws.com/
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ! "<DescribeAutoScalingInstancesResponse xmlns=\"http://autoscaling.amazonaws.com/doc/2011-01-01/\">\n
+        \ <DescribeAutoScalingInstancesResult>\n    <AutoScalingInstances>\n      <member>\n
+        \       <LaunchConfigurationName>myLaunchConfig</LaunchConfigurationName>\n
+        \       <LifecycleState>InService</LifecycleState>\n        <AutoScalingGroupName></AutoScalingGroupName>\n
+        \       <InstanceId>instance1</InstanceId>\n        <HealthStatus>HEALTHY</HealthStatus>\n
+        \       <ProtectedFromScaleIn>false</ProtectedFromScaleIn>\n        <AvailabilityZone>us-east-1d</AvailabilityZone>\n
+        \     </member>\n    </AutoScalingInstances>\n  </DescribeAutoScalingInstancesResult>\n
+        \ <ResponseMetadata>\n    <RequestId>12346</RequestId>\n
+        \ </ResponseMetadata>\n</DescribeAutoScalingInstancesResponse>\n"
+    http_version: 
+  recorded_at: Tue, 27 Sep 2016 05:47:40 GMT
+- request:
+    method: post
+    uri: https://elasticloadbalancing.us-east-1.amazonaws.com/
+    body:
+      encoding: UTF-8
+      string: Action=DescribeInstanceHealth&LoadBalancerName=myELB&Timestamp=2016-09-27T05%3A47%3A41Z&Version=2012-06-01
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ! "<DescribeInstanceHealthResponse xmlns=\"http://elasticloadbalancing.amazonaws.com/doc/2012-06-01/\">\n
+        \ <DescribeInstanceHealthResult>\n    <InstanceStates>\n      <member>\n        <Description>N/A</Description>\n
+        \       <InstanceId>instance1</InstanceId>\n        <ReasonCode>N/A</ReasonCode>\n
+        \       <State>InService</State>\n      </member>\n      <member>\n        <Description>N/A</Description>\n
+        \       <InstanceId>instance2</InstanceId>\n        <ReasonCode>N/A</ReasonCode>\n
+        \       <State>InService</State>\n      </member>\n      <member>\n        <Description>N/A</Description>\n
+        \       <InstanceId>instance3</InstanceId>\n        <ReasonCode>N/A</ReasonCode>\n
+        \       <State>InService</State>\n      </member>\n      <member>\n        <Description>N/A</Description>\n
+        \       <InstanceId>instance4</InstanceId>\n        <ReasonCode>N/A</ReasonCode>\n
+        \       <State>InService</State>\n      </member>\n    </InstanceStates>\n
+        \ </DescribeInstanceHealthResult>\n  <ResponseMetadata>\n    <RequestId>12347</RequestId>\n
+        \ </ResponseMetadata>\n</DescribeInstanceHealthResponse>\n"
+    http_version: 
+  recorded_at: Tue, 27 Sep 2016 05:47:41 GMT
+- request:
+    method: post
+    uri: https://autoscaling.us-east-1.amazonaws.com/
+    body:
+      encoding: UTF-8
+      string: Action=DescribeAutoScalingInstances&InstanceIds.member.1=instance2&Timestamp=2016-09-27T05%3A47%3A41Z&Version=2011-01-01
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ! "<DescribeAutoScalingInstancesResponse xmlns=\"http://autoscaling.amazonaws.com/doc/2011-01-01/\">\n
+        \ <DescribeAutoScalingInstancesResult>\n    <AutoScalingInstances>\n      <member>\n
+        \       <LaunchConfigurationName>myLaunchConfig</LaunchConfigurationName>\n
+        \       <LifecycleState>InService</LifecycleState>\n        <AutoScalingGroupName>myASG</AutoScalingGroupName>\n
+        \       <InstanceId>instance2</InstanceId>\n        <HealthStatus>HEALTHY</HealthStatus>\n
+        \       <ProtectedFromScaleIn>false</ProtectedFromScaleIn>\n        <AvailabilityZone>us-east-1c</AvailabilityZone>\n
+        \     </member>\n    </AutoScalingInstances>\n  </DescribeAutoScalingInstancesResult>\n
+        \ <ResponseMetadata>\n    <RequestId>12348</RequestId>\n
+        \ </ResponseMetadata>\n</DescribeAutoScalingInstancesResponse>\n"
+    http_version: 
+  recorded_at: Tue, 27 Sep 2016 05:47:42 GMT
+- request:
+    method: post
+    uri: https://elasticloadbalancing.us-east-1.amazonaws.com/
+    body:
+      encoding: UTF-8
+      string: Action=DescribeInstanceHealth&LoadBalancerName=myELB&Timestamp=2016-09-27T05%3A47%3A42Z&Version=2012-06-01
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ! "<DescribeInstanceHealthResponse xmlns=\"http://elasticloadbalancing.amazonaws.com/doc/2012-06-01/\">\n
+        \ <DescribeInstanceHealthResult>\n    <InstanceStates>\n      <member>\n        <Description>N/A</Description>\n
+        \       <InstanceId>instance1</InstanceId>\n        <ReasonCode>N/A</ReasonCode>\n
+        \       <State>InService</State>\n      </member>\n      <member>\n        <Description>N/A</Description>\n
+        \       <InstanceId>instance2</InstanceId>\n        <ReasonCode>N/A</ReasonCode>\n
+        \       <State>InService</State>\n      </member>\n      <member>\n        <Description>N/A</Description>\n
+        \       <InstanceId>instance3</InstanceId>\n        <ReasonCode>N/A</ReasonCode>\n
+        \       <State>InService</State>\n      </member>\n      <member>\n        <Description>N/A</Description>\n
+        \       <InstanceId>instance4</InstanceId>\n        <ReasonCode>N/A</ReasonCode>\n
+        \       <State>InService</State>\n      </member>\n    </InstanceStates>\n
+        \ </DescribeInstanceHealthResult>\n  <ResponseMetadata>\n    <RequestId>12349</RequestId>\n
+        \ </ResponseMetadata>\n</DescribeInstanceHealthResponse>\n"
+    http_version: 
+  recorded_at: Tue, 27 Sep 2016 05:47:42 GMT
+- request:
+    method: post
+    uri: https://autoscaling.us-east-1.amazonaws.com/
+    body:
+      encoding: UTF-8
+      string: Action=DescribeAutoScalingInstances&InstanceIds.member.1=instance3&Timestamp=2016-09-27T05%3A47%3A42Z&Version=2011-01-01
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ! "<DescribeAutoScalingInstancesResponse xmlns=\"http://autoscaling.amazonaws.com/doc/2011-01-01/\">\n
+        \ <DescribeAutoScalingInstancesResult>\n    <AutoScalingInstances>\n      <member>\n
+        \       <LaunchConfigurationName>myLaunchConfig</LaunchConfigurationName>\n
+        \       <LifecycleState>InService</LifecycleState>\n        <AutoScalingGroupName>myASG</AutoScalingGroupName>\n
+        \       <InstanceId>instance3</InstanceId>\n        <HealthStatus>HEALTHY</HealthStatus>\n
+        \       <ProtectedFromScaleIn>false</ProtectedFromScaleIn>\n        <AvailabilityZone>us-east-1e</AvailabilityZone>\n
+        \     </member>\n    </AutoScalingInstances>\n  </DescribeAutoScalingInstancesResult>\n
+        \ <ResponseMetadata>\n    <RequestId>12350</RequestId>\n
+        \ </ResponseMetadata>\n</DescribeAutoScalingInstancesResponse>\n"
+    http_version: 
+  recorded_at: Tue, 27 Sep 2016 05:47:42 GMT
+- request:
+    method: post
+    uri: https://elasticloadbalancing.us-east-1.amazonaws.com/
+    body:
+      encoding: UTF-8
+      string: Action=DescribeInstanceHealth&LoadBalancerName=myELB&Timestamp=2016-09-27T05%3A47%3A42Z&Version=2012-06-01
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ! "<DescribeInstanceHealthResponse xmlns=\"http://elasticloadbalancing.amazonaws.com/doc/2012-06-01/\">\n
+        \ <DescribeInstanceHealthResult>\n    <InstanceStates>\n      <member>\n        <Description>N/A</Description>\n
+        \       <InstanceId>instance1</InstanceId>\n        <ReasonCode>N/A</ReasonCode>\n
+        \       <State>InService</State>\n      </member>\n      <member>\n        <Description>N/A</Description>\n
+        \       <InstanceId>instance2</InstanceId>\n        <ReasonCode>N/A</ReasonCode>\n
+        \       <State>InService</State>\n      </member>\n      <member>\n        <Description>N/A</Description>\n
+        \       <InstanceId>instance3</InstanceId>\n        <ReasonCode>N/A</ReasonCode>\n
+        \       <State>InService</State>\n      </member>\n      <member>\n        <Description>N/A</Description>\n
+        \       <InstanceId>instance4</InstanceId>\n        <ReasonCode>N/A</ReasonCode>\n
+        \       <State>InService</State>\n      </member>\n    </InstanceStates>\n
+        \ </DescribeInstanceHealthResult>\n  <ResponseMetadata>\n    <RequestId>12351</RequestId>\n
+        \ </ResponseMetadata>\n</DescribeInstanceHealthResponse>\n"
+    http_version: 
+  recorded_at: Tue, 27 Sep 2016 05:47:42 GMT
+- request:
+    method: post
+    uri: https://autoscaling.us-east-1.amazonaws.com/
+    body:
+      encoding: UTF-8
+      string: Action=DescribeAutoScalingInstances&InstanceIds.member.1=instance4&Timestamp=2016-09-27T05%3A47%3A42Z&Version=2011-01-01
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ! "<DescribeAutoScalingInstancesResponse xmlns=\"http://autoscaling.amazonaws.com/doc/2011-01-01/\">\n
+        \ <DescribeAutoScalingInstancesResult>\n    <AutoScalingInstances>\n      <member>\n
+        \       <LaunchConfigurationName>myLaunchConfig</LaunchConfigurationName>\n
+        \       <LifecycleState>InService</LifecycleState>\n        <AutoScalingGroupName>myASG</AutoScalingGroupName>\n
+        \       <InstanceId>instance4</InstanceId>\n        <HealthStatus>HEALTHY</HealthStatus>\n
+        \       <ProtectedFromScaleIn>false</ProtectedFromScaleIn>\n        <AvailabilityZone>us-east-1e</AvailabilityZone>\n
+        \     </member>\n    </AutoScalingInstances>\n  </DescribeAutoScalingInstancesResult>\n
+        \ <ResponseMetadata>\n    <RequestId>12352</RequestId>\n
+        \ </ResponseMetadata>\n</DescribeAutoScalingInstancesResponse>\n"
+    http_version: 
+  recorded_at: Tue, 27 Sep 2016 05:47:42 GMT
+- request:
+    method: post
+    uri: https://elasticloadbalancing.us-east-1.amazonaws.com/
+    body:
+      encoding: UTF-8
+      string: Action=DescribeInstanceHealth&LoadBalancerName=myELB&Timestamp=2016-09-27T05%3A47%3A42Z&Version=2012-06-01
+  response:
+    status:
+      code: 200
+      message: OK
+    body:
+      encoding: US-ASCII
+      string: ! "<DescribeInstanceHealthResponse xmlns=\"http://elasticloadbalancing.amazonaws.com/doc/2012-06-01/\">\n
+        \ <DescribeInstanceHealthResult>\n    <InstanceStates>\n      <member>\n        <Description>N/A</Description>\n
+        \       <InstanceId>instance1</InstanceId>\n        <ReasonCode>N/A</ReasonCode>\n
+        \       <State>InService</State>\n      </member>\n      <member>\n        <Description>N/A</Description>\n
+        \       <InstanceId>instance2</InstanceId>\n        <ReasonCode>N/A</ReasonCode>\n
+        \       <State>InService</State>\n      </member>\n      <member>\n        <Description>N/A</Description>\n
+        \       <InstanceId>instance3</InstanceId>\n        <ReasonCode>N/A</ReasonCode>\n
+        \       <State>InService</State>\n      </member>\n      <member>\n        <Description>N/A</Description>\n
+        \       <InstanceId>instance4</InstanceId>\n        <ReasonCode>N/A</ReasonCode>\n
+        \       <State>InService</State>\n      </member>\n    </InstanceStates>\n
+        \ </DescribeInstanceHealthResult>\n  <ResponseMetadata>\n    <RequestId>12353</RequestId>\n
+        \ </ResponseMetadata>\n</DescribeInstanceHealthResponse>\n"
+    http_version: 
+  recorded_at: Tue, 27 Sep 2016 05:47:42 GMT
+recorded_with: VCR 2.9.3

data/lib/cf_deployer/deployment_strategy/auto_scaling_group_swap.rb

@@ -2,12 +2,25 @@ module CfDeployer
   module DeploymentStrategy
     class AutoScalingGroupSwap < BlueGreen
 
+      def cool_inactive_on_failure
+        yield
+      rescue => e
+        if both_stacks_active?
+          Log.error "Deployment failed - #{e.message} - and both stacks are active.  Cooling down failed stack.  Look into the failure, and try your deployment again."
+          cool_down(inactive_stack)
+        end
+
+        raise e
+      end
+
       def deploy
         check_blue_green_not_both_active 'Deployment'
         Log.info "Found active stack #{active_stack.name}" if active_stack
         delete_stack inactive_stack
-        create_inactive_stack
-        swap_group
+        cool_inactive_on_failure do
+          create_inactive_stack
+          swap_group
+        end
         run_hook(:'after-swap')
         Log.info "Active stack has been set to #{inactive_stack.name}"
         delete_stack(active_stack) if active_stack && !keep_previous_stack
@@ -22,8 +35,8 @@ module CfDeployer
 
       def switch
         check_blue_green_not_both_active 'Switch'
-        raise ApplicationError.new('Only one color stack exists, cannot switch to a non-existent version!') unless both_stacks_exist?
-        swap_group true
+        raise ApplicationError.new('Both stacks must exist to switch.') unless both_stacks_exist?
+        cool_inactive_on_failure { swap_group true }
       end
 
       private
@@ -35,7 +48,7 @@ module CfDeployer
 
       def swap_group is_switching_to_cooled = false
         is_switching_to_cooled ? warm_up_cooled_stack : warm_up_inactive_stack
-        cool_down_active_stack if active_stack && (is_switching_to_cooled || keep_previous_stack)
+        cool_down(active_stack) if active_stack && (is_switching_to_cooled || keep_previous_stack)
       end
 
       def keep_previous_stack
@@ -56,8 +69,8 @@ module CfDeployer
         warm_up_stack(inactive_stack, active_stack, true)
       end
 
-      def cool_down_active_stack
-        get_active_asgs(active_stack).each do |id|
+      def cool_down stack
+        get_active_asgs(stack).each do |id|
           asg_driver(id).cool_down
         end
       end

data/lib/cf_deployer/driver/auto_scaling_group.rb

@@ -3,7 +3,7 @@ module CfDeployer
     class AutoScalingGroup
       extend Forwardable
 
-      def_delegators :aws_group, :auto_scaling_instances, :ec2_instances, :load_balancers
+      def_delegators :aws_group, :auto_scaling_instances, :ec2_instances, :load_balancers, :desired_capacity
 
       attr_reader :group_name, :group
 
@@ -23,7 +23,7 @@ module CfDeployer
 
         CfDeployer::Driver::DryRun.guard "Skipping ASG warmup" do
           aws_group.set_desired_capacity desired
-          wait_for_healthy_instance desired
+          wait_for_desired_capacity
         end
       end
 
@@ -50,37 +50,54 @@ module CfDeployer
         instance_info
       end
 
-      private
-
-      def healthy_instance_count
-        begin
-          count = auto_scaling_instances.count do |instance|
-            instance.health_status == 'HEALTHY' && (load_balancers.empty? || instance_in_service?( instance.ec2_instance ))
+      def wait_for_desired_capacity
+        Timeout::timeout(@timeout){
+          until desired_capacity_reached?
+            sleep 15
           end
-          Log.info "Healthy instance count: #{count}"
-          count
-        rescue => e
-          Log.info "Unable to determine healthy instance count due to error: #{e.message}"
-          -1
-        end
+        }
       end
 
-      def instance_in_service? instance
-        load_balancers.all? do |load_balancer|
-          load_balancer.instances.health.any? do |health|
-            health[:instance] == instance ? health[:state] == 'InService' : false
-          end
+      def desired_capacity_reached?
+        healthy_instance_count >= desired_capacity
+      end
+
+      def healthy_instance_ids
+        instances = auto_scaling_instances.select do |instance|
+          'HEALTHY'.casecmp(instance.health_status) == 0
         end
+        instances.map(&:id)
       end
 
-      def wait_for_healthy_instance number
-        Timeout::timeout(@timeout){
-          while healthy_instance_count != number
-            sleep 15
-          end
-        }
+      def in_service_instance_ids
+        elbs = load_balancers
+        return [] if elbs.empty?
+
+        ids = elbs.collect(&:instances)
+                  .collect(&:health)
+                  .to_a
+                  .collect { |elb_healths|
+                     elb_healths.select { |health| health[:state] == 'InService' }
+                                .map { |health| health[:instance].id }
+                  }
+
+        ids.inject(:&)
       end
 
+      def healthy_instance_count
+        AWS.memoize do
+          instances = healthy_instance_ids
+          instances &= in_service_instance_ids unless load_balancers.empty?
+          Log.info "Healthy instance count: #{instances.count}"
+          instances.count
+        end
+      rescue => e
+        Log.error "Unable to determine healthy instance count due to error: #{e.message}"
+        -1
+      end
+
+      private
+
       def aws_group
         @my_group ||= AWS::AutoScaling.new.groups[group_name]
       end

data/lib/cf_deployer/logger.rb

@@ -11,6 +11,10 @@ module CfDeployer
       log.info message
     end
 
+    def self.error(message)
+      log.error message
+    end
+
     def self.log
       return @log if @log
       @log = Logger.new('cf_deployer')

data/lib/cf_deployer/version.rb

@@ -1,3 +1,3 @@
 module CfDeployer
-  VERSION = "1.5.0"
+  VERSION = "1.6.0"
 end

data/spec/aws_call_count/driver/auto_scaling_group_spec.rb

@@ -0,0 +1,19 @@
+require 'aws_call_count_spec_helper'
+
+describe CfDeployer::Driver::AutoScalingGroup do
+  it 'makes the minimum number of calls to AWS when there are 4 instances in the ASG' do
+    asg = 'myASG'
+
+    override_aws_environment(AWS_REGION: 'us-east-1') do
+      logs = nil
+      allow(CfDeployer::Log).to receive(:error) { |message| logs = message }
+      driver = CfDeployer::Driver::AutoScalingGroup.new asg
+      VCR.use_cassette("aws_call_count/driver/auto_scaling_group/healthy_instance_count") do
+        expect(driver.send(:healthy_instance_count)).to equal(4), "Logs: #{logs}"
+      end
+
+      expect(WebMock).to have_requested(:post, "https://autoscaling.us-east-1.amazonaws.com/").times(1)
+      expect(WebMock).to have_requested(:post, "https://elasticloadbalancing.us-east-1.amazonaws.com/").times(1)
+    end
+  end
+end

data/spec/aws_call_count_spec_helper.rb

@@ -0,0 +1,38 @@
+require 'spec_helper'
+require 'webmock/rspec'
+require 'vcr'
+
+VCR.configure do |config|
+  config.cassette_library_dir = "fixtures/vcr_cassettes"
+  config.hook_into :webmock
+end
+
+def override_aws_environment options = {}
+  options[:AWS_REGION] ||= 'us-east-1'
+  options[:AWS_ACCESS_KEY_ID] ||= 'someId'
+  options[:AWS_SECRET_ACCESS_KEY] ||= 'secretKey'
+
+  override_environment_variables(options) { yield }
+end
+
+def override_environment_variables options = {}
+  previous_values = override_previous_values(options)
+
+  yield
+
+  restore_values(previous_values)
+end
+
+def override_previous_values options = {}
+  previous_values = options.inject([]) do |memo, (key,value)|
+    memo << { key: key.to_s, value: value, existed: ENV.has_key?(key.to_s), old_value: ENV[key.to_s] }
+    ENV[key.to_s] = value
+    memo
+  end
+end
+
+def restore_values previous_values = []
+  previous_values.each do |value|
+    value[:existed] ? ENV[value[:key]] = value[:old_value] : ENV.delete(value[:key])
+  end
+end

data/spec/spec_helper.rb

@@ -3,7 +3,20 @@ Dir.glob("#{File.dirname File.absolute_path(__FILE__)}/fakes/*.rb") { |file| req
 
 CfDeployer::Log.log.outputters = nil
 
+RSPEC_LOG = Logger.new(STDOUT)
+RSPEC_LOG.level = Logger::WARN
+
+if ENV['DEBUG']
+  RSPEC_LOG.level = Logger::DEBUG
+  AWS.config :logger => RSPEC_LOG
+end
+
 def puts *args
 
 end
 
+def ignore_errors
+  yield
+rescue => e
+  RSPEC_LOG.debug "Intentionally ignoring error: #{e.message}"
+end

data/spec/unit/deployment_strategy/auto_scaling_group_swap_spec.rb

@@ -127,6 +127,114 @@ describe 'Auto Scaling Group Swap Deployment Strategy' do
     end
   end
 
+  context 'on deployment failure' do
+    context 'in stack creation' do
+      context 'and only one stack is active' do
+        it 'should not cool down the only available stack' do
+          strategy = create_strategy(blue: :active)
+          error = RuntimeError.new("Error before inactive_stack became active")
+
+          expect(strategy).to receive(:create_inactive_stack).and_raise(error)
+          expect(strategy).to_not receive(:cool_down)
+
+          ignore_errors { strategy.deploy }
+        end
+
+        it 'should report the deployment as a failure' do
+          strategy = create_strategy(blue: :active)
+          error = RuntimeError.new("Error before inactive_stack became active")
+
+          expect(strategy).to receive(:create_inactive_stack).and_raise(error)
+          expect { strategy.deploy }.to raise_error(error)
+        end
+      end
+
+      context 'and both stacks are active' do
+        it 'should cool down inactive stack' do
+          strategy = create_strategy(blue: :active)
+          inactive_stack = strategy.send(:green_stack)
+
+          expect(strategy).to receive(:create_inactive_stack) do
+            activate_stack(inactive_stack)
+            raise RuntimeError.new("Error after inactive_stack became active")
+          end
+
+          expect(strategy).to receive(:cool_down).with(inactive_stack)
+          ignore_errors { strategy.deploy }
+        end
+
+        it 'should report the deployment as a failure' do
+          strategy = create_strategy(blue: :active)
+          inactive_stack = strategy.send(:green_stack)
+          error = RuntimeError.new("Error after inactive_stack became active")
+
+          expect(strategy).to receive(:create_inactive_stack) do
+            activate_stack(inactive_stack)
+            raise error
+          end
+
+          allow(strategy).to receive(:cool_down)
+          expect { strategy.deploy }.to raise_error(error)
+        end
+      end
+    end
+
+    context 'in asg swap' do
+      # This shouldn't be possible - a stack normally becomes active at creation
+      context 'and only one stack is active' do
+        it 'should not cool down the only available stack' do
+          strategy = create_strategy(blue: :active)
+          error = RuntimeError.new("Error during swap")
+
+          # The stack would normally be active after create_inactive_stack
+          allow(strategy).to receive(:create_inactive_stack)
+          expect(strategy).to receive(:swap_group).and_raise(error)
+          expect(strategy).to_not receive(:cool_down)
+
+          ignore_errors { strategy.deploy }
+        end
+
+        it 'should report the deployment as a failure' do
+          strategy = create_strategy(blue: :active)
+          error = RuntimeError.new("Error during swap")
+
+          # The stack would normally be active after create_inactive_stack
+          allow(strategy).to receive(:create_inactive_stack)
+          expect(strategy).to receive(:swap_group).and_raise(error)
+          expect(strategy).to_not receive(:cool_down)
+
+          expect { strategy.deploy }.to raise_error(error)
+        end
+      end
+
+      context 'and both stacks are active' do
+        it 'should cool down inactive stack' do
+          strategy = create_strategy(blue: :active)
+          inactive_stack = strategy.send(:green_stack)
+          error = RuntimeError.new("Error during swap")
+
+          allow(strategy).to receive(:create_inactive_stack) { activate_stack(inactive_stack) }
+          expect(strategy).to receive(:swap_group).and_raise(error)
+          expect(strategy).to receive(:cool_down).with(inactive_stack)
+
+          ignore_errors { strategy.deploy }
+        end
+
+        it 'should report the deployment as a failure' do
+          strategy = create_strategy(blue: :active)
+          inactive_stack = strategy.send(:green_stack)
+          error = RuntimeError.new("Error during swap")
+
+          allow(strategy).to receive(:create_inactive_stack) { activate_stack(inactive_stack) }
+          expect(strategy).to receive(:swap_group).and_raise(error)
+          expect(strategy).to receive(:cool_down).with(inactive_stack)
+
+          expect { strategy.deploy }.to raise_error(error)
+        end
+      end
+    end
+  end
+
   context 'has active group' do
     it 'should deploy blue stack if green stack is active' do
       blue_stack.live!
@@ -306,62 +414,169 @@ describe 'Auto Scaling Group Swap Deployment Strategy' do
   describe '#switch' do
     context 'both stacks are active' do
       it 'should raise an error' do
-        green_stack.live!
-        blue_stack.live!
-        allow(CfDeployer::Driver::AutoScalingGroup).to receive(:new).with('greenASG') { green_asg_driver }
-        allow(CfDeployer::Driver::AutoScalingGroup).to receive(:new).with('blueASG') { blue_asg_driver }
-        allow(green_asg_driver).to receive(:describe) { {desired: 1, min: 1, max: 3 } }
-        allow(blue_asg_driver).to receive(:describe) { {desired: 1, min: 1, max: 3 } }
+        strategy = create_strategy(blue: :active, green: :active)
+        error = 'Found both auto-scaling-groups, ["greenASG", "blueASG"], in green and blue stacks are active. Switch aborted!'
 
-        strategy = CfDeployer::DeploymentStrategy.create(app, env, component, context)
-        expect{strategy.switch}.to raise_error 'Found both auto-scaling-groups, ["greenASG", "blueASG"], in green and blue stacks are active. Switch aborted!'
+        expect { strategy.switch }.to raise_error(error)
       end
     end
 
     context 'both stacks do not exist' do
-      it 'should raise an error' do
-        green_stack.live!
-        blue_stack.die!
-        allow(CfDeployer::Driver::AutoScalingGroup).to receive(:new).with('greenASG') { green_asg_driver }
-        allow(CfDeployer::Driver::AutoScalingGroup).to receive(:new).with('blueASG') { blue_asg_driver }
-        allow(green_asg_driver).to receive(:describe) { {desired: 1, min: 1, max: 3 } }
+      let(:error) { 'Both stacks must exist to switch.' }
+
+      context '(only green exists)' do
+        it 'should raise an error' do
+          strategy = create_strategy(green: :active, blue: :dead)
+
+          expect { strategy.switch }.to raise_error(error)
+        end
+      end
+
+      context '(only blue exists)' do
+        it 'should raise an error' do
+          strategy = create_strategy(green: :dead, blue: :active)
+
+          expect { strategy.switch }.to raise_error(error)
+        end
+      end
 
-        strategy = CfDeployer::DeploymentStrategy.create(app, env, component, context)
-        expect{ strategy.switch }.to raise_error 'Only one color stack exists, cannot switch to a non-existent version!'
+      context '(no stack exists)' do
+        it 'should raise an error' do
+          strategy = create_strategy(green: :dead, blue: :dead)
+
+          expect { strategy.switch }.to raise_error(error)
+        end
       end
     end
 
     context 'green stack is active' do
+      let(:strategy) { create_strategy(green: :active, blue: :inactive) }
+
       it 'should warm up blue stack and cool down green stack' do
-        green_stack.live!
-        blue_stack.live!
-        allow(CfDeployer::Driver::AutoScalingGroup).to receive(:new).with('blueASG') { blue_asg_driver }
-        allow(CfDeployer::Driver::AutoScalingGroup).to receive(:new).with('greenASG') { green_asg_driver }
-        options = {desired: 5, min: 3, max: 7}
-        allow(green_asg_driver).to receive(:describe) {options}
-        allow(blue_asg_driver).to receive(:describe) {{desired: 0, min: 0, max: 0}}
+        active_stack = strategy.send(:green_stack)
+        inactive_stack = strategy.send(:blue_stack)
+
+        expect(strategy).to receive(:warm_up_stack).with(inactive_stack, active_stack, true)
+        expect(strategy).to receive(:cool_down).with(active_stack)
+
+        strategy.switch
+      end
+
+      context 'swap fails' do
+        context 'before blue stack becomes active' do
+          let(:error) { 'Error before inactive stack becomes active' }
+
+          it 'does not cool down any stack' do
+            expect(strategy).to receive(:warm_up_stack).and_raise(error)
+            expect(strategy).to_not receive(:cool_down)
+
+            ignore_errors { strategy.switch }
+          end
 
-        expect(blue_asg_driver).to receive(:warm_up_cooled_group).with(options)
-        expect(green_asg_driver).to receive(:cool_down)
+          it 'reports the switch as a failure' do
+            expect(strategy).to receive(:warm_up_stack).and_raise(error)
+            allow(strategy).to receive(:cool_down)
+
+            expect { strategy.switch }.to raise_error(error)
+          end
+        end
+
+        context 'after both stacks became active' do
+          let(:error) { 'Error after inactive stack becomes active ' }
+
+          it 'cools down the blue stack' do
+            expect(strategy).to receive(:warm_up_stack) do
+              expect(strategy).to receive(:both_stacks_active?).and_return(true)
+              raise error
+            end
+
+            inactive_stack = strategy.send(:blue_stack)
+            expect(strategy).to receive(:cool_down).with(inactive_stack)
+
+            ignore_errors { strategy.switch }
+          end
+
+          it 'reports the switch as a failure' do
+            expect(strategy).to receive(:warm_up_stack) do
+              expect(strategy).to receive(:both_stacks_active?).and_return(true)
+              raise error
+            end
+
+            expect { strategy.switch }.to raise_error(error)
+          end
+        end
+      end
+    end
+
+    context 'blue stack is active' do
+      let(:strategy) { create_strategy(green: :inactive, blue: :active) }
+
+      it 'should warm up green stack and cool down blue stack' do
+        active_stack = strategy.send(:blue_stack)
+        inactive_stack = strategy.send(:green_stack)
+
+        expect(strategy).to receive(:warm_up_stack).with(inactive_stack, active_stack, true)
+        expect(strategy).to receive(:cool_down).with(active_stack)
 
-        strategy = CfDeployer::DeploymentStrategy.create(app, env, component, context)
         strategy.switch
       end
+
+      context 'swap fails' do
+        context 'before green stack becomes active' do
+          let(:error) { 'Error before inactive stack becomes active' }
+
+          it 'does not cool down any stack' do
+            expect(strategy).to receive(:warm_up_stack).and_raise(error)
+            expect(strategy).to_not receive(:cool_down)
+
+            ignore_errors { strategy.switch }
+          end
+
+          it 'reports the switch as a failure' do
+            expect(strategy).to receive(:warm_up_stack).and_raise(error)
+            allow(strategy).to receive(:cool_down)
+
+            expect { strategy.switch }.to raise_error(error)
+          end
+        end
+
+        context 'after both stacks become active' do
+          let(:error) { 'Error after inactive stack becomes active ' }
+
+          it 'cools down the green stack' do
+            expect(strategy).to receive(:warm_up_stack) do
+              expect(strategy).to receive(:both_stacks_active?).and_return(true)
+              raise error
+            end
+
+            inactive_stack = strategy.send(:green_stack)
+            expect(strategy).to receive(:cool_down).with(inactive_stack)
+
+            ignore_errors { strategy.switch }
+          end
+
+          it 'reports the switch as a failure' do
+            expect(strategy).to receive(:warm_up_stack) do
+              expect(strategy).to receive(:both_stacks_active?).and_return(true)
+              raise error
+            end
+
+            expect { strategy.switch }.to raise_error(error)
+          end
+        end
+      end
     end
   end
 
-  context '#cool_down_active_stack' do
+  context '#cool_down' do
     it 'should cool down only those ASGs which actually exist' do
       blue_stack.live!
-      green_stack.die!
-      allow(CfDeployer::Driver::AutoScalingGroup).to receive(:new).with('greenASG') { green_asg_driver }
       allow(CfDeployer::Driver::AutoScalingGroup).to receive(:new).with('blueASG') { blue_asg_driver }
-      allow(green_asg_driver).to receive(:describe) { {desired: 0, min: 0, max: 0 } }
       allow(blue_asg_driver).to receive(:describe) { {desired: 1, min: 1, max: 3 } }
 
       strategy = CfDeployer::DeploymentStrategy.create(app, env, component, context)
       expect(blue_asg_driver).to receive(:cool_down)
-      strategy.send(:cool_down_active_stack)
+      strategy.send(:cool_down, blue_stack)
     end
   end
 
@@ -488,4 +703,66 @@ describe 'Auto Scaling Group Swap Deployment Strategy' do
       asg_swap.send(:stack_active?, blue_stack).should be(true)
     end
   end
+
+  def default_options
+    {
+      app_name: 'app',
+      environment: 'environment',
+      component: 'component',
+      context: {
+        :'deployment-strategy' => 'auto-scaling-group-swap',
+        :settings => {
+          :'auto-scaling-group-name-output' => ['AutoScalingGroupID']
+        }
+      }
+    }
+  end
+
+  def create_strategy original_options = {}
+    options = default_options.merge(original_options)
+
+    create_stack(:blue, options.delete(:blue) || :dead, options)
+    create_stack(:green, options.delete(:green) || :dead, options)
+
+    CfDeployer::DeploymentStrategy.create(options[:app_name], options[:environment], options[:component], options[:context])
+  end
+
+  def create_stack color, status = :active, original_options = {}
+    options = default_options.merge(original_options)
+
+    stack = Fakes::Stack.new(name: color.to_s, outputs: {'web-elb-name' => "#{color}-elb"}, parameters: { name: color.to_s})
+
+    stack_color_name = (color.to_s == 'green' ? 'G' : 'B')
+    stack_name = "#{options[:app_name]}-#{options[:environment]}-#{options[:component]}-#{stack_color_name}"
+    allow(CfDeployer::Stack).to receive(:new).with(stack_name, options[:component], options[:context]).and_return(stack)
+
+    stack.tap do
+      case status
+      when :active; activate_stack(stack)
+      when :inactive; activate_stack(stack, { desired: 0, max: 0, min: 0 })
+      when :dead; kill_stack(stack)
+      else raise "Trying to create stack with unknown status; #{status}"
+      end
+    end
+  end
+
+  def activate_stack stack, instances = {}
+    stack.live!
+
+    allow(stack).to receive(:output).with('AutoScalingGroupID').and_return("#{stack.name}ASG")
+    allow(stack).to receive(:find_output).with('AutoScalingGroupID').and_return("#{stack.name}ASG")
+    allow(stack).to receive(:resource_statuses).and_return(asg_ids("#{stack.name}ASG"))
+
+    asg_driver = double("#{stack.name}_asg_driver")
+    instances[:desired] ||= 2
+    instances[:min] ||= 1
+    instances[:max] ||= 5
+
+    allow(asg_driver).to receive(:describe).and_return(instances)
+    allow(CfDeployer::Driver::AutoScalingGroup).to receive(:new).with("#{stack.name}ASG") { asg_driver }
+  end
+
+  def kill_stack stack
+    stack.die!
+  end
 end

data/spec/unit/driver/auto_scaling_group_spec.rb

@@ -27,79 +27,152 @@ describe 'Autoscaling group driver' do
 
   describe '#warm_up' do
     it 'should warm up the group to the desired size' do
-      expect(group).to receive(:auto_scaling_instances){[instance1, instance2]}
       expect(group).to receive(:set_desired_capacity).with(2)
+      expect(@driver).to receive(:wait_for_desired_capacity)
       @driver.warm_up 2
     end
 
     it 'should wait for the warm up of the group even if desired is the same as the minimum' do
-      expect(group).to receive(:auto_scaling_instances){[instance2]}
       expect(group).to receive(:set_desired_capacity).with(1)
+      expect(@driver).to receive(:wait_for_desired_capacity)
       @driver.warm_up 1
     end
 
     it 'should ignore warming up if desired number is less than min size of the group' do
       expect(group).not_to receive(:set_desired_capacity)
+      expect(@driver).not_to receive(:wait_for_desired_capacity)
       @driver.warm_up 0
     end
 
     it 'should warm up to maximum if desired number is greater than maximum size of group' do
-      expect(group).to receive(:auto_scaling_instances){[instance1, instance2, instance3, instance4]}
       expect(group).to receive(:set_desired_capacity).with(4)
+      expect(@driver).to receive(:wait_for_desired_capacity)
       @driver.warm_up 5
     end
   end
 
-  describe '#healthy_instance_count' do
-    it 'should respond with the number of instances that are HEALTHY' do
-      instance5 =  double('instance1', :health_status => 'UNHEALTHY')
-      allow(group).to receive(:auto_scaling_instances){[instance1, instance2, instance3, instance4, instance5]}
-      expect(@driver.send(:healthy_instance_count)).to eql 4
+  describe '#healthy_instance_ids' do
+    it 'returns the ids of all instances that are healthy' do
+      instance1 =  double('instance1', :health_status => 'HEALTHY', id: 'instance1')
+      instance2 =  double('instance2', :health_status => 'HEALTHY', id: 'instance2')
+      instance3 =  double('instance3', :health_status => 'UNHEALTHY', id: 'instance3')
+      instance4 =  double('instance4', :health_status => 'HEALTHY', id: 'instance4')
+      allow(group).to receive(:auto_scaling_instances){[instance1, instance2, instance3, instance4]}
+
+      expect(@driver.healthy_instance_ids).to eql ['instance1', 'instance2', 'instance4']
     end
 
-    it 'health check should be resilient against intermittent errors' do
-      instance5 = double('instance5')
-      expect(instance5).to receive(:health_status).and_raise(StandardError)
-      allow(group).to receive(:auto_scaling_instances){ [ instance5 ] }
-      expect(@driver.send(:healthy_instance_count)).to eql -1
+    it 'returns the ids of all instances that are healthy (case insensitive)' do
+      instance1 =  double('instance1', :health_status => 'HealThy', id: 'instance1')
+      allow(group).to receive(:auto_scaling_instances){[instance1]}
+
+      expect(@driver.healthy_instance_ids).to eql ['instance1']
     end
+  end
 
-    context 'when an elb is associated with the auto scaling group' do
-      it 'should not include instances that are HEALTHY but not associated with the elb' do
-        instance_collection = double('instance_collection', :health => [{:instance => ec2_instance1, :state => 'InService'}])
-        load_balancer = double('load_balancer', :instances => instance_collection)
-        allow(group).to receive(:load_balancers) { [load_balancer] }
-        allow(group).to receive(:auto_scaling_instances) { [instance1, instance2] }
+  describe '#in_service_instance_ids' do
+    context 'when there are no load balancers' do
+      it 'returns no ids' do
+        allow(group).to receive(:load_balancers).and_return([])
 
-        expect(@driver.send(:healthy_instance_count)).to eql 1
+        expect(@driver.in_service_instance_ids).to eq []
       end
+    end
 
-      it 'should only include instances registered with an elb that are InService' do
-        allow(group).to receive(:auto_scaling_instances) { [instance1, instance2, instance3] }
-        instance_collection = double('instance_collection', :health => [{:instance => ec2_instance1, :state => 'InService'},
-                                                                        {:instance => ec2_instance2, :state => 'OutOfService'},
-                                                                        {:instance => ec2_instance3, :state => 'OutOfService'}])
-        load_balancer = double('load_balancer', :instances => instance_collection)
-        allow(group).to receive(:load_balancers) { [load_balancer] }
+    context 'when there is only 1 elb' do
+      it 'returns the ids of all instances that are in service' do
+        health1 = { state: 'InService', instance: double('i1', id: 'instance1') }
+        health2 = { state: 'OutOfService', instance: double('i2', id: 'instance2') }
+        health3 = { state: 'InService', instance: double('i3', id: 'instance3') }
+        health4 = { state: 'InService', instance: double('i4', id: 'instance4') }
 
-        expect(@driver.send(:healthy_instance_count)).to eql 1
+        instance_collection = double('instance_collection', health: [health1, health2, health3, health4])
+        elb = double('elb', instances: instance_collection)
+        allow(group).to receive(:load_balancers).and_return([ elb ])
+
+        expect(@driver.in_service_instance_ids).to eql ['instance1', 'instance3', 'instance4']
       end
     end
 
-    context 'when there are multiple elbs for an auto scaling group' do
-      it 'should not include instances that are not registered with all load balancers' do
-        instance_collection1 = double('instance_collection1', :health => [{:instance => ec2_instance1, :state => 'InService'}])
-        instance_collection2 = double('instance_collection2', :health => [])
-        load_balancer1 = double('load_balancer1', :instances => instance_collection1)
-        load_balancer2 = double('load_balancer2', :instances => instance_collection2)
-        allow(group).to receive(:load_balancers) { [load_balancer1, load_balancer2] }
-        allow(group).to receive(:auto_scaling_instances) { [instance1] }
+    context 'when there are multiple elbs' do
+      it 'returns only the ids of instances that are in all ELBs' do
+        health1 = { state: 'InService', instance: double('i1', id: 'instance1') }
+        health2 = { state: 'InService', instance: double('i2', id: 'instance2') }
+        health3 = { state: 'InService', instance: double('i3', id: 'instance3') }
+        health4 = { state: 'InService', instance: double('i4', id: 'instance4') }
+        health5 = { state: 'InService', instance: double('i5', id: 'instance5') }
+
+        instance_collection1 = double('instance_collection1', health: [health1, health2, health3])
+        instance_collection2 = double('instance_collection2', health: [health2, health3, health4])
+        instance_collection3 = double('instance_collection3', health: [health2, health3, health5])
+        elb1 = double('elb1', instances: instance_collection1)
+        elb2 = double('elb2', instances: instance_collection2)
+        elb3 = double('elb3', instances: instance_collection3)
+        allow(group).to receive(:load_balancers).and_return([ elb1, elb2, elb3 ])
+
+        # Only instance 2 and 3 are associated with all ELB's
+        expect(@driver.in_service_instance_ids).to eql ['instance2', 'instance3']
+      end
+
+      it 'returns only the ids instances that are InService in all ELBs' do
+        health11 = { state: 'OutOfService', instance: double('i1', id: 'instance1') }
+        health12 = { state: 'InService', instance: double('i2', id: 'instance2') }
+        health13 = { state: 'InService', instance: double('i3', id: 'instance3') }
+
+        health21 = { state: 'InService', instance: double('i1', id: 'instance1') }
+        health22 = { state: 'InService', instance: double('i2', id: 'instance2') }
+        health23 = { state: 'OutOfService', instance: double('i3', id: 'instance3') }
+
+        health31 = { state: 'InService', instance: double('i1', id: 'instance1') }
+        health32 = { state: 'InService', instance: double('i2', id: 'instance2') }
+        health33 = { state: 'InService', instance: double('i3', id: 'instance3') }
+
+        instance_collection1 = double('instance_collection1', health: [health11, health12, health13])
+        instance_collection2 = double('instance_collection2', health: [health21, health22, health23])
+        instance_collection3 = double('instance_collection3', health: [health31, health32, health33])
 
-        expect(@driver.send(:healthy_instance_count)).to eql 0
+        elb1 = double('elb1', instances: instance_collection1)
+        elb2 = double('elb2', instances: instance_collection2)
+        elb3 = double('elb3', instances: instance_collection3)
+
+        allow(group).to receive(:load_balancers).and_return([ elb1, elb2, elb3 ])
+
+        # Only instance 2 is InService across all ELB's
+        expect(@driver.in_service_instance_ids).to eql ['instance2']
       end
     end
   end
 
+  describe '#healthy_instance_count' do
+    context 'when there are no load balancers' do
+      it 'should return the number of healthy instances' do
+        healthy_instance_ids = ['1', '3', '4', '5']
+        allow(@driver).to receive(:load_balancers).and_return([])
+        expect(@driver).to receive(:healthy_instance_ids).and_return(healthy_instance_ids)
+
+        expect(@driver.healthy_instance_count).to eql(healthy_instance_ids.count)
+      end
+    end
+
+    context 'when load balancers exist' do
+      it 'should return the number of instances that are both healthy, and in service' do
+        healthy_instance_ids = ['1', '3', '4', '5']
+        in_service_instance_ids = ['3', '4']
+        allow(@driver).to receive(:load_balancers).and_return(double('elb', empty?: false))
+        expect(@driver).to receive(:healthy_instance_ids).and_return(healthy_instance_ids)
+        expect(@driver).to receive(:in_service_instance_ids).and_return(in_service_instance_ids)
+
+        # Only instances 3 and 4 are both healthy and in service
+        expect(@driver.healthy_instance_count).to eql(2)
+      end
+    end
+
+    it 'health check should be resilient against intermittent errors' do
+      expect(@driver).to receive(:healthy_instance_ids).and_raise("Some error")
+      expect(@driver.healthy_instance_count).to eql -1
+    end
+  end
+
   describe '#cool_down' do
     it 'should cool down group' do
       expect(group).to receive(:update).with({min_size: 0, max_size: 0})
@@ -113,7 +186,7 @@ describe 'Autoscaling group driver' do
       hash = {:max => 5, :min => 2, :desired => 3}
       allow(group).to receive(:auto_scaling_instances){[instance1, instance2, instance3]}
       expect(group).to receive(:update).with({:min_size => 2, :max_size => 5})
-      expect(group).to receive(:set_desired_capacity).with(3)
+      expect(@driver).to receive(:warm_up).with(3)
       @driver.warm_up_cooled_group hash
     end
   end
@@ -131,4 +204,47 @@ describe 'Autoscaling group driver' do
       expect(@driver.instance_statuses).to eq( { 'i-abcd1234' => returned_status } )
     end
   end
+
+  describe '#wait_for_desired_capacity' do
+    it 'completes if desired capacity reached' do
+      expect(@driver).to receive(:desired_capacity_reached?).and_return(true)
+
+      @driver.wait_for_desired_capacity
+    end
+
+    it 'times out if desired capacity is not reached' do
+      expect(@driver).to receive(:desired_capacity_reached?).and_return(false)
+
+      expect { @driver.wait_for_desired_capacity }.to raise_error(Timeout::Error)
+    end
+  end
+
+  describe '#desired_capacity_reached?' do
+    it 'returns true if healthy instance count matches desired capacity' do
+      expected_number = 5
+
+      expect(group).to receive(:desired_capacity).and_return(expected_number)
+      expect(@driver).to receive(:healthy_instance_count).and_return(expected_number)
+
+      expect(@driver.desired_capacity_reached?).to be_true
+    end
+
+    it 'returns false if healthy instance count is less than desired capacity' do
+      expected_number = 5
+
+      expect(group).to receive(:desired_capacity).and_return(expected_number)
+      expect(@driver).to receive(:healthy_instance_count).and_return(expected_number - 1)
+
+      expect(@driver.desired_capacity_reached?).to be_false
+    end
+
+    it 'returns true if healthy instance count is more than desired capacity' do
+      expected_number = 5
+
+      expect(group).to receive(:desired_capacity).and_return(expected_number)
+      expect(@driver).to receive(:healthy_instance_count).and_return(expected_number + 1)
+
+      expect(@driver.desired_capacity_reached?).to be_true
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cf_deployer
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.6.0
 platform: ruby
 authors:
 - Jame Brechtel
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-11 00:00:00.000000000 Z
+date: 2016-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -139,6 +139,34 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 10.3.0
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.9.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.9.3
 description: For automatic blue green deployment flow on CloudFormation.
 email:
 - jbrechtel@gmail.com
@@ -162,6 +190,7 @@ files:
 - Rakefile
 - bin/cf_deploy
 - cf_deployer.gemspec
+- fixtures/vcr_cassettes/aws_call_count/driver/auto_scaling_group/healthy_instance_count.yml
 - lib/cf_deployer.rb
 - lib/cf_deployer/application.rb
 - lib/cf_deployer/application_error.rb
@@ -188,6 +217,8 @@ files:
 - lib/cf_deployer/stack.rb
 - lib/cf_deployer/status_presenter.rb
 - lib/cf_deployer/version.rb
+- spec/aws_call_count/driver/auto_scaling_group_spec.rb
+- spec/aws_call_count_spec_helper.rb
 - spec/fakes/instance.rb
 - spec/fakes/route53_client.rb
 - spec/fakes/stack.rb
@@ -240,6 +271,8 @@ specification_version: 4
 summary: Support multiple components deployment using CloudFormation templates with
   multiple blue green strategies.
 test_files:
+- spec/aws_call_count/driver/auto_scaling_group_spec.rb
+- spec/aws_call_count_spec_helper.rb
 - spec/fakes/instance.rb
 - spec/fakes/route53_client.rb
 - spec/fakes/stack.rb