hako 1.3.0 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d2aaacf745660c3193a66d0527ff2ec56dac915b
-  data.tar.gz: 3110462e23bdaf2318eb964735405e38bdcd1176
+  metadata.gz: be720c4761b90dbbae3c8d302cbe3c50421c915a
+  data.tar.gz: 40a6297d8e167ea157cb3080749d56c70ab9cbe0
 SHA512:
-  metadata.gz: 0e2f3a18e41a7cf61b25feea55feaa258d0365759d6cd24adaeb01b0b78c67cddbf6a1f94158cc2234b384506f612ed0b0645921331ccc5304ef51b8ab6ca7a0
-  data.tar.gz: d36842d199259f48306ba64fef90db984d84909971f2599e11429d2f95e6d6be511b8d6629b099cc2e825cd482f3830dfa79f6295b2a56da0017765250e63487
+  metadata.gz: b5f6863cebd0493b0d47a72bc82296490195446aa8a8fd74a2f401f0b67046aa472f14266928ceba3e71694da7aa8a5050d4e6bfa81399cdb44a552c5600e2e4
+  data.tar.gz: 13e21ae6fa1ae860a5598f8abe5910b5b8e759889ec9a439f9984b79d407588c4b43719a51f90b09eb2bd1530913f1993fb206e54bad07d172330b18c240fc7b

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 1.3.1 (2017-05-16)
+## Bug fixes
+- Retry DescribeAutoScalingGroups when rate limited
+
 # 1.3.0 (2017-05-15)
 ## New features
 - Add `oneshot_notification_prefix` option

data/lib/hako/schedulers/ecs.rb

@@ -815,6 +815,8 @@ module Hako
         raise "Unable to find rollback target. #{task_definition.task_definition_arn} is INACTIVE?"
       end
 
+      MIN_ASG_INTERVAL = 1
+      MAX_ASG_INTERVAL = 120
       # @param [Aws::ECS::Types::TaskDefinition] task_definition
       # @return [Boolean] true if the capacity is reserved
       def on_no_tasks_started(task_definition)
@@ -823,8 +825,17 @@ module Hako
         end
 
         autoscaling = Aws::AutoScaling::Client.new
+        interval = MIN_ASG_INTERVAL
         loop do
-          asg = autoscaling.describe_auto_scaling_groups(auto_scaling_group_names: [@autoscaling_group_for_oneshot]).auto_scaling_groups[0]
+          begin
+            asg = autoscaling.describe_auto_scaling_groups(auto_scaling_group_names: [@autoscaling_group_for_oneshot]).auto_scaling_groups[0]
+          rescue Aws::AutoScaling::Errors::Throttling => e
+            Hako.logger.error(e)
+            interval = [interval * 2, MAX_ASG_INTERVAL].min
+            Hako.logger.info("Retrying after #{interval} seconds...")
+            sleep interval
+            next
+          end
           unless asg
             raise Error.new("AutoScaling Group '#{@autoscaling_group_for_oneshot}' does not exist")
           end
@@ -835,11 +846,12 @@ module Hako
             return true
           end
 
+          interval = [interval / 2, MIN_ASG_INTERVAL].max
           # Check autoscaling group health
           current = asg.instances.count { |i| i.lifecycle_state == 'InService' }
           if asg.desired_capacity != current
-            Hako.logger.debug("#{asg.auto_scaling_group_name} isn't in desired state. desired_capacity=#{asg.desired_capacity} in-service instances=#{current}")
-            sleep 1
+            Hako.logger.debug("#{asg.auto_scaling_group_name} isn't in desired state. desired_capacity=#{asg.desired_capacity} in-service instances=#{current}. Retry after #{interval} seconds")
+            sleep interval
             next
           end
 
@@ -849,8 +861,8 @@ module Hako
             out_instances.delete(ci.ec2_instance_id)
           end
           unless out_instances.empty?
-            Hako.logger.debug("There's instances that is running but not registered as container instances: #{out_instances}")
-            sleep 1
+            Hako.logger.debug("There's instances that is running but not registered as container instances: #{out_instances}. Retry after #{interval} seconds")
+            sleep interval
             next
           end
 
@@ -858,6 +870,7 @@ module Hako
           desired = current + 1
           Hako.logger.info("Increment desired_capacity of #{asg.auto_scaling_group_name} from #{current} to #{desired}")
           autoscaling.set_desired_capacity(auto_scaling_group_name: asg.auto_scaling_group_name, desired_capacity: desired)
+          sleep interval
         end
       end
 

data/lib/hako/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Hako
-  VERSION = '1.3.0'
+  VERSION = '1.3.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hako
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.3.1
 platform: ruby
 authors:
 - Kohei Suzuki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-05-15 00:00:00.000000000 Z
+date: 2017-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk