eb_deployer 0.4.2 → 0.4.3

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+0.4.3
+=====
+* Backoff and retry when AWS::ElasticBeanstalk::Errors::Throttling happens during polling events
+
 0.4.2
 =====
 * Allow provding different deploy strategy for different components.

data/lib/eb_deployer.rb

@@ -9,6 +9,7 @@ require 'erb'
 require 'fileutils'
 
 require 'eb_deployer/version'
+require 'eb_deployer/utils'
 require 'eb_deployer/aws_driver'
 require 'eb_deployer/deployment_strategy'
 require 'eb_deployer/cloud_formation_provisioner'

data/lib/eb_deployer/config_loader.rb

@@ -3,6 +3,7 @@ require 'digest'
 
 module EbDeployer
   class ConfigLoader
+    include Utils
 
     class EvalBinding
       def initialize(package_digest)
@@ -44,10 +45,6 @@ module EbDeployer
 
     private
 
-    def symbolize_keys(hash)
-      hash.inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo}
-    end
-
     def load_config_settings(config_file, package_digest)
       yaml = ERB.new(File.read(config_file)).result(eval_binding(package_digest))
       symbolize_keys(YAML.load(yaml))

data/lib/eb_deployer/environment.rb

@@ -1,5 +1,7 @@
 module EbDeployer
   class Environment
+    include Utils
+
     attr_writer :resource_stacks, :settings, :creation_opts, :components, :component_under_deploy, :strategy_name
     attr_reader :name
 
@@ -53,10 +55,6 @@ module EbDeployer
       @components.detect { |c| c.name == name }
     end
 
-    def symbolize_keys(hash)
-      hash.inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo}
-    end
-
     def resource_stack_name
       "#{app_name}-#{@name}"
     end

data/lib/eb_deployer/event_poller.rb

@@ -1,7 +1,9 @@
 module EbDeployer
   class EventPoller
-    def initialize(app, env, beanstalk)
-      @app, @env, @beanstalk = app, env, beanstalk
+    include Utils
+
+    def initialize(app, env, eb_driver)
+      @app, @env, @eb_driver = app, env, eb_driver
     end
 
     def poll(start_time = Time.now, &block)
@@ -16,7 +18,6 @@ module EbDeployer
       end
     end
 
-
     private
 
     def digest(event)
@@ -28,15 +29,21 @@ module EbDeployer
     end
 
     def fetch_events(start_time, &block)
-      events, next_token = @beanstalk.fetch_events(@app, @env, :start_time => start_time.iso8601)
+      events, next_token = fetch_events_from_eb(:start_time => start_time.iso8601)
       yield(events)
       fetch_next(next_token, &block) if next_token
     end
 
     def fetch_next(next_token, &block)
-      events, next_token = @beanstalk.fetch_events(@app, @env, :next_token => next_token)
+      events, next_token = fetch_events_from_eb(:next_token => next_token)
       yield(events)
       fetch_next(next_token, &block) if next_token
     end
+
+    def fetch_events_from_eb(options)
+      backoff(AWS::ElasticBeanstalk::Errors::Throttling) do
+        @eb_driver.fetch_events(@app, @env, options)
+      end
+    end
   end
 end

data/lib/eb_deployer/utils.rb

@@ -0,0 +1,28 @@
+module EbDeployer
+  module Utils
+    BACKOFF_INITIAL_SLEEP = 1
+
+    # A util deal with throttling exceptions
+    # example:
+    #  backoff(AWS::EC2::Errors::RequestLimitExceeded) do
+    #     ...
+    #  end
+    def backoff(error_class, retry_limit=9, &block)
+      next_sleep = BACKOFF_INITIAL_SLEEP
+      begin
+        yield
+      rescue error_class
+        raise if retry_limit == 0
+        sleep(next_sleep)
+        next_sleep *= 2
+        retry_limit -= 1
+        retry
+      end
+    end
+
+    # convert top level key in a hash to symbol
+    def symbolize_keys(hash)
+      hash.inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo}
+    end
+  end
+end

data/lib/eb_deployer/version.rb

@@ -1,3 +1,3 @@
 module EbDeployer
-  VERSION = "0.4.2"
+  VERSION = "0.4.3"
 end

data/test/blue_green_deploy_test.rb

@@ -2,55 +2,27 @@ require 'deploy_test'
 
 class BlueGreenDeployTest < DeployTest
   def test_blue_green_deployment_strategy_should_create_blue_env_on_first_deployment
-    deploy(:application => 'simple',
-           :environment => "production",
-           :strategy => 'blue-green',
-           :version_label => 42)
-
+    do_deploy(42)
     assert @eb.environment_exists?('simple', t('production-a', 'simple'))
     assert_equal 'simple-production',  @eb.environment_cname_prefix('simple', t('production-a', 'simple'))
   end
 
 
   def test_blue_green_deployment_should_create_green_env_if_blue_exists
-    deploy(:application => 'simple',
-           :environment => "production",
-           :strategy => 'blue-green',
-           :version_label => 42)
-
-    deploy(:application => 'simple',
-           :environment => "production",
-           :strategy => 'blue-green',
-           :version_label => 43)
-
+    do_deploy(42)
+    do_deploy(43)
     assert @eb.environment_exists?('simple', t('production-a', 'simple'))
     assert @eb.environment_exists?('simple', t('production-b', 'simple'))
   end
 
 
   def test_blue_green_deployment_should_swap_cname_to_make_active_most_recent_updated_env
-    deploy(:application => 'simple',
-           :environment => "production",
-           :strategy => 'blue-green',
-           :version_label => 42)
-
-    deploy(:application => 'simple',
-           :environment => "production",
-           :strategy => 'blue-green',
-           :version_label => 43)
-
+    do_deploy(42)
+    do_deploy(43)
     assert_match(/simple-production-inactive/,  @eb.environment_cname_prefix('simple', t('production-a', 'simple')))
-
     assert_equal 'simple-production',  @eb.environment_cname_prefix('simple', t('production-b', 'simple'))
-
-
-    deploy(:application => 'simple',
-           :environment => "production",
-           :strategy => 'blue-green',
-           :version_label => 44)
-
+    do_deploy(44)
     assert_match(/simple-production-inactive/,  @eb.environment_cname_prefix('simple', t('production-b', 'simple')))
-
     assert_equal 'simple-production',  @eb.environment_cname_prefix('simple', t('production-a', 'simple'))
   end
 
@@ -59,11 +31,7 @@ class BlueGreenDeployTest < DeployTest
     smoked_host = []
     smoke_test = lambda { |host| smoked_host << host }
     [42, 43, 44].each do |version_label|
-      deploy(:application => 'simple',
-             :environment => "production",
-             :strategy => 'blue-green',
-             :smoke_test => smoke_test,
-             :version_label => version_label)
+      do_deploy(version_label, :smoke_test => smoke_test)
     end
 
     assert_equal ['simple-production.elasticbeanstalk.com',
@@ -73,42 +41,22 @@ class BlueGreenDeployTest < DeployTest
 
 
   def test_blue_green_deployment_should_delete_and_recreate_inactive_env_if_phoenix_mode_is_enabled
-    deploy(:application => 'simple',
-           :environment => "production",
-           :strategy => 'blue-green',
-           :version_label => 42,
-           :phoenix_mode => true)
-
-    deploy(:application => 'simple',
-           :environment => "production",
-           :strategy => 'blue-green',
-           :version_label => 43,
-           :phoenix_mode => true)
-
+    do_deploy(42, :phoenix_mode => true)
+    do_deploy(43, :phoenix_mode => true)
     assert_equal [],  @eb.environments_been_deleted('simple')
 
     inactive_env = t('production-a', 'simple')
     assert_match(/inactive/,  @eb.environment_cname_prefix('simple', inactive_env))
 
-
-    deploy(:application => 'simple',
-           :environment => "production",
-           :strategy => 'blue-green',
-           :version_label => 44,
-           :phoenix_mode => true)
-
+    do_deploy(44, :phoenix_mode => true)
     assert_equal [inactive_env], @eb.environments_been_deleted('simple')
 
     assert_equal 'simple-production',  @eb.environment_cname_prefix('simple', inactive_env)
   end
 
   def test_destroy_should_clean_up_env
-    2.times do
-      deploy(:application => 'simple',
-             :environment => "production",
-             :strategy => 'blue-green',
-             :version_label => 44,
-             :phoenix_mode => true)
+    [42, 44].each do |version|
+      do_deploy(version)
     end
 
     destroy(:application => 'simple', :environment => 'production')
@@ -117,5 +65,13 @@ class BlueGreenDeployTest < DeployTest
   end
 
 
+  private
+
+  def do_deploy(version_label, options={})
+    deploy( {:application => 'simple',
+              :environment => "production",
+              :strategy => 'blue-green',
+            }.merge(options).merge(:version_label => version_label))
+  end
 
 end

data/test/deploy_test.rb

@@ -2,7 +2,7 @@ require 'test_helper'
 
 class DeployTest < MiniTest::Unit::TestCase
   def setup
-    @eb = EBStub.new
+    @eb = ErrorRaisingWrapper.new(EBStub.new)
     @s3_driver = S3Stub.new
     @cf_driver = CFStub.new
     @sample_package = sample_file('app-package.war')

data/test/inplace_update_deploy_test.rb

@@ -79,4 +79,22 @@ class InplaceUpdateDeployTest < DeployTest
     assert !@eb.environment_exists?('simple', t('production', 'simple'))
   end
 
+  def test_deploy_should_raise_error_when_constantly_hitting_throttling_error
+    throttling_error = AWS::ElasticBeanstalk::Errors::Throttling.new("bang!")
+    @eb.set_error(:fetch_events, throttling_error)
+    assert_raises(AWS::ElasticBeanstalk::Errors::Throttling) do
+      deploy(:application => 'simple', :environment => "production")
+    end
+  end
+
+  def test_deploy_should_retry_on_temporary_throttling_error
+    throttling_error = AWS::ElasticBeanstalk::Errors::Throttling.new("bang!")
+    error_seq = [throttling_error] * 5
+    @eb.set_error_generator(:fetch_events) do
+      error_seq.pop
+    end
+    deploy(:application => 'simple', :environment => "production")
+    assert @eb.environment_exists?('simple', t('production', 'simple'))
+  end
+
 end

data/test/test_helper.rb

@@ -6,6 +6,54 @@ require 'aws_driver_stubs'
 require 'minitest/autorun'
 require 'minitest/pride'
 
+def silence_warnings(&block)
+  old_verbose, $VERBOSE = $VERBOSE, nil
+  yield
+ensure
+  $VERBOSE = old_verbose
+end
+
+silence_warnings { EbDeployer::Utils::BACKOFF_INITIAL_SLEEP = 0 }
+
+class ErrorRaisingWrapper < SimpleDelegator
+  def initialize(stub)
+    @errors = {}
+    super(stub)
+  end
+
+  def set_error(method, error)
+    set_error_generator(method) do
+      error
+    end
+  end
+
+  def set_error_generator(method, &error_gen)
+    define_delegate_method(method)
+    @errors[method] = Proc.new(&error_gen)
+  end
+
+  private
+  def define_delegate_method(method)
+    method = method.to_s
+    original_method_name = "__#{method}_without_error"
+    raise "method #{method} not defined" unless self.respond_to?(method)
+    return if self.respond_to?(original_method_name)
+
+    self.instance_eval <<-CODE
+      def #{original_method_name}(*args, &block)
+        self.__get_obj__.send(:#{method}, *args, &block)
+      end
+
+      def #{method}(*args, &block)
+        if error_gen = @errors[:fetch_events]
+          error = error_gen.call
+          raise error if error
+        end
+        super
+      end
+    CODE
+  end
+end
 
 class MiniTest::Unit::TestCase
   def sample_file(file_name, content='s' * 100)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: eb_deployer
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.3
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-01 00:00:00.000000000 Z
+date: 2014-04-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -69,6 +69,7 @@ files:
 - lib/eb_deployer/package.rb
 - lib/eb_deployer/resource_stacks.rb
 - lib/eb_deployer/smoke_test.rb
+- lib/eb_deployer/utils.rb
 - lib/eb_deployer/version.rb
 - lib/eb_deployer/version_cleaner.rb
 - test/aws_driver_stubs.rb
@@ -99,7 +100,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3697140249092762497
+      hash: 567704794056721741
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -108,7 +109,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3697140249092762497
+      hash: 567704794056721741
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.29